hdf5

src/ifcconvert/IfcConvert.cpp

@@ -61,6 +61,7 @@ void printUsage(const boost::program_options::options_description& generic_optio
 	          << "  .igs   IGES           Initial Graphics Exchange Specification" << std::endl
 	          << "  .xml   XML            Property definitions and decomposition tree" << std::endl
 			  << "  .svg   SVG            Scalable Vector Graphics (2d floor plan)" << std::endl
+			  << "  .hdf   HDF5           Efficient binary IFC serialization" << std::endl
 	          << std::endl
 	          << "Command line options" << std::endl << generic_options << std::endl
 	          << "Advanced options" << std::endl << geom_options << std::endl;
@@ -131,6 +132,8 @@ int main(int argc, char** argv) {
 		("bounds", boost::program_options::value<std::string>(&bounds),
 			"Specifies the bounding rectangle, for example 512x512, to which the " 
 			"output will be scaled. Only used when converting to SVG.")
+		("compress",
+			"Compress HDF5.")
 		("include", 
 			"Specifies that the entities listed after --entities are to be included")
 		("exclude", 
@@ -250,6 +253,20 @@ int main(int argc, char** argv) {
 		} catch (...) {}
 		write_log();
 		return exit_code;
+	} else if (output_extension == ".hdf") {
+		int exit_code = 1;
+		// try {
+			IfcParse::IfcFile f;
+			if (!f.Init(input_filename)) {
+				Logger::Message(Logger::LOG_ERROR, "Unable to parse .ifc file");
+			} else {
+				f.write_hdf5(output_filename, vmap.count("compress") != 0);
+				exit_code = 0;
+			}
+		// } catch (...) {}
+		write_log();
+		// std::cin.get();
+		return exit_code;
 	}
 
 	IfcGeom::IteratorSettings settings;

src/ifcexpressparser/schema_class.py

@@ -65,6 +65,10 @@ class SchemaClass:
             for name in collection.keys():
                 statements.append('%(cpp_type)s* %(name)s_type = 0;' % locals())
                 
+        statements.append('#ifdef _MSC_VER'          )
+        statements.append('#pragma optimize("", off)')
+        statements.append('#endif'                   )
+                          
         statements.append('schema_definition* populate_schema() {')
         
         emitted_types = set()
@@ -145,6 +149,10 @@ class SchemaClass:
         
         statements.extend(('}',''))
         
+        statements.append('#ifdef _MSC_VER'         )
+        statements.append('#pragma optimize("", on)')
+        statements.append('#endif'                  )
+        
         statements.extend(('const schema_definition& get_schema() {',
                            '',
                            '    static const schema_definition* s = populate_schema();',

src/ifcparse/Ifc2x3-schema.cpp

@@ -1009,6 +1009,11 @@ enumeration_type* IfcWindowPanelPositionEnum_type = 0;
 enumeration_type* IfcWindowStyleConstructionEnum_type = 0;
 enumeration_type* IfcWindowStyleOperationEnum_type = 0;
 enumeration_type* IfcWorkControlTypeEnum_type = 0;
+
+#ifdef _MSC_VER
+#  pragma optimize( "", off )
+#endif
+
 schema_definition* populate_schema() {
     IfcAbsorbedDoseMeasure_type = new type_declaration(IfcSchema::Type::IfcAbsorbedDoseMeasure, new simple_type(simple_type::real_type));
     IfcAccelerationMeasure_type = new type_declaration(IfcSchema::Type::IfcAccelerationMeasure, new simple_type(simple_type::real_type));
@@ -10374,6 +10379,10 @@ schema_definition* populate_schema() {
     return new schema_definition("IFC2X3", declarations, true);
 }
 
+#ifdef _MSC_VER
+#  pragma optimize( "", on )
+#endif
+
 const schema_definition& get_schema() {
 
     static const schema_definition* s = populate_schema();

src/ifcparse/IfcFile.h

@@ -135,6 +135,8 @@ public:
 	std::pair<IfcSchema::IfcNamedUnit*, double> getUnit(IfcSchema::IfcUnitEnum::IfcUnitEnum);
 
 	const schema_definition* schema() const { return schema_; }
+
+	void write_hdf5(const std::string&, bool compress) const;
 };
 
 }

src/ifcparse/IfcHdf5File.cpp

@@ -0,0 +1,671 @@
+#include "IfcHdf5File.h"
+
+#include "H5pubconf.h"
+
+#ifndef H5_HAVE_FILTER_DEFLATE
+#pragma message("warning: HDF5 compression support is recommended")
+#endif
+
+class UnmetDependencyException : public std::exception {};
+
+H5::DataType* IfcParse::IfcHdf5File::map_type(const IfcParse::parameter_type* pt) {
+	H5::DataType* h5_dt;
+	if (pt->as_aggregation_type()) {
+		h5_dt = new H5::VarLenType(map_type(pt->as_aggregation_type()->type_of_element()));		
+	} else if (pt->as_named_type()) {
+		if (pt->as_named_type()->declared_type()->as_entity()) {
+			h5_dt = new H5::DataType();
+			h5_dt->copy(*instance_reference);
+		} else {
+			IfcSchema::Type::Enum ty = pt->as_named_type()->declared_type()->type();
+			if (declared_types.find(ty) == declared_types.end()) {
+				throw UnmetDependencyException();
+			} else {
+				h5_dt = new H5::DataType();
+				h5_dt->copy(*declared_types[ty]);
+			}
+		}
+	} else if (pt->as_simple_type()) {
+		const H5::DataType* orig = default_types[pt->as_simple_type()->declared_type()];
+		h5_dt = new H5::DataType();
+		h5_dt->copy(*orig);
+	} else {
+		throw UnmetDependencyException();
+	}
+	return h5_dt;
+}
+
+H5::CompType* create_compound(const std::vector< IfcParse::IfcHdf5File::compound_member >& members) {
+	size_t s = 0, o = 0;
+	for (auto it = members.begin(); it != members.end(); ++it) {
+		s += it->second->getSize();
+	}
+
+	H5::CompType* h5_dt = new H5::CompType(s);
+	for (auto it = members.begin(); it != members.end(); ++it) {
+		h5_dt->insertMember(it->first, o, *it->second);
+		o += it->second->getSize();
+	}
+
+	return h5_dt;
+}
+
+H5::EnumType* create_enumeration(const std::vector<std::string>& items, int offset = 0) {
+	size_t numbytes = 0;
+	size_t size = items.size();
+	while (size != 0) {
+		size >>= 8;
+		numbytes ++;
+	}
+
+	int i = offset;
+	H5::EnumType* h5_enum = new H5::EnumType(numbytes);
+	for (auto it = items.begin(); it != items.end(); ++it, ++i) {
+		h5_enum->insert(it->c_str(), &i);
+	}
+
+	return h5_enum;
+}
+
+H5::EnumType* map_enumeration(const IfcParse::enumeration_type* en) {
+	return create_enumeration(en->enumeration_items());
+}
+
+H5::DataType* IfcParse::IfcHdf5File::commit(H5::DataType* dt, const std::string& name) {
+	dt->commit(schema_group, name);
+	return dt;
+}
+
+H5::CompType* IfcParse::IfcHdf5File::map_entity(const IfcParse::entity* e) {
+	std::vector<const IfcParse::entity::attribute*> attributes = e->all_attributes();
+	
+	std::vector< IfcParse::IfcHdf5File::compound_member > h5_attributes;
+	h5_attributes.reserve(attributes.size() + 2);
+
+	h5_attributes.push_back(std::make_pair(std::string("set_unset_bitmap"), &H5::PredType::NATIVE_INT16));
+    h5_attributes.push_back(std::make_pair(std::string("Entity-Instance-Identifier"), &H5::PredType::NATIVE_INT32));
+	
+	const std::vector<bool>& attributes_derived_in_subtype = e->derived();
+	auto jt = attributes_derived_in_subtype.begin();
+
+	for (auto it = attributes.begin(); it != attributes.end(); ++it, ++jt) {
+		if (*jt) {
+			continue;
+		}
+		const std::string& name = (*it)->name();
+		const bool is_optional = (*it)->optional();
+		H5::DataType* type = map_type((*it)->type_of_attribute());
+		h5_attributes.push_back(std::make_pair(name, type));
+	}
+
+	return create_compound(h5_attributes);
+}
+
+void IfcParse::IfcHdf5File::init_default_types() {
+	{
+		std::vector< compound_member > members;
+		members.push_back( std::make_pair(std::string("_HDF5_dataset_index_"),  new H5::PredType(H5::PredType::NATIVE_INT16)) );
+		members.push_back( std::make_pair(std::string("_HDF5_instance_index_"), new H5::PredType(H5::PredType::NATIVE_INT32)) );
+		instance_reference = static_cast<H5::CompType*>(commit(create_compound(members), "_HDF_INSTANCE_REFERENCE_HANDLE_"));
+	}
+
+	object_reference_type = &H5::PredType::STD_REF_OBJ;
+
+	{
+		std::vector<std::string> names;
+		names.push_back("BOOLEAN-FALSE");
+		names.push_back("BOOLEAN-TRUE");
+		default_types[simple_type::boolean_type] = create_enumeration(names);
+	}
+
+	{
+		std::vector<std::string> names;
+		names.push_back("LOGICAL-UNKNOWN");
+		names.push_back("LOGICAL-FALSE");
+		names.push_back("LOGICAL-TRUE");
+		default_types[simple_type::logical_type] = create_enumeration(names, -1);
+	}
+
+	default_types[simple_type::binary_type]  = &H5::PredType::NATIVE_OPAQUE; // vlen?
+	default_types[simple_type::real_type]    = &H5::PredType::NATIVE_DOUBLE;
+	default_types[simple_type::number_type]  = &H5::PredType::NATIVE_DOUBLE;
+	default_types[simple_type::string_type]  = new H5::StrType(H5::PredType::C_S1, H5T_VARIABLE);
+	default_types[simple_type::integer_type] = &H5::PredType::NATIVE_INT;
+	
+	default_type_names[simple_type::logical_type] = "logical";
+	default_type_names[simple_type::boolean_type] = "boolean";
+	default_type_names[simple_type::binary_type]  = "binary";
+	default_type_names[simple_type::real_type]    = "real";
+	default_type_names[simple_type::number_type]  = "number";
+	default_type_names[simple_type::string_type]  = "string";
+	default_type_names[simple_type::integer_type] = "integer";
+
+	default_cpp_type_names[IfcUtil::Argument_BOOL]   = "boolean";
+	default_cpp_type_names[IfcUtil::Argument_DOUBLE] = "real";
+	default_cpp_type_names[IfcUtil::Argument_STRING] = "string";
+	default_cpp_type_names[IfcUtil::Argument_INT]    = "integer";
+}
+
+void IfcParse::IfcHdf5File::visit_select(const IfcParse::select_type* pt, std::set<const IfcParse::declaration*>& leafs) {
+	for (auto it = pt->select_list().begin(); it != pt->select_list().end(); ++it) {
+		if ((*it)->as_select_type()) {
+			visit_select((*it)->as_select_type(), leafs);
+		} else {
+			leafs.insert(*it);
+		}
+	}
+}
+
+std::string IfcParse::IfcHdf5File::flatten_aggregate_name(const IfcParse::parameter_type* at) const {
+	if (at->as_aggregation_type()) {
+		return std::string("aggregate-of-") + flatten_aggregate_name(at->as_aggregation_type()->type_of_element());
+	} else if (at->as_simple_type()) {
+		return default_type_names.find(at->as_simple_type()->declared_type())->second;
+	} else if (at->as_named_type()) {
+		// TODO: Unwind type name
+		return at->as_named_type()->declared_type()->name();
+	} else {
+		throw;
+	}
+}
+
+H5::DataType* IfcParse::IfcHdf5File::map_select(const IfcParse::select_type* pt) {
+	std::set<const IfcParse::declaration*> leafs;
+	visit_select(pt, leafs);
+
+	bool all_entity_instance_refs = true;
+	for (auto it = leafs.begin(); it != leafs.end(); ++it) {
+		if (!(*it)->as_entity()) {
+			all_entity_instance_refs = false;
+			break;
+		}
+	}
+
+	if (all_entity_instance_refs) {
+		return 0;
+	} else {
+		std::set<std::string> member_names;
+		std::vector<compound_member> h5_attributes;
+		
+		h5_attributes.push_back(std::make_pair(std::string("select_bitmap"), &H5::PredType::NATIVE_INT8));
+		h5_attributes.push_back(std::make_pair(std::string("type_path"), default_types[simple_type::string_type]));
+
+		for (auto it = leafs.begin(); it != leafs.end(); ++it) {
+			std::string name;
+			const H5::DataType* dt = 0;
+
+			const IfcParse::declaration* decl = (*it);
+
+			if ((*it)->as_type_declaration()) {
+				while (decl->as_type_declaration()) {
+					const IfcParse::parameter_type* pt = decl->as_type_declaration()->declared_type();
+					const IfcParse::named_type* nt = pt->as_named_type();
+					const IfcParse::simple_type* st = pt->as_simple_type();
+					const IfcParse::aggregation_type* at = pt->as_aggregation_type();
+					if (nt) {
+						decl = nt->declared_type();
+					} else if (st) {
+						name = default_type_names[st->declared_type()];
+						dt = default_types[st->declared_type()];
+						break;
+					} else if (at) {
+						name = flatten_aggregate_name(at);
+						dt = map_type(at);
+						break;
+					}
+				}
+			}
+
+			if (!dt) {
+				if ((*it)->as_entity()) {
+					name = "instance";
+					dt = instance_reference;
+				} else if ((*it)->as_enumeration_type()) {
+					name = (*it)->name();
+					dt = declared_types[(*it)->type()];
+				} else {
+					throw;
+				}
+			}
+
+			if (member_names.find(name) != member_names.end()) {
+				continue;
+			}
+			member_names.insert(name);
+
+			name += "-value";
+			h5_attributes.push_back(std::make_pair(name, dt));
+		}
+
+		return create_compound(h5_attributes);
+	}
+}
+
+void IfcParse::IfcHdf5File::advance(void*& ptr, size_t n) const {
+	ptr = ( uint8_t*) ptr + n;
+}
+
+template <typename T>
+void IfcParse::IfcHdf5File::write(void*& ptr, const T& t) const {
+	*((T*)ptr) = t;
+	advance(ptr, sizeof(T));
+}
+
+template <>
+void IfcParse::IfcHdf5File::write(void*& ptr, const std::string& s) const {
+	char* c = new(allocator.allocate(s.size()+1)) char [s.size()+1];
+	strcpy(c, s.c_str());
+	write(ptr, c);
+}
+
+template <unsigned int T> struct uint_of_size     {};
+template <>               struct uint_of_size <1> { typedef  uint8_t type; };
+template <>               struct uint_of_size <2> { typedef uint16_t type; };
+template <>               struct uint_of_size <4> { typedef uint32_t type; };
+template <>               struct uint_of_size <8> { typedef uint64_t type; };
+
+template <unsigned int T> struct int_of_size     {};
+template <>               struct int_of_size <1> { typedef  int8_t type; };
+template <>               struct int_of_size <2> { typedef int16_t type; };
+template <>               struct int_of_size <4> { typedef int32_t type; };
+template <>               struct int_of_size <8> { typedef int64_t type; };
+
+template <unsigned int T> struct float_of_size     {};
+template <>               struct float_of_size <4> { typedef  float type; };
+template <>               struct float_of_size <8> { typedef double type; };
+
+template <typename T>
+void IfcParse::IfcHdf5File::write_number_of_size(void*& ptr, size_t n, T i) const {
+	if (!std::numeric_limits<T>::is_integer) {
+		if (n == 4) {
+			write(ptr, static_cast< float_of_size<4>::type > (i));
+		} else if (n == 8) {
+			write(ptr, static_cast< float_of_size<8>::type > (i));
+		}
+	} else if (std::numeric_limits<T>::is_signed) {
+		if (n == 1) {
+			write(ptr, static_cast< int_of_size<1>::type > (i));
+		} else if (n == 2) {
+			write(ptr, static_cast< int_of_size<2>::type > (i));
+		} else if (n == 4) {
+			write(ptr, static_cast< int_of_size<4>::type > (i));
+		} else if (n == 8) {
+			write(ptr, static_cast< int_of_size<8>::type > (i));
+		}
+	} else {
+		if (n == 1) {
+			write(ptr, static_cast< uint_of_size<1>::type > (i));
+		} else if (n == 2) {
+			write(ptr, static_cast< uint_of_size<2>::type > (i));
+		} else if (n == 4) {
+			write(ptr, static_cast< uint_of_size<4>::type > (i));
+		} else if (n == 8) {
+			write(ptr, static_cast< uint_of_size<8>::type > (i));
+		}
+	}
+}
+
+void IfcParse::IfcHdf5File::write_vlen_t(void*& ptr, size_t n_elements, void* vlen_data) const {
+	advance(ptr, HOFFSET(hvl_t, len));
+	void* temp_ptr;
+	write_number_of_size(temp_ptr = ptr, sizeof(size_t), n_elements);
+	advance(ptr, HOFFSET(hvl_t, p));
+	write(ptr, vlen_data);
+}
+
+template <typename T>
+void IfcParse::IfcHdf5File::write_aggregate(void*& ptr, const T& ts) const {
+	size_t elem_size = get_datatype<typename T::value_type>()->getSize();
+	size_t n_elements = ts.size();
+	size_t size_in_bytes = elem_size * n_elements;
+	void* aggr_data = allocator.allocate(size_in_bytes);
+	void* aggr_ptr = aggr_data;
+	for (T::const_iterator it = ts.begin(); it != ts.end(); ++it) {
+		write_number_of_size(aggr_ptr, elem_size, *it);
+	}
+	write_vlen_t(ptr, n_elements, aggr_data);
+}
+
+template <>
+void IfcParse::IfcHdf5File::write_aggregate(void*& ptr, const std::vector<std::string>& ts) const {
+	size_t elem_size = sizeof(char*);
+	size_t n_elements = ts.size();
+	size_t size_in_bytes = elem_size * n_elements;
+	void* aggr_data = allocator.allocate(size_in_bytes);
+	void* aggr_ptr = aggr_data;
+	for (std::vector<std::string>::const_iterator it = ts.begin(); it != ts.end(); ++it) {
+		write(aggr_ptr, *it);
+	}
+	write_vlen_t(ptr, n_elements, aggr_data);
+}
+
+template <>
+void IfcParse::IfcHdf5File::write_aggregate(void*& ptr, const IfcEntityList::ptr& ts) const {
+	size_t elem_size = instance_reference->getSize();
+	size_t n_elements = ts->size();
+	size_t size_in_bytes = elem_size * n_elements;
+	void* aggr_data = allocator.allocate(size_in_bytes);
+	void* aggr_ptr = aggr_data;
+	for (IfcEntityList::it it = ts->begin(); it != ts->end(); ++it) {
+		auto ref = make_instance_reference(*it);
+		write_number_of_size(aggr_ptr, instance_reference->getMemberDataType(0).getSize(), ref.first);
+		write_number_of_size(aggr_ptr, instance_reference->getMemberDataType(1).getSize(), ref.second);
+	}
+	write_vlen_t(ptr, n_elements, aggr_data);
+}
+
+void IfcParse::IfcHdf5File::write_schema(const IfcParse::schema_definition& schema) {
+
+	// From h5ex_g_compact.c
+	// Compact groups?
+	H5::FileAccPropList* plist = new H5::FileAccPropList();
+	plist->setLibverBounds(H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
+
+	// H5::FileCreatPropList* plist2 = new H5::FileCreatPropList();
+	// H5Pset_file_space(plist2->getId(), H5F_FILE_SPACE_VFD, (hsize_t)0);
+	
+	file = new H5::H5File(name_, H5F_ACC_TRUNC, H5::FileCreatPropList::DEFAULT, *plist);
+
+	schema_group = file->createGroup(schema.name() + "_encoding");
+	population_group = file->createGroup("population");
+	
+	init_default_types();
+			
+	for (auto it = schema.enumeration_types().begin(); it != schema.enumeration_types().end(); ++it) {
+		declared_types[(*it)->type()] = commit(map_enumeration((*it)), (*it)->name());
+	}
+		
+	const std::vector<const type_declaration*>& ts = schema.type_declarations();
+	std::set<const type_declaration*> processed;
+	while (processed.size() < ts.size()) {
+		for (auto it = ts.begin(); it != ts.end(); ++it) {
+			const std::string& name = (*it)->name();
+			if (processed.find(*it) != processed.end()) continue;
+			try {
+				declared_types[(*it)->type()] = commit(map_type((*it)->declared_type()), (*it)->name());
+				processed.insert(*it);
+			} catch(const UnmetDependencyException&) {}
+			auto pt = (*it)->declared_type();
+		}
+	}
+
+	for (auto it = schema.select_types().begin(); it != schema.select_types().end(); ++it) {
+		H5::DataType* dt = map_select(*it);
+		if (dt) {
+			declared_types[(*it)->type()] = commit(dt, (*it)->name());
+		} else {
+			declared_types[(*it)->type()] = instance_reference;
+		}
+	}
+		
+	for (auto it = schema.entities().begin(); it != schema.entities().end(); ++it) {
+		declared_types[(*it)->type()] = commit(map_entity(*it), (*it)->name());
+	}
+};
+
+std::pair<size_t, size_t> IfcParse::IfcHdf5File::make_instance_reference(const IfcUtil::IfcBaseClass* instance) const {
+	const std::vector<IfcUtil::IfcBaseClass*>& es = sorted_entities.find(instance->declaration().type())->second;
+	
+	auto dataset_id = std::lower_bound(dataset_names.begin(), dataset_names.end(), instance->declaration().type());
+	auto instance_id = std::lower_bound(es.begin(), es.end(), instance); // TODO: << make a vector of ids to make sure pointers are stable
+	
+	size_t dataset_offset = std::distance(dataset_names.begin(), dataset_id);
+	size_t instance_offset = std::distance(es.begin(), instance_id);
+
+	return std::make_pair(dataset_offset, instance_offset);
+}
+
+void IfcParse::IfcHdf5File::write_select(void*& ptr, IfcUtil::IfcBaseClass* instance, const H5::CompType* datatype) const {
+	int member_index = -1;
+
+	if (instance->declaration().as_entity()) {
+		member_index = datatype->getMemberIndex("instance-value");
+		size_t offset = datatype->getMemberOffset(member_index);
+		auto ref = make_instance_reference(instance);
+		void* ptr_member = (uint8_t*) ptr + offset;
+		write_number_of_size(ptr_member, instance_reference->getMemberDataType(0).getSize(), ref.first);
+		write_number_of_size(ptr_member, instance_reference->getMemberDataType(1).getSize(), ref.second);
+	} else {
+		Argument& wrapped_data = *instance->data().getArgument(0);
+		IfcUtil::ArgumentType ty = wrapped_data.type();
+		if (default_cpp_type_names.find(ty) == default_cpp_type_names.end()) {
+			Logger::Message(Logger::LOG_ERROR, "Unsupported select valuation encountered", instance);
+		} else {
+			std::string member_name = default_cpp_type_names.find(ty)->second + "-value";
+			member_index = datatype->getMemberIndex(member_name);
+			size_t offset = datatype->getMemberOffset(member_index);
+			size_t member_size = datatype->getMemberDataType(member_index).getSize();
+			void* ptr_member = (uint8_t*) ptr + offset;
+			switch(wrapped_data.type()) {
+				case IfcUtil::Argument_BOOL: {
+					bool instance = wrapped_data;
+					write_number_of_size(ptr_member, member_size, static_cast<uint8_t>(instance ? 1 : 0));
+					break; }
+				case IfcUtil::Argument_DOUBLE: {
+					double d = wrapped_data;
+					write_number_of_size(ptr_member, member_size, d);
+					break; }
+				case IfcUtil::Argument_STRING: {
+					std::string s = wrapped_data;
+					write(ptr_member, s);
+					break; }
+				case IfcUtil::Argument_INT: {
+					int i = wrapped_data;
+					write_number_of_size(ptr_member, member_size, i);
+					break; }
+				default:
+					Logger::Message(Logger::LOG_ERROR, "Unsupported select valuation encountered", instance);
+					break;
+			}
+		}							
+	}
+
+	if (member_index == -1) {
+		member_index = 0;
+	} else {
+		member_index -= 2; // select_bitmap, type_path
+	}
+
+	void* temp_ptr = ptr;
+	write_number_of_size(temp_ptr, H5::PredType::NATIVE_INT8.getSize(), member_index);
+	write(temp_ptr, instance->declaration().name());
+
+	// TODO: Should string be set to "", or keep as null?
+	// std::cout << datatype->getMemberIndex("string-value") << std::endl;
+
+	advance(ptr, datatype->getSize());
+}
+
+void IfcParse::IfcHdf5File::write_population(const IfcFile& f, bool compress) {
+	std::set<IfcSchema::Type::Enum> tys;
+	
+	for (auto it = f.begin(); it != f.end(); ++it) {
+		IfcSchema::Type::Enum ty = it->second->declaration().type();
+		tys.insert(ty);
+		// This already is sorted on entity instance name
+		sorted_entities[ty].push_back(it->second);
+	}
+
+	dataset_names.assign(tys.begin(), tys.end());
+	std::sort(dataset_names.begin(), dataset_names.end());
+	
+	for (auto it = dataset_names.begin(); it != dataset_names.end(); ++it) {
+		const std::vector<IfcUtil::IfcBaseClass*>& es = sorted_entities.find(*it)->second;
+		hsize_t dims = es.size();
+		// don't chunk too small
+		// hsize_t chunk = (std::min)(std::max(es.size() / 64, (size_t)8), es.size());
+		// hsize_t chunk = (std::min)(  (std::max)(es.size() / 8, (size_t) 128 ) , es.size());
+		hsize_t chunk = dims;
+
+		const H5::DSetCreatPropList* plist;
+		// H5O_MESG_MAX_SIZE = 65536
+		const bool compact = es.size() * declared_types[*it]->getSize() < (1 << 15);
+		if (compress && !compact) { 
+			H5::DSetCreatPropList* plist_ = new H5::DSetCreatPropList;
+			plist_->setChunk(1, &chunk);
+			// D'oh. Order is significant, according to h5ex_d_shuffle.c
+			plist_->setShuffle();
+			plist_->setDeflate(9);
+			plist = plist_;
+		} else if (compact) {
+			H5::DSetCreatPropList* plist_ = new H5::DSetCreatPropList;
+			// Set compact according to h5ex_d_compact.c
+			plist_->setLayout(H5D_COMPACT);
+			plist = plist_;
+		} else {
+			plist = &H5::DSetCreatPropList::DEFAULT;
+		}
+
+		H5::DataSet ds = population_group.createDataSet(IfcSchema::Type::ToString(*it) + "_instances", *declared_types[*it], H5::DataSpace(1, &dims), *plist);
+		
+		void* data = allocator.allocate(declared_types[*it]->getSize() * static_cast<size_t>(dims));
+		
+		void* ptr = data;
+		const H5::CompType* dt = (H5::CompType*) declared_types[*it];
+		int ind = 0;
+
+		for (auto jt = es.begin(); jt != es.end(); ++jt, ++ind) {
+			int member_idx = 0;
+
+			H5::DataType member_type;
+
+			IfcAbstractEntity& dat = (*jt)->data();
+			const declaration& decl = (*jt)->declaration();			
+			std::vector<const IfcParse::entity::attribute*> attributes = decl.as_entity()->all_attributes();
+			const std::vector<bool>& attributes_derived_in_subtype = decl.as_entity()->derived();
+			std::vector<bool>::const_iterator is_derived = attributes_derived_in_subtype.begin();
+
+			member_type = dt->getMemberDataType(member_idx++);
+			uint32_t set_unset_mask = 0;
+			void* set_unset_ptr = ptr;
+			size_t set_unset_size = member_type.getSize();
+			// skip for now write later
+			advance(ptr, set_unset_size);
+
+			member_type = dt->getMemberDataType(member_idx++);
+			write_number_of_size(ptr, member_type.getSize(), static_cast<unsigned int>(dat.id()));
+			
+			for (unsigned i = 0; i < dat.getArgumentCount(); ++i, ++is_derived) {
+				if (*is_derived) {
+					continue;
+				}
+
+				const IfcParse::entity::attribute* schema_attr = attributes[i];
+				Argument& attr_value = *dat.getArgument(i);
+				member_type = dt->getMemberDataType(member_idx++);
+
+				if (attr_value.isNull()) {
+					if (member_type == *default_types[simple_type::string_type]) {
+						// HDF5 otherwise crashes on derefencing a zero pointer for the string type
+						write(ptr, new(allocator.allocate(1)) char(0));
+					} else {
+						memset(ptr, 0, member_type.getSize());
+						advance(ptr, member_type.getSize());
+					}
+				} else {
+					set_unset_mask |= 1 << i;
+
+					if (member_type == *instance_reference) {
+						IfcUtil::IfcBaseClass* v = attr_value;
+						auto ref = make_instance_reference(v);
+						write_number_of_size(ptr, instance_reference->getMemberDataType(0).getSize(), ref.first);
+						write_number_of_size(ptr, instance_reference->getMemberDataType(1).getSize(), ref.second);
+					} else if (member_type == *default_types[simple_type::real_type]) {
+						double d = attr_value;
+						write_number_of_size(ptr, member_type.getSize(), d);
+					} else if (member_type == *default_types[simple_type::string_type]) {
+						std::string s = attr_value;
+						write(ptr, s);
+					} else if (member_type == *default_types[simple_type::integer_type]) {
+						int i = attr_value;
+						write_number_of_size(ptr, member_type.getSize(), i);
+					} else if (member_type == *default_types[simple_type::boolean_type] ||
+					           member_type == *default_types[simple_type::logical_type]) 
+					{
+						bool b = attr_value;
+						write_number_of_size(ptr, member_type.getSize(), static_cast<uint8_t>(b ? 1 : 0));
+					} else if (member_type.getClass() == H5T_ENUM) {
+						// NB: Note that boolean and logical are also enums
+						// NB2: In IfcOpenShell an enum value can be read as a string
+						std::string s = attr_value;
+						const std::vector<std::string>& enum_values = schema_attr->type_of_attribute()->as_named_type()->declared_type()->as_enumeration_type()->enumeration_items();
+						size_t d = std::distance(enum_values.begin(), std::find(enum_values.begin(), enum_values.end(), s));
+						write_number_of_size(ptr, member_type.getSize(), d);
+					} else if (member_type.getClass() == H5T_VLEN) {
+						const parameter_type* pt = schema_attr->type_of_attribute();
+						while (pt->as_named_type()) {
+							pt = pt->as_named_type()->declared_type()->as_type_declaration()->declared_type();
+						}
+						const named_type* nt = pt->as_aggregation_type()->type_of_element()->as_named_type();
+						if (nt && nt->declared_type()->as_select_type()) {
+							const H5::DataType* datatype = declared_types[nt->declared_type()->type()];
+							IfcEntityList::ptr es = attr_value;
+							if (datatype == instance_reference) {
+								// For a SELECTs with only ENTITY leaves, a blind instance reference type is used
+								write_aggregate(ptr, es);
+							} else {
+								size_t size_in_bytes = datatype->getSize();
+								size_t num_elements = es->size();
+								void* buffer_ptr;
+								// void* buffer = buffer_ptr = new uint8_t[size_in_bytes * num_elements];
+								void* buffer = buffer_ptr = allocator.allocate(size_in_bytes * num_elements);
+								for (auto it = es->begin(); it != es->end(); ++it) {
+									write_select(buffer_ptr, *it, static_cast<const H5::CompType*>(datatype));
+								}
+								write_vlen_t(ptr, num_elements, buffer);
+							}
+
+						} else {
+							if (sizeof(hvl_t) != member_type.getSize()) throw;
+							switch(attr_value.type()) {
+							case IfcUtil::Argument_AGGREGATE_OF_INT: {
+								std::vector<int> ds = attr_value;
+								write_aggregate(ptr, ds);
+								break; }
+							case IfcUtil::Argument_AGGREGATE_OF_DOUBLE: {
+								std::vector<double> ds = attr_value;
+								write_aggregate(ptr, ds);
+								break; }
+							case IfcUtil::Argument_AGGREGATE_OF_ENTITY_INSTANCE: {
+								IfcEntityList::ptr es = attr_value;
+								write_aggregate(ptr, es);
+								break; }
+							case IfcUtil::Argument_AGGREGATE_OF_STRING: {
+								std::vector<std::string> ss = attr_value;
+								write_aggregate(ptr, ss);
+								break; }
+							default:
+								// Can be an empty list in which case parser does not know type
+								if (attr_value.size() > 0) {
+									Logger::Message(Logger::LOG_ERROR, "Unsupported aggregate encountered", *jt);
+								}
+								memset(ptr, 0, member_type.getSize());
+								ptr = (uint8_t*) ptr + member_type.getSize();
+							}
+						}
+					} else if (member_type.getClass() == H5T_COMPOUND) {
+						memset(ptr, 0, member_type.getSize());
+						
+						IfcUtil::ArgumentType ty = attr_value.type();
+						if (ty != IfcUtil::Argument_ENTITY_INSTANCE) throw;
+						IfcUtil::IfcBaseClass* b = attr_value;
+
+						write_select(ptr, attr_value, static_cast<H5::CompType*>(&member_type));
+					}
+				}
+			}
+
+			write_number_of_size(set_unset_ptr, set_unset_size, set_unset_mask);			
+		}
+
+		ds.write(data, *declared_types[*it]);
+		ds.close();
+
+		allocator.free();
+		
+		for (auto it = es.begin(); it != es.end(); ++it) {
+			static_cast<IfcParse::Entity*>(&(**it).data())->Unload();
+		}
+
+	}
+}

src/ifcparse/IfcHdf5File.h

@@ -0,0 +1,122 @@
+/********************************************************************************
+ *                                                                              *
+ * This file is part of IfcOpenShell.                                           *
+ *                                                                              *
+ * IfcOpenShell is free software: you can redistribute it and/or modify         *
+ * it under the terms of the Lesser GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3.0 of the License, or          *
+ * (at your option) any later version.                                          *
+ *                                                                              *
+ * IfcOpenShell is distributed in the hope that it will be useful,              *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of               *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
+ * Lesser GNU General Public License for more details.                          *
+ *                                                                              *
+ * You should have received a copy of the Lesser GNU General Public License     *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.         *
+ *                                                                              *
+ ********************************************************************************/
+
+#ifndef IFCHDF5FILE_H
+#define IFCHDF5FILE_H
+
+#include "H5Cpp.h"
+
+#include "../ifcparse/IfcUtil.h"
+#include "../ifcparse/IfcFile.h"
+
+namespace IfcParse {
+		
+	class IfcHdf5File {
+	private:
+		class allocator_t {
+		private:
+			// TODO: Change this?
+			mutable std::vector<uint8_t*> buffers;
+		public:
+			void* allocate(size_t n) const {
+				return new uint8_t[n];
+			}
+			void free() {
+				std::vector<uint8_t*>::const_iterator it;
+				for (it = buffers.begin(); it != buffers.end(); ++it) {
+					delete[] *it;
+				}
+				buffers.clear();
+			}
+			~allocator_t() {
+				free();
+			}
+		};
+
+		allocator_t allocator;
+
+		std::string name_;
+		schema_definition schema_;
+
+		H5::H5File* file;
+		H5::Group schema_group;
+		H5::Group population_group;
+
+		// TODO: const
+		H5::CompType* instance_reference;
+		const H5::DataType* object_reference_type;
+
+		std::map<IfcSchema::Type::Enum, H5::DataType*> declared_types;
+		// TODO: why? get_datatype()
+		std::map<IfcParse::simple_type::data_type, const H5::DataType*> default_types;
+		std::map<IfcParse::simple_type::data_type, std::string> default_type_names;
+		std::map<IfcUtil::ArgumentType, std::string> default_cpp_type_names;
+
+		std::map<IfcSchema::Type::Enum, std::vector<IfcUtil::IfcBaseClass*> > sorted_entities;
+		std::vector<IfcSchema::Type::Enum> dataset_names;
+
+		H5::DataType* map_type(const IfcParse::parameter_type* pt);
+		H5::CompType* map_entity(const IfcParse::entity* e);
+		H5::DataType* map_select(const IfcParse::select_type* pt);
+		H5::DataType* commit(H5::DataType* dt, const std::string& name);
+
+		void init_default_types();
+		void visit_select(const IfcParse::select_type* pt, std::set<const IfcParse::declaration*>& leafs);
+		std::string flatten_aggregate_name(const IfcParse::parameter_type* at) const;
+		std::pair<size_t, size_t> make_instance_reference(const IfcUtil::IfcBaseClass* instance) const;
+		void write_select(void*& ptr, IfcUtil::IfcBaseClass* instance, const H5::CompType* datatype) const;
+
+		void write_schema(const schema_definition& schema);
+		void write_population(const IfcFile& f, bool compress);
+
+		template <typename T>
+		const H5::DataType* get_datatype() const {
+			return default_types.find(IfcUtil::cpp_to_schema_type<T>::schema_type)->second;
+		}
+
+		void advance(void*& ptr, size_t n) const;
+		
+		template <typename T>
+		void write(void*& ptr, const T& t) const;
+		
+		template <>
+		void write(void*& ptr, const std::string& s) const;
+
+		void write_vlen_t(void*& ptr, size_t n_elements, void* vlen_data) const;
+
+		template <typename T>
+		void write_number_of_size(void*& ptr, size_t n, T i) const;
+
+		template <typename T>
+		void write_aggregate(void*& ptr, const T& ts) const;
+	public:
+		typedef std::pair<std::string, const H5::DataType*> compound_member;
+
+		IfcHdf5File(const IfcFile* f, const std::string& name, bool compress)
+			: name_(name)
+			, schema_(*f->schema())
+		{
+			write_schema(*f->schema());
+			write_population(*f, compress);
+		};
+	};
+
+}
+
+#endif

src/ifcparse/IfcParse.cpp

@@ -38,6 +38,7 @@
 #include "../ifcparse/IfcFile.h"
 #include "../ifcparse/IfcSIPrefix.h"
 #include "../ifcparse/IfcSchema.h"
+#include "../ifcparse/IfcHdf5File.h"
 
 using namespace IfcParse;
 
@@ -830,6 +831,11 @@ void Entity::Load(std::vector<unsigned int>& ids, bool seek) const {
 	if ( ! TokenFunc::isOperator(semilocon,';') ) file->tokens->stream->Seek(old_offset);
 }
 
+void Entity::Unload() {
+	delete args;
+	args = 0;
+}
+
 IfcSchema::Type::Enum Entity::type() const {
 	return _type;
 }
@@ -1536,3 +1542,7 @@ std::pair<IfcSchema::IfcNamedUnit*, double> IfcFile::getUnit(IfcSchema::IfcUnitE
 	}
 	return return_value;
 }
+
+void IfcFile::write_hdf5(const std::string& name, bool compress) const {
+	IfcHdf5File(this, name, compress);
+}

src/ifcparse/IfcParse.h

@@ -130,7 +130,7 @@ namespace IfcParse {
 		std::vector<Argument*> list;
 		void push(Argument* l);
 	public:
-		~ArgumentList();
+		virtual ~ArgumentList();
 
 		void read(IfcSpfLexer* t, std::vector<unsigned int>& ids);
 
@@ -206,7 +206,7 @@ namespace IfcParse {
 		IfcUtil::IfcBaseClass* entity;
 	public:
 		EntityArgument(const Token& t);
-		~EntityArgument();
+		virtual ~EntityArgument();
 
 		IfcUtil::ArgumentType type() const;
 
@@ -251,6 +251,7 @@ namespace IfcParse {
 		~Entity();
 		IfcEntityList::ptr getInverse(IfcSchema::Type::Enum type, int attribute_index);
 		void Load(std::vector<unsigned int>& ids, bool seek=false) const;
+		void Unload();
 		Argument* getArgument (unsigned int i);
 		unsigned int getArgumentCount() const;
 		std::string toString(bool upper=false) const;

src/ifcparse/IfcSchema.h

@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 #include <algorithm>
+#include <iterator>
 
 #ifdef USE_IFC4
 #include "../ifcparse/Ifc4enum.h"
@@ -243,12 +244,14 @@ namespace IfcParse {
 		const std::vector<const attribute*> all_attributes() const {
 			std::vector<const attribute*> attrs;
 			attrs.reserve(derived_.size());
-			std::vector<const attribute*>::iterator it = attrs.begin();
+			// std::vector<const attribute*>::iterator it = attrs.begin();
 			if (supertype_) {
 				const std::vector<const attribute*> supertype_attrs = supertype_->all_attributes();
-				it = std::copy(supertype_attrs.begin(), supertype_attrs.end(), it);
+				// it = std::copy(supertype_attrs.begin(), supertype_attrs.end(), it);
+				std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs));
 			}
-			std::copy(attributes_.begin(), attributes_.end(), it);
+			// std::copy(attributes_.begin(), attributes_.end(), it);
+			std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs));
 			return attrs;
 		}
 
@@ -266,6 +269,7 @@ namespace IfcParse {
 		std::vector<const type_declaration*> type_declarations_;
 		std::vector<const select_type*> select_types_;
 		std::vector<const enumeration_type*> enumeration_types_;
+		std::vector<const entity*> entities_;
 
 		class declaration_by_name_cmp : public std::binary_function<const declaration*, const std::string&, bool> {
 		public:
@@ -284,6 +288,7 @@ namespace IfcParse {
 				if ((**it).as_type_declaration()) type_declarations_.push_back((**it).as_type_declaration());
 				if ((**it).as_select_type()) select_types_.push_back((**it).as_select_type());
 				if ((**it).as_enumeration_type()) enumeration_types_.push_back((**it).as_enumeration_type());
+				if ((**it).as_entity()) entities_.push_back((**it).as_entity());
 			}
 		}
 
@@ -307,10 +312,13 @@ namespace IfcParse {
 			return declaration_by_name(IfcSchema::Type::ToString(name));
 		}
 
-		const std::vector<const declaration*>& declarations() { return declarations_; }
+		const std::vector<const declaration*>& declarations() const { return declarations_; }
-		const std::vector<const type_declaration*>& type_declarations() { return type_declarations_; }
+		const std::vector<const type_declaration*>& type_declarations() const { return type_declarations_; }
-		const std::vector<const select_type*>& select_types() { return select_types_; }
+		const std::vector<const select_type*>& select_types() const { return select_types_; }
-		const std::vector<const enumeration_type*>& enumeration_types() { return enumeration_types_; }
+		const std::vector<const enumeration_type*>& enumeration_types() const { return enumeration_types_; }
+		const std::vector<const entity*>& entities() const { return entities_; }
+
+		const std::string& name() const { return name_; }
 	};
 
 }

src/ifcparse/IfcUtil.h

@@ -130,6 +130,23 @@ namespace IfcUtil {
 	};
 
 	bool valid_binary_string(const std::string& s);
+
+	template <typename T> struct cpp_to_schema_type;
+	template <>           struct cpp_to_schema_type < boost::dynamic_bitset<> > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::binary_type;  };
+	template <>           struct cpp_to_schema_type < bool                    > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::boolean_type; };
+	template <>           struct cpp_to_schema_type < int                     > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::integer_type; };
+	template <>           struct cpp_to_schema_type < double                  > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::real_type;    };
+	template <>           struct cpp_to_schema_type < std::string             > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::string_type;  };
+
+	template <IfcParse::simple_type::data_type T> 
+	            struct schema_to_cpp_type;
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::binary_type  > { typedef boost::dynamic_bitset<> cpp_type; };
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::boolean_type > { typedef bool                    cpp_type; };
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::integer_type > { typedef int                     cpp_type; };
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::logical_type > { typedef bool                    cpp_type; };
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::number_type  > { typedef double                  cpp_type; };
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::real_type    > { typedef double                  cpp_type; };
+	template <> struct schema_to_cpp_type < IfcParse::simple_type::string_type  > { typedef std::string             cpp_type; };
 }
 
 template <class T>