Work on LOD use case

src/hdf5_usecases/lod.cpp

@@ -44,6 +44,8 @@ int main(int, char** argv) {
 	IfcGeom::Kernel kernel;
 	kernel.initializeUnits(*f.entitiesByType<IfcSchema::IfcUnitAssignment>()->begin());
 
+	/*
+
 	std::set<IfcSchema::Type::Enum> geometric_types{ IfcSchema::Type::IfcStyledItem };
 	std::set<IfcSchema::Type::Enum> other_types;
 
@@ -96,15 +98,36 @@ int main(int, char** argv) {
 		other_types.begin(), other_types.end(), 
 		std::inserter(only_geometric, only_geometric.begin()));
 
+	std::cerr << "\nOnly geometric:" << std::endl;
+
 	for (auto ty : only_geometric) {
 		std::cerr << IfcSchema::Type::ToString(ty) << std::endl;
 	}
 
-	std::cin.get();
+	*/
 
-	// IfcSchema::Type::IfcAxis2Placement3D,IfcSchema::Type::IfcCartesianPoint,IfcSchema::Type::IfcDirection,IfcSchema::Type::IfcGeometricRepresentationSubContext
-		
-	std::vector<IfcUtil::IfcBaseClass*> new_defs, old_defs, old_geom_defs, all_old_defs;
+	std::set<IfcSchema::Type::Enum> only_geometric{
+		IfcSchema::Type::IfcAxis2Placement2D					,
+		IfcSchema::Type::IfcCartesianTransformationOperator3D,
+		IfcSchema::Type::IfcCircleProfileDef					,
+		IfcSchema::Type::IfcClosedShell						,
+		IfcSchema::Type::IfcConnectedFaceSet					,
+		IfcSchema::Type::IfcExtrudedAreaSolid				,
+		IfcSchema::Type::IfcFace								,
+		IfcSchema::Type::IfcFaceBasedSurfaceModel			,
+		IfcSchema::Type::IfcFaceBound						,
+		IfcSchema::Type::IfcFaceOuterBound					,
+		IfcSchema::Type::IfcFacetedBrep						,
+		//IfcSchema::Type::IfcGeometricRepresentationContext	,
+		IfcSchema::Type::IfcMappedItem						,
+		IfcSchema::Type::IfcPolyLoop							,
+		IfcSchema::Type::IfcProductDefinitionShape			,
+		IfcSchema::Type::IfcRepresentationMap				,
+		IfcSchema::Type::IfcShapeRepresentation				,
+		IfcSchema::Type::IfcStyledItem
+	};
+
+	std::vector<IfcUtil::IfcBaseClass*> new_defs, old_defs, old_geom_defs, all_old_defs, all_defs;
 
 	std::for_each(only_geometric.begin(), only_geometric.end(), [&f, &old_geom_defs](IfcSchema::Type::Enum t) {
 		auto insts = f.entitiesByType(t);
@@ -115,7 +138,7 @@ int main(int, char** argv) {
 		});
 	});
 
-	std::for_each(f.begin(), f.end(), [&fn, &only_geometric, &old_defs](const auto& pair) {
+	std::for_each(f.begin(), f.end(), [&only_geometric, &old_defs](const auto& pair) {
 		IfcUtil::IfcBaseClass* inst = pair.second;
 		auto ty = inst->declaration().type();
 		if (only_geometric.find(ty) == only_geometric.end()) {
@@ -125,7 +148,9 @@ int main(int, char** argv) {
 
 	auto id = f.FreshId();
 
-	std::for_each(defs.begin(), defs.end(), [&kernel, &id, &old_defs, &new_defs](IfcSchema::IfcProductDefinitionShape* def) {
+	int num_points_added = 0;
+
+	std::for_each(defs.begin(), defs.end(), [&kernel, &id, &old_defs, &new_defs, &num_points_added](IfcSchema::IfcProductDefinitionShape* def) {
 		Bnd_Box box;
 		auto reps = def->Representations();
 		IfcSchema::IfcRepresentationContext* context;
@@ -134,7 +159,7 @@ int main(int, char** argv) {
 			IfcGeom::IfcRepresentationShapeItems items;
 
 			if (rep->RepresentationIdentifier() == "Body") {
-				kernel.convert(rep, items);
+				// kernel.convert(rep, items);
 				context = rep->ContextOfItems();
 			}
 
@@ -143,6 +168,8 @@ int main(int, char** argv) {
 			}
 		});
 
+		box.Add(gp_Pnt(0, 0, 0));
+
 		double x1, y1, z1, x2, y2, z2;
 		box.Get(x1, y1, z1, x2, y2, z2);
 
@@ -157,10 +184,10 @@ int main(int, char** argv) {
 		
 		IfcSchema::IfcProductDefinitionShape* new_def = new IfcSchema::IfcProductDefinitionShape(boost::none, boost::none, new_reps);
 		
-		new_def->data().isWritable()->setId(id++);
+		corner->data().isWritable()->setId(id++);
 
 		// NB: Corner is added to existing definitions
-		old_defs.push_back(corner);
+		old_defs.push_back(corner); ++num_points_added;
 		new_defs.push_back(bbox);
 		new_defs.push_back(rep);
 		new_defs.push_back(new_def);
@@ -168,12 +195,10 @@ int main(int, char** argv) {
 		return new_def;
 	});
 
-	H5::H5File hdf(argv[1] + std::string(".hdf"), H5F_ACC_TRUNC);
-	
-	H5::Group population = hdf.createGroup("population");
-	H5::Group low = hdf.createGroup("low");
-	H5::Group hi = hdf.createGroup("hi");
-	
+	std::for_each(new_defs.begin(), new_defs.end(), [&id](auto new_def) {
+		new_def->data().isWritable()->setId(id++);
+	});
+
 	auto iden = [](IfcUtil::IfcBaseClass* inst) { return inst; };
 
 	std::set<IfcSchema::Type::Enum> old_new_geom_types;
@@ -188,10 +213,16 @@ int main(int, char** argv) {
 
 	all_old_defs.insert(all_old_defs.end(), old_defs.begin(), old_defs.end());
 	all_old_defs.insert(all_old_defs.end(), old_geom_defs.begin(), old_geom_defs.end());
+	all_defs.insert(all_defs.end(), all_old_defs.begin(), all_old_defs.end());
+	all_defs.insert(all_defs.end(), new_defs.begin(), new_defs.end());
 
-	if (std::distance(all_old_defs.begin(), all_old_defs.end()) != std::distance(f.begin(), f.end())) {
-		std::cerr << "Missign instances";
-		abort();
+	{
+		int num_insts1 = std::distance(all_old_defs.begin(), all_old_defs.end());
+		int num_insts2 = std::distance(f.begin(), f.end());
+		if ((num_insts1 - num_points_added) != num_insts2) {
+			std::cerr << "Missign instances " << num_insts1 << " - " << num_points_added << " != " << num_insts2 << std::endl;
+			abort();
+		}
 	}
 
 	multifile_instance_locator* locator = new multifile_instance_locator(
@@ -199,16 +230,36 @@ int main(int, char** argv) {
 		all_old_defs.begin(), all_old_defs.end());
 
 	IfcParse::Hdf5Settings settings;
-	// settings.profile() = IfcParse::Hdf5Settings::standard_referenced;
+	settings.profile() = IfcParse::Hdf5Settings::padded;
+	settings.compress() = true;
 
-	IfcParse::IfcHdf5File ifc_hdf5(hdf, get_schema(), settings);
-	ifc_hdf5.write_schema();
-	
-	IfcParse::IfcSpfHeader header;
-	header.set_default();
+	H5::H5File hdf(argv[1] + std::string(".hdf"), H5F_ACC_TRUNC);
+	H5::H5File geom_hdf(argv[1] + std::string("-geometry.hdf"), H5F_ACC_TRUNC);
+	H5::H5File box_hdf(argv[1] + std::string("-bounding-boxes.hdf"), H5F_ACC_TRUNC);
+
+	hdf.createGroup("geometry").close();
+	hdf.createGroup("bounding-boxes").close();
+
+	hdf.mount("geometry", geom_hdf, H5::PropList::DEFAULT);
+	hdf.mount("bounding-boxes", box_hdf, H5::PropList::DEFAULT);
+
+	H5::Group main_population = hdf.createGroup("population");
+	H5::Group geom_population = hdf.createGroup("geometry/population");
+	H5::Group box_population = hdf.createGroup("bounding-boxes/population");
 
 	{
-		IfcParse::instance_enumerator enumerator(&f.header(), old_defs.begin(), old_defs.end());
-		ifc_hdf5.write_population(population, enumerator, locator);
-	}	
+		IfcParse::IfcHdf5File ifc_hdf5(hdf, get_schema(), settings);
+		IfcParse::instance_categorizer categorizer(all_defs.begin(), all_defs.end());
+		ifc_hdf5.write_schema(&categorizer);
+
+		{
+			IfcParse::instance_enumerator enumerator(&f.header(), old_defs.begin(), old_defs.end());
+			ifc_hdf5.write_population(main_population, enumerator, locator);
+		}
+		
+		{
+			IfcParse::instance_enumerator enumerator(&f.header(), old_geom_defs.begin(), old_geom_defs.end());
+			ifc_hdf5.write_population(geom_population, enumerator, locator);
+		}
+	}
 }

src/hdf5_usecases/multifile_instance_locator.h

@@ -82,7 +82,9 @@ public:
 	}
 
 	template <typename ForwardIteratorTypes, typename ForwardIteratorInsts>
-	multifile_instance_locator(ForwardIteratorTypes types_all_begin, ForwardIteratorTypes types_all_end, ForwardIteratorInsts insts_begin, ForwardIteratorInsts insts_end) {
+	multifile_instance_locator(ForwardIteratorTypes types_all_begin, ForwardIteratorTypes types_all_end, ForwardIteratorInsts insts_begin, ForwardIteratorInsts insts_end)
+		: subsequent_group_(nullptr)
+	{
 		initial_population_.first.assign(types_all_begin, types_all_end);
 		std::sort(initial_population_.first.begin(), initial_population_.first.end());
 
@@ -136,7 +138,32 @@ public:
 	}
 
 	std::pair<int, int> operator()(IfcUtil::IfcBaseClass* v) {
-		throw std::runtime_error("This locator can only be used in referenced profile");
+		if (IfcSchema::Type::IsSimple(v->declaration().type())) {
+			throw std::exception("Simple type not expected here");
+		}
+
+		IfcSchema::Type::Enum t = v->declaration().type();
+
+		// This remains constant for subsequent model serializations?
+		auto& dataset_names = initial_population_.first;
+		auto tt = std::lower_bound(dataset_names.begin(), dataset_names.end(), t);
+		int a = std::distance(dataset_names.begin(), tt);
+		int b;
+
+		const std::vector<IfcUtil::IfcBaseEntity*>& insts = instances(t);
+
+		auto it = std::lower_bound(insts.begin(), insts.end(), v, [](IfcUtil::IfcBaseClass* other, IfcUtil::IfcBaseClass* val) {
+			// An ordering based on ID is not equivalent to an ordering based on pointer address!
+			return other->data().id() < val->data().id();
+		});
+
+		if (it == insts.end()) {
+			throw std::runtime_error("Unable to find instance");
+		}
+
+		b = std::distance(insts.begin(), it);
+
+		return std::make_pair(a, b);
 	}
 
 	void make_reference(IfcUtil::IfcBaseClass* v, void*& ptr) {

src/ifcparse/IfcHdf5File.cpp

@@ -15,7 +15,7 @@ static boost::optional< std::vector<Argument*> > no_instances = boost::none;
 class type_mapper {
 public:
 	type_mapper();
-	type_mapper(const IfcParse::schema_definition* schema_def, IfcParse::IfcFile* ifc_file, H5::H5File* hdf5_file, const IfcParse::Hdf5Settings& settings);
+	type_mapper(const IfcParse::schema_definition* schema_def, IfcParse::instance_categorizer* ifc_file, H5::H5File* hdf5_file, const IfcParse::Hdf5Settings& settings);
 
 	H5::DataType* commit(H5::DataType* dt, const std::string& name);
 	
@@ -34,7 +34,7 @@ private:
 	bool referenced_;
 	
 	const IfcParse::schema_definition* schema_def_;
-	IfcParse::IfcFile* ifc_file_;
+	IfcParse::instance_categorizer* ifc_file_;
 	H5::H5File* hdf5_file_;
 	H5::Group schema_group_;
 	H5::DataType* instance_reference_;
@@ -311,8 +311,6 @@ class sorted_instance_locator : public IfcParse::abstract_instance_locator {
 private:
 	H5::H5File& hdf5_file_;
 	IfcParse::instance_enumerator& file_;
-	std::vector< IfcSchema::Type::Enum > dataset_names_;
-	std::vector< std::vector<IfcUtil::IfcBaseEntity*>* > cache_;
 	bool referenced_;
 	std::string population_group_path_;
 
@@ -325,46 +323,13 @@ public:
 		population_group_path_ = population_group.getObjName() + "/";
 
 		std::set<IfcSchema::Type::Enum> dataset_names_temp;
-		for (auto it = file_.begin(); it != file_.end(); ++it) {
-			dataset_names_temp.insert(it->second->declaration().type());
-		}
-		dataset_names_.assign(dataset_names_temp.begin(), dataset_names_temp.end());
-		std::sort(dataset_names_.begin(), dataset_names_.end());
-
-		cache_.resize(IfcSchema::Type::UNDEFINED);
-
-		if (referenced_) {
-			// Why not always populate?
-			const IfcSchema::Type::Enum begin = IfcSchema::Type::Ifc2DCompositeCurve;
-			const IfcSchema::Type::Enum end = IfcSchema::Type::UNDEFINED;
-			for (int i = begin; i != end; i++) {
-				instances((IfcSchema::Type::Enum)i);
-			}
-		}
 	}
 
-	const_iterator begin() const { return dataset_names_.begin(); }
-	const_iterator end() const { return dataset_names_.end(); }
+	const_iterator begin() const { return file_.dataset_names().begin(); }
+	const_iterator end() const { return file_.dataset_names().end(); }
 
 	const std::vector<IfcUtil::IfcBaseEntity*>& instances(IfcSchema::Type::Enum t) {
-		if (cache_[t] == nullptr) {
-			auto li = file_.entitiesByType(t);
-			std::vector<IfcUtil::IfcBaseEntity*>* vs = cache_[t] = new std::vector<IfcUtil::IfcBaseEntity*>();
-			if (li) {
-				vs->reserve(li->size());
-
-				for (auto jt = li->begin(); jt != li->end(); ++jt) {
-					if ((*jt)->declaration().type() == t) {
-						vs->push_back(static_cast<IfcUtil::IfcBaseEntity*>(*jt));
-					}
-				}
-
-				std::sort(vs->begin(), vs->end(), [](IfcUtil::IfcBaseEntity* i1, IfcUtil::IfcBaseEntity* i2) {
-					return i1->data().id() < i2->data().id();
-				});
-			}
-		}
-		return *cache_[t];
+		return file_.by_type(t);
 	}
 
 	std::pair<int, int> operator()(IfcUtil::IfcBaseClass* v) {
@@ -373,9 +338,8 @@ public:
 		}
 
 		IfcSchema::Type::Enum t = v->declaration().type();
-		auto tt = std::lower_bound(dataset_names_.begin(), dataset_names_.end(), t);
-		int a = std::distance(dataset_names_.begin(), tt);
-		int b;
+		auto tt = std::lower_bound(begin(), end(), t);
+		int a = std::distance(begin(), tt);
 
 		const std::vector<IfcUtil::IfcBaseEntity*>& insts = instances(t);
 
@@ -388,7 +352,7 @@ public:
 			throw std::runtime_error("Unable to find instance");
 		}
 
-		b = std::distance(insts.begin(), it);
+		int b = std::distance(insts.begin(), it);
 		
 		return std::make_pair(a, b);
 	}
@@ -399,7 +363,7 @@ public:
 		H5::DataSpace space(1, &dims);
 		hsize_t coord = ref.second;
 		space.selectElements(H5S_SELECT_SET, 1, &coord);
-		const std::string path = population_group_path_ + IfcSchema::Type::ToString(dataset_names_[ref.first]);
+		const std::string path = population_group_path_ + IfcSchema::Type::ToString(*(begin() + ref.first));
 		hdf5_file_.reference(ptr, path + "_instances", space);
 		advance(ptr, sizeof(hdset_reg_ref_t));
 	}
@@ -858,7 +822,7 @@ type_mapper::type_mapper() {
 	default_cpp_type_names_[IfcUtil::Argument_INT] = "integer";
 }
 
-type_mapper::type_mapper(const IfcParse::schema_definition* schema_def, IfcParse::IfcFile* ifc_file, H5::H5File* hdf5_file, const IfcParse::Hdf5Settings& settings)
+type_mapper::type_mapper(const IfcParse::schema_definition* schema_def, IfcParse::instance_categorizer* ifc_file, H5::H5File* hdf5_file, const IfcParse::Hdf5Settings& settings)
 	: schema_def_(schema_def)
 	, ifc_file_(ifc_file)
 	, hdf5_file_(hdf5_file)
@@ -1254,18 +1218,18 @@ H5::DataType* type_mapper::operator()(const IfcParse::select_type* pt, const boo
 
 class apply_attribute_visitor_instance_list {
 private:
-	const IfcEntityList::ptr& instances_;
+	const std::vector<IfcUtil::IfcBaseClass*>& instances_;
 	int attribute_idx_;
 public:
-	apply_attribute_visitor_instance_list(const IfcEntityList::ptr& instances, int attribute_idx)
+	apply_attribute_visitor_instance_list(const std::vector<IfcUtil::IfcBaseClass*>& instances, int attribute_idx)
 		: instances_(instances)
 		, attribute_idx_(attribute_idx)
 	{}
 
 	template <typename T>
 	typename T::return_type apply(T& t) const {
-		for (auto it = instances_->begin(); it != instances_->end(); ++it) {
-			apply_attribute_visitor((**it).data().getArgument(attribute_idx_), (**it).declaration().as_entity()->all_attributes()[attribute_idx_]).apply(t);
+		for (auto inst : instances_) {
+			apply_attribute_visitor(inst->data().getArgument(attribute_idx_), inst->declaration().as_entity()->all_attributes()[attribute_idx_]).apply(t);
 		}
 	}
 };
@@ -1296,23 +1260,16 @@ public:
 
 H5::CompType* type_mapper::operator()(const IfcParse::entity* e, const boost::optional< std::vector<Argument*> >&, int, const hsize_t*) {
 
-	IfcEntityList::ptr instances;
+	boost::optional<std::vector<IfcUtil::IfcBaseClass*> > instances;
 
 	if (ifc_file_ != nullptr) {
 		// List of entity instances of this type, no subtypes
-		IfcEntityList::ptr incl_subtypes = ifc_file_->entitiesByType(e->name());
-		instances.reset(new IfcEntityList);
-		if (incl_subtypes) {
-			for (auto it = incl_subtypes->begin(); it != incl_subtypes->end(); ++it) {
-				if ((**it).declaration().type() == e->type()) {
-					instances->push(*it);
-				}
-			}
-		}
-
-		if (instances->size() == 0) {
-			return 0;
+		auto& insts = ifc_file_->instances(e->type());
+		if (insts.size() == 0) {
+			return nullptr;
 		}
+		instances.emplace();
+		std::copy(insts.begin(), insts.end(), std::back_inserter(*instances));			
 	}
 
 	std::vector<const IfcParse::entity::attribute*> attributes = e->all_attributes();
@@ -1339,7 +1296,7 @@ H5::CompType* type_mapper::operator()(const IfcParse::entity* e, const boost::op
 			has_optional_attributes = true;
 
 			if (instances) {
-				apply_attribute_visitor_instance_list dispatch(instances, idx);
+				apply_attribute_visitor_instance_list dispatch(*instances, idx);
 				all_null_visitor visitor;
 				dispatch.apply(visitor);
 				if (visitor.all_null()) {
@@ -1383,7 +1340,7 @@ H5::CompType* type_mapper::operator()(const IfcParse::entity* e, const boost::op
 		int dims = 0;
 
 		if (padded_ && instances) {
-			apply_attribute_visitor_instance_list dispatch(instances, idx);
+			apply_attribute_visitor_instance_list dispatch(*instances, idx);
 			dispatch.apply(visitor);
 			if (visitor) {
 				max_length = visitor.max_length();
@@ -1608,7 +1565,7 @@ public:
 	}
 };
 
-void IfcParse::IfcHdf5File::write_schema(IfcParse::IfcFile* ifc_file) {
+void IfcParse::IfcHdf5File::write_schema(IfcParse::instance_categorizer* ifc_file) {
 	
 	if (settings_.profile() == Hdf5Settings::padded && ifc_file == nullptr) {
 		throw std::runtime_error("An IFC file is required to write a schema in padded profile");
@@ -1751,7 +1708,11 @@ void IfcParse::IfcHdf5File::write_population(H5::Group& population_group, instan
 			const std::string current_entity_name = IfcSchema::Type::ToString(*dsn_it);
 			const std::string dataset_path = current_entity_name + "_instances";
 
-			hsize_t num_instances = locator->instances(*dsn_it).size();
+			hsize_t num_instances = ifcfile.by_type(*dsn_it).size();
+
+			if (num_instances == 0) continue;
+
+			// locator->instances(*dsn_it).size();
 			H5::DataType entity_datatype = schema_group.openDataType(current_entity_name);
 
 			create_dataset(&population_group, dataset_path, entity_datatype, 1, &num_instances).close();
@@ -1763,9 +1724,12 @@ void IfcParse::IfcHdf5File::write_population(H5::Group& population_group, instan
 		
 		const std::string current_entity_name = IfcSchema::Type::ToString(*dsn_it);
 		const std::string dataset_path = current_entity_name + "_instances";
-		const std::vector<IfcUtil::IfcBaseEntity*>& instances = locator->instances(*dsn_it);
+
+		auto instances = ifcfile.by_type(*dsn_it);
 		hsize_t num_instances = instances.size();
 
+		if (num_instances == 0) continue;
+
 		std::cerr << current_entity_name << std::endl;
 
 		H5::DataType entity_datatype = schema_group.openDataType(current_entity_name);

src/ifcparse/IfcHdf5File.h

@@ -42,75 +42,149 @@ namespace IfcParse {
 		virtual void make_reference(IfcUtil::IfcBaseClass*, void*&) = 0;
 	};
 
+	template <typename T>
+	class vector_vector_iterator {
+	private:
+		const std::vector<std::vector<T> >& container_;
+		const typename std::vector<std::vector<T> >::const_iterator a_;
+		boost::optional<typename std::vector<T>::const_iterator> b_;
+
+	public:
+		vector_vector_iterator(const std::vector<std::vector<T> >& container)
+			: container_(container)
+			, a_(container_.begin())
+		{
+			if (a_ != container_.end()) {
+				b_ = a_->begin();
+			}
+		}
+
+		vector_vector_iterator(const std::vector<std::vector<T> >& container, typename std::vector<std::vector<T> >::const_iterator x)
+			: container_(container)
+			, a_(x)
+		{
+			if (a_ != container_.end()) {
+				b_ = a_->begin();
+			}
+		}
+
+		bool operator==(const vector_vector_iterator<T>& other) const {
+			if (a_ != container_.end()) {
+				return a_ == other.a_ && b_ == other.b_;
+			} else {
+				return other.a_ == container_.end();
+			}
+		}
+
+		bool operator!=(const vector_vector_iterator<T>& other) const {
+			return !((*this) == other);
+		}
+
+		T& operator*() { return *b_; }
+		const T& operator*() const { return *b_; }
+		T* operator->() { return b_.operator->(); }
+
+		vector_vector_iterator<T>& operator++() {
+			if (!b_) {
+				throw std::exception;
+			}
+			++(*b_);
+			if ((*b_) == a_->end()) {
+				do {
+					++a;
+				} while (a_ != container_.end() && a_->size() == 0);
+				if (a_ != container_.end()) {
+					b_ = a_->begin();
+				}
+			}			
+		}
+	};
+
+	class instance_categorizer {
+	private:
+		std::vector< IfcSchema::Type::Enum > dataset_names_;
+		std::vector< std::vector<IfcUtil::IfcBaseEntity*> > instances_;
+		std::vector<IfcUtil::IfcBaseEntity*> empty_;
+
+	public:
+
+		typedef vector_vector_iterator<IfcUtil::IfcBaseEntity*> const_iterator;
+
+		template <typename ForwardIt>
+		instance_categorizer(ForwardIt begin, ForwardIt end) {
+			std::set<IfcSchema::Type::Enum> dataset_names_set;
+			std::map<IfcSchema::Type::Enum, std::vector<IfcUtil::IfcBaseEntity*> > instances_map;
+
+			std::for_each(begin, end, [&dataset_names_set, &instances_map](IfcUtil::IfcBaseClass* inst) {
+				if (inst->declaration().as_entity()) {
+					dataset_names_set.insert(inst->declaration().type());
+					instances_map[inst->declaration().type()].push_back((IfcUtil::IfcBaseEntity*) inst);
+				}
+			});
+
+			dataset_names_.assign(dataset_names_set.begin(), dataset_names_set.end());
+			for (IfcSchema::Type::Enum ty : dataset_names_) {
+				auto&& insts = instances_map[ty];
+				std::sort(insts.begin(), insts.end(), [](IfcUtil::IfcBaseEntity* i1, IfcUtil::IfcBaseEntity* i2) {
+					return i1->data().id() < i2->data().id();
+				});
+				instances_.emplace_back(insts);				
+			}
+		}
+
+		const std::vector<IfcUtil::IfcBaseEntity*>& instances(IfcSchema::Type::Enum t) {
+			auto it = std::lower_bound(dataset_names_.begin(), dataset_names_.end(), t);
+			if (it == dataset_names_.end() || (*it) != t) return empty_;
+			return instances_[std::distance(dataset_names_.begin(), it)];
+		}
+
+		const std::vector<IfcSchema::Type::Enum>& dataset_names() const { return dataset_names_; };
+
+		const_iterator begin() const { return const_iterator(instances_); }
+		const_iterator end() const { return const_iterator(instances_, instances_.end()); }
+	};
+
 	class instance_enumerator {
 	private:
-		boost::optional<IfcFile*> file_;
-		boost::optional< std::pair<
-			IfcSpfHeader*,
-			std::map<unsigned int, IfcUtil::IfcBaseClass*>
-		> > map_;
+		const IfcSpfHeader* header_;
+		instance_categorizer* categories_;
 	public:
-		typedef IfcFile::const_iterator const_iterator;
+		typedef instance_categorizer::const_iterator const_iterator;
 
-		instance_enumerator(IfcFile* file)
-			: file_(file)
+		instance_enumerator()
+			: header_(nullptr)
+			, categories_(nullptr)
 		{}
 
-		template <typename T>
-		instance_enumerator(IfcSpfHeader* header, T begin, T end) {
-			std::map<unsigned int, IfcUtil::IfcBaseClass*> map;
-			std::for_each(begin, end, [&map](IfcUtil::IfcBaseClass* inst) {
-				// Use an incrementing id here in order not to fold distinct instances from different files with the same identifier
-				map.insert(std::make_pair(map.size(), inst));
+		explicit instance_enumerator(IfcFile* file)
+			: header_(&file->header())
+		{
+			std::vector<IfcUtil::IfcBaseClass*> temp;
+			std::transform(file->begin(), file->end(), std::back_inserter(temp), [](auto& pair) {
+				return pair.second;
 			});
-			map_ = std::make_pair(header, map);
+			categories_ = new instance_categorizer(temp.begin(), temp.end());
 		}
 
+		template <typename T>
+		instance_enumerator(IfcSpfHeader* header, T begin, T end)
+			: header_(header)
+			, categories_(new instance_categorizer(begin, end))
+		{}
+
 		const IfcSpfHeader& header() const {
-			if (map_) {
-				return *map_->first;
-			} else if (file_) {
-				return (**file_).header();
-			} else {
-				throw std::runtime_error("");
-			}
+			return *header_;
 		}
 
-		const_iterator begin() const {
-			if (map_) {
-				return map_->second.begin();
-			} else if (file_) {
-				return (**file_).begin();
-			} else {
-				throw std::runtime_error("");
-			}
+		const std::vector<IfcUtil::IfcBaseEntity*>& by_type(IfcSchema::Type::Enum t) {
+			return categories_->instances(t);
 		}
 
-		const_iterator end() const {
-			if (map_) {
-				return map_->second.end();
-			} else if (file_) {
-				return (**file_).end();
-			} else {
-				throw std::runtime_error("");
-			}
-		}
+		const_iterator begin() const { return categories_->begin(); }
+		const_iterator end() const { return categories_->end(); }
 
-		IfcEntityList::ptr entitiesByType(IfcSchema::Type::Enum t) { 
-			if (map_) {
-				IfcEntityList::ptr insts(new IfcEntityList);
-				for (auto it = begin(); it != end(); ++it) {
-					if (it->second->declaration().is(t)) {
-						insts->push(it->second);
-					}
-				}
-				return insts;
-			} else if (file_) {
-				return (**file_).entitiesByType(t);
-			} else {
-				throw std::runtime_error("");
-			}
-		}
+		const std::vector<IfcSchema::Type::Enum>& dataset_names() const { return categories_->dataset_names(); }
+		instance_categorizer* categorizer() const { return categories_; }
 	};
 
 	class IfcHdf5File {
@@ -187,7 +261,7 @@ namespace IfcParse {
 	public:
 		typedef std::pair<std::string, const H5::DataType*> compound_member;
 		
-		void write_schema(IfcFile* f = nullptr);
+		void write_schema(IfcParse::instance_categorizer* ifc_file = nullptr);
 		void write_population(H5::Group&, instance_enumerator&, IfcParse::abstract_instance_locator* locator = nullptr);
 		
 		IfcHdf5File(IfcFile* f, const std::string& name, const Hdf5Settings& settings)
@@ -197,17 +271,15 @@ namespace IfcParse {
 			, ifcfile_(f)
 			, settings_(settings)
 		{
-			write_schema(f);
-			instance_enumerator insts = f;
+			write_schema(ifcfile_.categorizer());
 			H5::Group population_group = file_->createGroup("population");
-			write_population(population_group, insts);
+			write_population(population_group, ifcfile_);
 		};
 
 		IfcHdf5File(H5::H5File& file, const IfcParse::schema_definition& schema, const Hdf5Settings& settings)
 			: file_(&file)
 			, schema_(&schema)
 			, settings_(settings)
-			, ifcfile_(nullptr)
 		{
 		};