Work towards serialization of data

cmake/CMakeLists.txt

@@ -328,6 +328,7 @@ if (WITH_ROCKSDB)
         message(FATAL_ERROR "Unable to find rocksdb library in: ${ROCKSDB_LIBRARY_DIR}")
     endif()
 
+    # @todo IFOPSH_ prefix?
     add_definitions(-DWITH_ROCKSDB)
     set(SWIG_DEFINES ${SWIG_DEFINES} -DWITH_ROCKSDB)
 

src/ifcconvert/IfcConvert.cpp

@@ -40,6 +40,7 @@
 #include "../serializers/SvgSerializer.h"
 #include "../serializers/USDSerializer.h"
 #include "../serializers/TtlWktSerializer.h"
+#include "../serializers/RocksDbSerializer.h"
 
 #include "../ifcgeom/IfcGeomFilter.h"
 #include "../ifcgeom/Iterator.h"
@@ -126,7 +127,8 @@ void print_usage(bool suggest_help = true)
         << "  .stp   STEP           Standard for the Exchange of Product Data\n"
         << "  .igs   IGES           Initial Graphics Exchange Specification\n"
         << "  .xml   XML            Property definitions and decomposition tree\n"
-        << "  .svg   SVG            Scalable Vector Graphics (2D floor plan)\n"
+		<< "  .rdb   RocksDB        RocksDB Key-Value store serialization of IFC data\n"
+		<< "  .svg   SVG            Scalable Vector Graphics (2D floor plan)\n"
 #ifdef WITH_HDF5
 		<< "  .h5    HDF            Hierarchical Data Format storing positions, normals and indices\n"
 #endif
@@ -642,6 +644,8 @@ int main(int argc, char** argv) {
 		CACHE = IfcUtil::path::from_utf8(".cache"),
 		HDF = IfcUtil::path::from_utf8(".h5"),
 		XML = IfcUtil::path::from_utf8(".xml"),
+		// @todo this is just temporary as it doesn't make sense to require an extension for a DB
+		RDB = IfcUtil::path::from_utf8(".rdb"),
 		CITY_JSON = IfcUtil::path::from_utf8(".cityjson"),
 		IFC = IfcUtil::path::from_utf8(".ifc"),
 		USD = IfcUtil::path::from_utf8(".usd"),
@@ -696,6 +700,27 @@ int main(int argc, char** argv) {
 		write_log(!quiet);
 		return exit_code;
 	}
+#ifdef WITH_ROCKSDB
+	else if (output_extension == RDB) {
+		int exit_code = EXIT_FAILURE;
+		try {
+			if (init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) {
+				time_t start, end;
+				time(&start);
+				RocksDbSerializer s(ifc_file, IfcUtil::path::to_utf8(output_filename));
+				Logger::Status("Populating RockDB Key-Value store...");
+				s.finalize();
+				time(&end);
+				Logger::Status("Done! Conversion took " + format_duration(start, end));
+				exit_code = EXIT_SUCCESS;
+			}
+		} catch (const std::exception& e) {
+			Logger::Error(e);
+		}
+		write_log(!quiet);
+		return exit_code;
+	}
+#endif
 #ifdef IFOPSH_WITH_CITYJSON
 	else if (output_extension == CITY_JSON || (output_extension == OBJ || output_extension == DAE || output_extension == GLB) && vmap.count("exterior-only") && exterior_only_algo != "none") {
 

src/ifcparse/IfcEntityInstanceData.cpp

@@ -29,124 +29,6 @@ public:
     int operator()(const aggregate_of_aggregate_of_instance::ptr& i) const { return i->size(); }
 };
 
-namespace {
-
-    // Trait to detect contiguous containers (vector / string)
-    template <typename T>
-    struct is_contiguous_container : std::false_type {};
-    template <typename T, typename Alloc>
-    struct is_contiguous_container<std::vector<T, Alloc>> : std::true_type {};
-    template <typename CharT, typename Traits, typename Alloc>
-    struct is_contiguous_container<std::basic_string<CharT, Traits, Alloc>> : std::true_type {};
-
-    template <typename T>
-    bool serialize(std::string& val, const T& t) {
-        return false;
-    }
-
-    template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
-    bool serialize(std::string& val, const T& t) {
-        auto s = sizeof(typename T::value_type) * t.size();
-        val.resize(s);
-        val[0] = TypeEncoder::encode_type<T>();
-        memcpy(val.data() + 1, t.data(), s);
-        return true;
-    }
-
-    bool serialize(std::string& val, const IfcUtil::IfcBaseClass* t) {
-        auto s = sizeof(size_t);
-        val.resize(s + 2);
-        val[0] = TypeEncoder::encode_type<IfcUtil::IfcBaseClass*>();
-        // 1 = entity - stored by id (entity name)
-        // 2 = type - stored by identity (internal counter in class)
-        val[1] = t->declaration().as_entity() ? 1 : 2;
-        size_t iden = t->declaration().as_entity() ? t->id() : t->identity();
-        memcpy(val.data() + 2, &iden, s);
-        return true;
-    }
-
-
-    bool serialize(std::string& val, const EnumerationReference& v) {
-        auto s = sizeof(size_t);
-        val.resize(s * 2 + 1);
-        val[0] = TypeEncoder::encode_type<EnumerationReference>();
-        size_t vv = v.enumeration()->index_in_schema();
-        memcpy(val.data() + 1, &vv, sizeof(size_t));
-        vv = v.index();
-        memcpy(val.data() + 1, &vv, sizeof(size_t));
-        return true;
-    }
-
-    bool serialize(std::string& val, aggregate_of_instance::ptr& t) {
-        std::vector<size_t> ids;
-        // @nb this has to be identity, because needs to work for typedecls as well
-        std::transform(t->begin(), t->end(), std::back_inserter(ids), [](auto& x) { return x->identity(); });
-        return false;
-    }
-
-    bool serialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
-        return false;
-    }
-
-    /*
-    template <typename T>
-    bool deserialize(std::string& val, const T& t) {}
-    */
-
-    template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
-    bool deserialize(std::string& val, T& t) {
-        // @todo vector of vector
-        if (val[0] != TypeEncoder::encode_type<T>()) {
-            return false;
-        }
-        auto s = (val.size() - 1) / sizeof(typename T::value_type);
-        t.resize(s);
-        memcpy(t.data(), val.data() + 1, s * sizeof(typename T::value_type));
-        return true;
-    }
-
-    template <typename T, typename std::enable_if<std::is_integral_v<T> || std::is_floating_point_v<T>, int>::type = 0>
-    bool deserialize(std::string& val, T& t) {
-        if (val[0] != TypeEncoder::encode_type<T>()) {
-            return false;
-        }
-        auto s = (val.size() - 1) / sizeof(T);
-        memcpy(&t, val.data() + 1, sizeof(T));
-        return true;
-    }
-
-    bool deserialize(std::string& val, boost::logic::tribool& t) {
-        if (val[0] != TypeEncoder::encode_type<boost::logic::tribool>()) {
-            return false;
-        }
-        if (val[1] == 0) {
-            t = false;
-        } else if (val[1] == 1) {
-            t = true;
-        } else if (val[1] == 2) {
-            t = boost::logic::indeterminate;
-        } else {
-            return false;
-        }
-    }
-
-    bool deserialize(std::string& val, boost::dynamic_bitset<>& t) {
-        if (val[0] != TypeEncoder::encode_type<boost::dynamic_bitset<>>()) {
-            return false;
-        }
-        t = boost::dynamic_bitset<>(val.substr(1));
-        return true;
-    }
-
-    bool deserialize(std::string& val, aggregate_of_instance::ptr& t) {
-        return false;
-    }
-
-    bool deserialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
-        return false;
-    }
-}
-
 namespace {
     template<typename T>
     inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
@@ -162,7 +44,7 @@ namespace {
             {
                 std::string str;
                 array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
-                deserialize(str, val);
+                impl::deserialize(str, val);
             }
             return val;
         }
@@ -326,3 +208,138 @@ IfcUtil::ArgumentType AttributeValue::type() const
 {
     return static_cast<IfcUtil::ArgumentType>(dispatch_index_(array_, storage_model_, instance_name_, index_));
 }
+
+bool impl::serialize(std::string& val, const IfcUtil::IfcBaseClass* t)
+{
+    auto s = sizeof(size_t);
+    val.resize(s + 2);
+    val[0] = TypeEncoder::encode_type<IfcUtil::IfcBaseClass*>();
+    // 1 = entity - stored by id (entity name)
+    // 2 = type - stored by identity (internal counter in class)
+    val[1] = t->declaration().as_entity() ? 1 : 2;
+    size_t iden = t->declaration().as_entity() ? t->id() : t->identity();
+    memcpy(val.data() + 2, &iden, s);
+    return true;
+}
+
+bool impl::serialize(std::string& val, const EnumerationReference& v)
+{
+    auto s = sizeof(size_t);
+    val.resize(s * 2 + 1);
+    val[0] = TypeEncoder::encode_type<EnumerationReference>();
+    size_t vv = v.enumeration()->index_in_schema();
+    memcpy(val.data() + 1, &vv, sizeof(size_t));
+    vv = v.index();
+    memcpy(val.data() + 1, &vv, sizeof(size_t));
+    return true;
+}
+
+bool impl::serialize(std::string& val, const aggregate_of_instance::ptr& t)
+{
+    std::vector<size_t> ids;
+    // @nb this has to be identity, because needs to work for typedecls as well
+    std::transform(t->begin(), t->end(), std::back_inserter(ids), [](auto& x) { return x->identity(); });
+    return false;
+}
+
+bool impl::serialize(std::string& val, const aggregate_of_aggregate_of_instance::ptr& t)
+{
+    return false;
+}
+
+bool impl::serialize(std::string& val, const Blank& t)
+{
+    return true;
+}
+
+bool impl::serialize(std::string& val, const Derived& t)
+{
+    return true;
+}
+
+bool impl::serialize(std::string& val, const empty_aggregate_t& t)
+{
+    return false;
+}
+
+bool impl::serialize(std::string& val, const empty_aggregate_of_aggregate_t& t)
+{
+    return false;
+}
+
+bool impl::serialize(std::string& val, const boost::logic::tribool& t)
+{
+    using T = char;
+    T tt = t == boost::logic::indeterminate ? 2 : t ? 1 : 0;
+    val.resize(sizeof(T) + 1);
+    val[0] = TypeEncoder::encode_type<T>();
+    memcpy(val.data() + 1, &tt, sizeof(T));
+    return true;
+}
+
+bool impl::serialize(std::string& val, const boost::dynamic_bitset<>& t)
+{
+    return false;
+}
+
+bool impl::deserialize(std::string& val, boost::logic::tribool& t) {
+    if (val[0] != TypeEncoder::encode_type<boost::logic::tribool>()) {
+        return false;
+    }
+    if (val[1] == 0) {
+        t = false;
+    } else if (val[1] == 1) {
+        t = true;
+    } else if (val[1] == 2) {
+        t = boost::logic::indeterminate;
+    } else {
+        return false;
+    }
+}
+
+bool impl::deserialize(std::string& val, boost::dynamic_bitset<>& t) {
+    if (val[0] != TypeEncoder::encode_type<boost::dynamic_bitset<>>()) {
+        return false;
+    }
+    t = boost::dynamic_bitset<>(val.substr(1));
+    return true;
+}
+
+bool impl::deserialize(std::string& val, aggregate_of_instance::ptr& t) {
+    return false;
+}
+
+bool impl::deserialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
+    return false;
+}
+
+template<typename T>
+void rocks_db_attribute_storage::set(std::size_t index, const T& value)
+{
+    std::string v;
+    impl::serialize(v, value);
+    fs_->db->Put(rocksdb::WriteOptions{}, prefix_ + ("|" + std::to_string(index)), v);
+}
+
+template void rocks_db_attribute_storage::set<Blank>(size_t index, const Blank& value);
+template void rocks_db_attribute_storage::set<int>(size_t index, const int& value);
+template void rocks_db_attribute_storage::set<bool>(size_t index, const bool& value);
+template void rocks_db_attribute_storage::set<boost::logic::tribool>(size_t index, const boost::logic::tribool& value);
+template void rocks_db_attribute_storage::set<double>(size_t index, const double& value);
+template void rocks_db_attribute_storage::set<std::string>(size_t index, const std::string& value);
+template void rocks_db_attribute_storage::set<boost::dynamic_bitset<>>(size_t index, const boost::dynamic_bitset<>& value);
+template void rocks_db_attribute_storage::set<EnumerationReference>(size_t index, const EnumerationReference& value);
+template void rocks_db_attribute_storage::set<IfcUtil::IfcBaseClass*>(size_t index, IfcUtil::IfcBaseClass* const& value);
+template void rocks_db_attribute_storage::set<std::vector<int>>(size_t index, const std::vector<int>& value);
+template void rocks_db_attribute_storage::set<std::vector<double>>(size_t index, const std::vector<double>& value);
+template void rocks_db_attribute_storage::set<std::vector<std::string>>(size_t index, const std::vector<std::string>& value);
+template void rocks_db_attribute_storage::set<std::vector<boost::dynamic_bitset<>>>(size_t index, const std::vector<boost::dynamic_bitset<>>& value);
+template void rocks_db_attribute_storage::set<aggregate_of_instance::ptr>(size_t index, const aggregate_of_instance::ptr& value);
+template void rocks_db_attribute_storage::set<std::vector<std::vector<int>>>(size_t index, const std::vector<std::vector<int>>& value);
+template void rocks_db_attribute_storage::set<std::vector<std::vector<double>>>(size_t index, const std::vector<std::vector<double>>& value);
+template void rocks_db_attribute_storage::set<aggregate_of_aggregate_of_instance::ptr>(size_t index, const aggregate_of_aggregate_of_instance::ptr& value);
+
+// @todo why do these need to be included, but are not in BaseEntity::set()?
+template void rocks_db_attribute_storage::set<Derived>(size_t index, const Derived& value);
+template void rocks_db_attribute_storage::set<empty_aggregate_t>(size_t index, const empty_aggregate_t& value);
+template void rocks_db_attribute_storage::set<empty_aggregate_of_aggregate_t>(size_t index, const empty_aggregate_of_aggregate_t& value);

src/ifcparse/IfcEntityInstanceData.h

@@ -161,6 +161,89 @@ namespace IfcParse {
     }
 }
 
+
+namespace impl {
+
+    // Trait to detect contiguous containers (vector / string)
+    template <typename T>
+    struct is_contiguous_container : std::false_type {};
+    template <typename T, typename Alloc>
+    struct is_contiguous_container<std::vector<T, Alloc>> : std::true_type {};
+    template <typename CharT, typename Traits, typename Alloc>
+    struct is_contiguous_container<std::basic_string<CharT, Traits, Alloc>> : std::true_type {};
+
+    template <typename T, typename std::enable_if<is_contiguous_container<T>::value && is_contiguous_container<typename T::value_type>::value, int>::type = 0>
+    bool serialize(std::string& val, const T& t) {
+        return false;
+    }
+
+    template <typename T, typename std::enable_if<is_contiguous_container<T>::value && !is_contiguous_container<typename T::value_type>::value, int>::type = 0>
+    bool serialize(std::string& val, const T& t) {
+        auto s = sizeof(typename T::value_type) * t.size();
+        val.resize(s + 1);
+        val[0] = TypeEncoder::encode_type<T>();
+        memcpy(val.data() + 1, t.data(), s);
+        return true;
+    }
+
+    template <typename T, typename std::enable_if<std::is_integral_v<T> || std::is_floating_point_v<T>, int>::type = 0>
+    bool serialize(std::string& val, const T& t) {
+        val.resize(sizeof(T) + 1);
+        val[0] = TypeEncoder::encode_type<T>();
+        memcpy(val.data() + 1, &t, sizeof(T));
+        return true;
+    }
+
+    bool serialize(std::string& val, const Blank& t);
+
+    bool serialize(std::string& val, const Derived& t);
+    bool serialize(std::string& val, const empty_aggregate_t& t);
+    bool serialize(std::string& val, const empty_aggregate_of_aggregate_t& t);
+
+    bool serialize(std::string& val, const boost::logic::tribool& t);
+
+    bool serialize(std::string& val, const boost::dynamic_bitset<>& t);
+    
+    bool serialize(std::string& val, const IfcUtil::IfcBaseClass* t);
+
+    bool serialize(std::string& val, const EnumerationReference& v);
+
+    bool serialize(std::string& val, const aggregate_of_instance::ptr& t);
+
+    bool serialize(std::string& val, const aggregate_of_aggregate_of_instance::ptr& t);
+
+    template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
+    bool deserialize(std::string& val, T& t) {
+        // @todo vector of vector
+        if (val[0] != TypeEncoder::encode_type<T>()) {
+            return false;
+        }
+        auto s = (val.size() - 1) / sizeof(typename T::value_type);
+        t.resize(s);
+        memcpy(t.data(), val.data() + 1, s * sizeof(typename T::value_type));
+        return true;
+    }
+
+    template <typename T, typename std::enable_if<std::is_integral_v<T> || std::is_floating_point_v<T>, int>::type = 0>
+    bool deserialize(std::string & val, T & t) {
+        if (val[0] != TypeEncoder::encode_type<T>()) {
+            return false;
+        }
+        auto s = (val.size() - 1) / sizeof(T);
+        memcpy(&t, val.data() + 1, sizeof(T));
+        return true;
+    }
+
+    bool deserialize(std::string& val, boost::logic::tribool& t);
+
+    bool deserialize(std::string& val, boost::dynamic_bitset<>& t);
+
+    bool deserialize(std::string& val, aggregate_of_instance::ptr& t);
+
+    bool deserialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t);
+}
+
+
 // short lived
 struct AttributeValue {
     uint8_t index_;
@@ -224,7 +307,10 @@ struct AttributeValue {
 
 struct rocks_db_attribute_storage {
 private:
-    IfcParse::impl::rocks_db_file_storage* fs;
+    // @todo not needed as call always passes through EntityInstanceData
+    IfcParse::impl::rocks_db_file_storage* fs_;
+    // @todo not needed should be passed from call stack
+    const char* prefix_;
 
     template<typename Visitor, std::size_t Index>
     auto apply_visitor_impl(Visitor&& visitor, std::size_t idx, std::integral_constant<std::size_t, Index>) const {
@@ -237,15 +323,44 @@ private:
     }
 
 public:
+    rocks_db_attribute_storage(IfcParse::impl::rocks_db_file_storage* fs, const char* const prefix)
+        : fs_(fs)
+        , prefix_(prefix)
+    {}
+
+    rocks_db_attribute_storage(rocks_db_attribute_storage& other)
+        : fs_(other.fs_)
+        , prefix_(other.prefix_)
+    {}
+
+    rocks_db_attribute_storage(rocks_db_attribute_storage&& other)
+        : fs_(other.fs_)
+        , prefix_(other.prefix_)
+    {}
+
+    rocks_db_attribute_storage& operator=(const rocks_db_attribute_storage& other) {
+        if (this != &other) {
+            fs_ = other.fs_;
+            prefix_ = other.prefix_;
+        }
+        return *this;
+    }
+
+    rocks_db_attribute_storage& operator=(const rocks_db_attribute_storage&& other) {
+        if (this != &other) {
+            fs_ = other.fs_;
+            prefix_ = other.prefix_;
+        }
+        return *this;
+    }
+
     size_t size() const {
-        // @todo
+        // @todo is this actually needed?
         return 8;
     }
 
     template<typename T>
-    void set(std::size_t index, T&& value) {
+    void set(std::size_t index, const T& value);
-        // @todo
-    }
 
     template<typename T>
     bool has(std::size_t index) const {

src/ifcparse/IfcFile.cpp

@@ -353,7 +353,7 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s
         size_t s;
         memcpy(&s, v.data(), sizeof(size_t));
         auto decl = file->schema()->declarations()[s];
-        IfcEntityInstanceData data(rocks_db_attribute_storage{});
+        IfcEntityInstanceData data(rocks_db_attribute_storage(this, "i|"));
         auto inst = file->schema()->instantiate(decl, std::move(data));
         inst->id_ = instanceId;
         instance_cache_.insert({ inst->identity(), inst });
@@ -363,34 +363,6 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s
     throw std::runtime_error("");
 }
 
-
-#include "rocksdb/merge_operator.h"
-
-namespace {
-
-    class ConcatenateIdMergeOperator : public rocksdb::AssociativeMergeOperator {
-    public:
-        virtual bool Merge(const rocksdb::Slice& key,
-            const rocksdb::Slice* existing_value,
-            const rocksdb::Slice& value,
-            std::string* new_value,
-            rocksdb::Logger* logger) const override {
-            if (existing_value) {
-                new_value->assign(existing_value->data(), existing_value->size());
-                new_value->append(value.data(), value.size());
-            } else {
-                new_value->assign(value.data(), value.size());
-            }
-            return true;
-        }
-
-        virtual const char* Name() const override {
-            return "ConcatenateIdMergeOperator";
-        }
-    };
-
-}
-
 // @todo naming
 IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile)
     : file(ffile)
@@ -399,11 +371,15 @@ IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string&
     , byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
     , byid_(db, "d|")
     , bytype_(db, "t|")
+    , byidentity_(&byid_, [this](size_t v) { return assert_existance(v); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
 {
     rocksdb::Options options;
     options.create_if_missing = true;
     options.merge_operator.reset(new ConcatenateIdMergeOperator());
     rocksdb::Status status = rocksdb::DB::Open(options, filepath, &db);
+    if (!status.ok()) {
+        throw std::runtime_error(status.ToString());
+    }
 }
 
 IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int id)

src/ifcparse/IfcFile.h

@@ -37,6 +37,32 @@
 #include <iterator>
 #include <map>
 
+#include "rocksdb/merge_operator.h"
+
+namespace {
+    // @todo move to a proper place
+    class ConcatenateIdMergeOperator : public rocksdb::AssociativeMergeOperator {
+    public:
+        virtual bool Merge(const rocksdb::Slice& key,
+            const rocksdb::Slice* existing_value,
+            const rocksdb::Slice& value,
+            std::string* new_value,
+            rocksdb::Logger* logger) const override {
+            if (existing_value) {
+                new_value->assign(existing_value->data(), existing_value->size());
+                new_value->append(value.data(), value.size());
+            } else {
+                new_value->assign(value.data(), value.size());
+            }
+            return true;
+        }
+
+        virtual const char* Name() const override {
+            return "ConcatenateIdMergeOperator";
+        }
+    };
+}
+
 namespace IfcParse {
 
 class IFC_PARSE_API file_open_status {
@@ -315,6 +341,11 @@ namespace impl {
         }
 
         void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
+
+        template <typename T>
+        T* create();
+
+        IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl);
     };
 
     struct rocks_db_file_storage {
@@ -328,9 +359,9 @@ namespace impl {
         typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
         identity_by_id_t byid_;
 
-        // typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_identity_t;
+        typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
         // storage is now Instance name -> Identity -> Pointer (cached)
-        // entity_by_identity_t byidentity_;
+        entity_by_id_t byidentity_;
 
         // index in schema to binary serialized ids
         typedef rocksdb_map_adapter<size_t, std::string> instance_id_str_by_type_t;
@@ -531,6 +562,11 @@ namespace impl {
         IfcUtil::IfcBaseClass* instance_by_id(int id);
 
         void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
+
+        template <typename T>
+        T* create();
+
+        IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl);
     };
 }
 
@@ -591,7 +627,8 @@ private:
     IfcFile(std::istream& stream, int length);
     IfcFile(void* data, int length);
     IfcFile(IfcParse::IfcSpfStream* stream);
-    IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"));
+    // @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
+    IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = ifcspf, const std::string& path = "");
 
     ~IfcFile() {
         for (const auto& p : byidentity_) {
@@ -722,10 +759,13 @@ private:
 
     typedef VariantMap<impl::in_memory_file_storage::entity_by_guid_t, impl::rocks_db_file_storage::entity_by_guid_t> entity_by_guid_t;
     entity_by_guid_t byguid_;
-    typedef VariantMap<impl::in_memory_file_storage::identity_by_id_t, impl::rocks_db_file_storage::identity_by_id_t> identity_by_id_t;
+    typedef VariantMap<impl::in_memory_file_storage::entity_by_id_t, impl::rocks_db_file_storage::entity_by_id_t> entity_by_id_t;
-    identity_by_id_t byid_;
+    entity_by_id_t byid_;
     typedef VariantMap<impl::in_memory_file_storage::entity_by_iden_t, impl::rocks_db_file_storage::entity_by_iden_cache_t> entity_by_iden_t;
     entity_by_iden_t byidentity_;
+    typedef VariantMap<impl::in_memory_file_storage::identity_by_id_t, impl::rocks_db_file_storage::identity_by_id_t> identity_by_id_t;
+    identity_by_id_t idenbyid_;
+
 
     // @todo
     entity_by_guid_t internal_guid_map() { return byguid_; };
@@ -735,6 +775,31 @@ private:
     void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
 
     void build_inverses_(IfcUtil::IfcBaseClass*);
+
+    template <typename T>
+    T* create() {
+        return std::visit([](auto& m) -> T* {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage> || 
+                std::is_same_v<std::decay_t<decltype(m)>, impl::rocks_db_file_storage>)
+            {
+                return m.create<T>();
+            } else {
+                return nullptr;
+            }
+        }, storage_);
+    }
+
+    IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl) {
+        return std::visit([decl](auto& m) -> IfcUtil::IfcBaseClass* {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage> ||
+                std::is_same_v<std::decay_t<decltype(m)>, impl::rocks_db_file_storage>)
+            {
+                return m.create(decl);
+            } else {
+                return nullptr;
+            }
+        }, storage_);
+    }
 };
 
 #ifdef WITH_IFCXML
@@ -743,6 +808,44 @@ IFC_PARSE_API IfcFile* parse_ifcxml(const std::string& filename);
 
 } // namespace IfcParse
 
+template <typename T>
+T* IfcParse::impl::in_memory_file_storage::create() {
+    if constexpr (std::is_same_v<std::decay_t<std::invoke_result_t<T::Class>>, IfcParse::entity>) {
+        return file->addEntity(new T(in_memory_attribute_storage(T::Class().attribute_count())))->as<T>();
+    } else if constexpr (std::is_same_v<std::decay_t<std::invoke_result_t<T::Class>>, IfcParse::type_declaration>) {
+        return file->addEntity(new T(in_memory_attribute_storage(1)))->as<T>();
+    } else {
+        static_assert(false, "Requires and entity or type declaration");
+    }
+}
+
+IfcUtil::IfcBaseClass* IfcParse::impl::in_memory_file_storage::create(const IfcParse::declaration* decl) {
+    if (auto* ent = decl->as_entity()) {
+        return file->addEntity(file->schema()->instantiate(decl, in_memory_attribute_storage(ent->attribute_count())));
+    } else if (auto* typedecl = decl->as_type_declaration()) {
+        return file->addEntity(file->schema()->instantiate(decl, in_memory_attribute_storage(1)));
+    } else {
+        throw std::runtime_error("Requires and entity or type declaration");
+    }
+}
+
+template <typename T>
+T* IfcParse::impl::rocks_db_file_storage::create() {
+    if constexpr (std::is_same_v<std::decay_t<std::invoke_result_t<T::Class>>, IfcParse::entity> || std::is_same_v<std::decay_t<std::invoke_result_t<T::Class>>, IfcParse::type_declaration>) {
+        return file->addEntity(new T(rocks_db_attribute_storage{}))->as<T>();
+    } else {
+        static_assert(false, "Requires and entity or type declaration");
+    }
+}
+
+IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::create(const IfcParse::declaration* decl) {
+    if (decl->as_entity() || decl->as_type_declaration()) {
+        return file->addEntity(file->schema()->instantiate(decl, rocks_db_attribute_storage(this, "i|")));
+    } else {
+        throw std::runtime_error("Requires and entity or type declaration");
+    }
+}
+
 namespace std {
 template <>
 struct iterator_traits<IfcParse::IfcFile::type_iterator> {

src/ifcparse/IfcParse.cpp

@@ -1258,6 +1258,7 @@ IfcFile::IfcFile(const std::string& path, filetype ty) {
         std::get<impl::in_memory_file_storage>(storage_).file = this;
         std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
     } else {
+        // @todo this can only be used for databases that already exist, because otherwise there is no way to specify the schema
         storage_ = impl::rocks_db_file_storage(path, this);
         std::get<impl::rocks_db_file_storage>(storage_).read_schema(schema_);
     }
@@ -1285,12 +1286,17 @@ IfcFile::IfcFile(IfcParse::IfcSpfStream* s) {
     ifcroot_type_ = schema_->declaration_by_name("IfcRoot");
 }
 
-IfcFile::IfcFile(const IfcParse::schema_definition* schema)
+IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const std::string& path)
     : schema_(schema)
     , ifcroot_type_(schema_->declaration_by_name("IfcRoot"))
     , max_id_(0)
+    , _header(ty == rocksdb ? this : nullptr)
 {
-    storage_.emplace<1>();
+    if (ty == ifcspf) {
+        storage_.emplace<1>();
+    } else {
+        storage_.emplace<2>(path, this);
+    }
     setDefaultHeaderValues();
 }
 
@@ -1866,7 +1872,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
             Logger::Message(Logger::LOG_WARNING, ss.str());
         }
         // The mapping by entity instance name is updated.
-        byid_.insert({ new_id, new_entity->identity() });
+        idenbyid_.insert({ new_id, new_entity->identity() });
         byidentity_.insert({ new_entity->identity(), new_entity });
     } else if (new_entity->file_ == nullptr) {
         // For non-entity instances, no mappings are updated, but the file
@@ -2210,7 +2216,7 @@ std::ostream& operator<<(std::ostream& out, const IfcParse::IfcFile& file) {
 
     typedef std::vector<IfcUtil::IfcBaseClass*> vector_t;
     vector_t sorted;
-    std::transform(file.begin(), file.end(), std::back_inserter(sorted), [&file](const auto& x) { return file.byidentity_.find(x.second)->second; });
+    std::transform(file.begin(), file.end(), std::back_inserter(sorted), [&file](const auto& x) { return x.second; });
     std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a->id() < b->id(); });
 
     for (auto& e : sorted) {
@@ -2457,7 +2463,7 @@ void IfcParse::IfcFile::build_inverses_(IfcUtil::IfcBaseClass* inst) {
 
 void IfcParse::IfcFile::build_inverses() {
     for (const auto& pair : *this) {
-        build_inverses_(byidentity_.find(pair.second)->second);
+        build_inverses_(pair.second);
     }
 }
 

src/ifcparse/IfcSpfHeader.cpp

@@ -55,7 +55,12 @@ namespace {
 HeaderEntity::HeaderEntity(const char* const datatype, size_t size, IfcFile* file)
     : datatype_(datatype)
     , file_(file)
-    , data_(file ? read_from_spf_file(file, size) : IfcEntityInstanceData(in_memory_attribute_storage(size)))
+    , data_((file && file->storage_.index() == 1)
+        ? read_from_spf_file(file, size)
+        : (file && file->storage_.index() == 2)
+        ? IfcEntityInstanceData(rocks_db_attribute_storage(&std::get<IfcParse::impl::rocks_db_file_storage>(file->storage_), "h|"))
+        : IfcEntityInstanceData(in_memory_attribute_storage(size))
+    )
 {}
 
 HeaderEntity::~HeaderEntity() {
@@ -185,21 +190,21 @@ const FileSchema& IfcSpfHeader::file_schema() const {
 
 FileDescription& IfcSpfHeader::file_description() {
     if (file_description_ == nullptr) {
-        throw IfcException("File description not set");
+        file_description_ = new FileDescription(file_);
     }
     return *file_description_;
 }
 
 FileName& IfcSpfHeader::file_name() {
     if (file_name_ == nullptr) {
-        throw IfcException("File name not set");
+        file_name_ = new FileName(file_);
     }
     return *file_name_;
 }
 
 FileSchema& IfcSpfHeader::file_schema() {
     if (file_schema_ == nullptr) {
-        throw IfcException("File schema not set");
+        file_schema_ = new FileSchema(file_);
     }
     return *file_schema_;
 }

src/ifcparse/map_transformer.h

@@ -31,6 +31,7 @@ public:
     using base_mapped_type = typename BaseMap::mapped_type;
     using transformed_mapped_type = std::invoke_result_t<Transform, base_mapped_type>;
     using value_type = std::pair<key_type, transformed_mapped_type>;
+    using mapped_type = transformed_mapped_type;
 
 private:
     BaseMap* base_map_;