Exploration of using rocksdb as file+instance storage

src/ifcparse/IfcBaseClass.h

@@ -117,6 +117,8 @@ public:
     
     void unset_attribute_value(size_t i);
 
+    AttributeValue get_attribute_value(size_t index) const;
+
     uint32_t identity() const { return identity_; }
 
     uint32_t id() const { return id_; }
@@ -131,7 +133,7 @@ class IFC_PARSE_API IfcBaseEntity : public IfcBaseClass {
     IfcBaseEntity(IfcEntityInstanceData&& data);
 
     IfcBaseEntity(size_t n)
-        : IfcBaseClass(IfcEntityInstanceData(storage_t(n)))
+        : IfcBaseClass(IfcEntityInstanceData(in_memory_attribute_storage(n)))
     {}
 
     virtual const IfcParse::declaration& declaration() const = 0;
@@ -172,7 +174,7 @@ class IFC_PARSE_API IfcBaseType : public IfcBaseClass {
     {}
 
     IfcBaseType()
-        : IfcBaseClass(IfcEntityInstanceData(storage_t(1)))
+        : IfcBaseClass(IfcEntityInstanceData(in_memory_attribute_storage(1)))
     {}
 
     virtual const IfcParse::declaration& declaration() const = 0;

src/ifcparse/IfcEntityInstanceData.cpp

@@ -1,5 +1,6 @@
 #include "IfcEntityInstanceData.h"
 #include "IfcBaseClass.h"
+#include "IfcFile.h"
 
 // @todo is size() still needed?
 class SizeVisitor {
@@ -28,100 +29,300 @@ public:
     int operator()(const aggregate_of_aggregate_of_instance::ptr& i) const { return i->size(); }
 };
 
+namespace {
+
+    // Trait to detect contiguous containers (vector / string)
+    template <typename T>
+    struct is_contiguous_container : std::false_type {};
+    template <typename T, typename Alloc>
+    struct is_contiguous_container<std::vector<T, Alloc>> : std::true_type {};
+    template <typename CharT, typename Traits, typename Alloc>
+    struct is_contiguous_container<std::basic_string<CharT, Traits, Alloc>> : std::true_type {};
+
+    template <typename T>
+    bool serialize(std::string& val, const T& t) {
+        return false;
+    }
+
+    template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
+    bool serialize(std::string& val, const T& t) {
+        auto s = sizeof(typename T::value_type) * t.size();
+        val.resize(s);
+        val[0] = TypeEncoder::encode_type<T>();
+        memcpy(val.data() + 1, t.data(), s);
+        return true;
+    }
+
+    bool serialize(std::string& val, const IfcUtil::IfcBaseClass* t) {
+        auto s = sizeof(size_t);
+        val.resize(s + 2);
+        val[0] = TypeEncoder::encode_type<IfcUtil::IfcBaseClass*>();
+        // 1 = entity - stored by id (entity name)
+        // 2 = type - stored by identity (internal counter in class)
+        val[1] = t->declaration().as_entity() ? 1 : 2;
+        size_t iden = t->declaration().as_entity() ? t->id() : t->identity();
+        memcpy(val.data() + 2, &iden, s);
+        return true;
+    }
+
+
+    bool serialize(std::string& val, const EnumerationReference& v) {
+        auto s = sizeof(size_t);
+        val.resize(s * 2 + 1);
+        val[0] = TypeEncoder::encode_type<EnumerationReference>();
+        size_t vv = v.enumeration()->index_in_schema();
+        memcpy(val.data() + 1, &vv, sizeof(size_t));
+        vv = v.index();
+        memcpy(val.data() + 1, &vv, sizeof(size_t));
+        return true;
+    }
+
+    bool serialize(std::string& val, aggregate_of_instance::ptr& t) {
+        std::vector<size_t> ids;
+        // @nb this has to be identity, because needs to work for typedecls as well
+        std::transform(t->begin(), t->end(), std::back_inserter(ids), [](auto& x) { return x->identity(); });
+        return false;
+    }
+
+    bool serialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
+        return false;
+    }
+
+    /*
+    template <typename T>
+    bool deserialize(std::string& val, const T& t) {}
+    */
+
+    template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
+    bool deserialize(std::string& val, T& t) {
+        // @todo vector of vector
+        if (val[0] != TypeEncoder::encode_type<T>()) {
+            return false;
+        }
+        auto s = (val.size() - 1) / sizeof(typename T::value_type);
+        t.resize(s);
+        memcpy(t.data(), val.data() + 1, s * sizeof(typename T::value_type));
+        return true;
+    }
+
+    template <typename T, typename std::enable_if<std::is_integral_v<T> || std::is_floating_point_v<T>, int>::type = 0>
+    bool deserialize(std::string& val, T& t) {
+        if (val[0] != TypeEncoder::encode_type<T>()) {
+            return false;
+        }
+        auto s = (val.size() - 1) / sizeof(T);
+        memcpy(&t, val.data() + 1, sizeof(T));
+        return true;
+    }
+
+    bool deserialize(std::string& val, boost::logic::tribool& t) {
+        if (val[0] != TypeEncoder::encode_type<boost::logic::tribool>()) {
+            return false;
+        }
+        if (val[1] == 0) {
+            t = false;
+        } else if (val[1] == 1) {
+            t = true;
+        } else if (val[1] == 2) {
+            t = boost::logic::indeterminate;
+        } else {
+            return false;
+        }
+    }
+
+    bool deserialize(std::string& val, boost::dynamic_bitset<>& t) {
+        if (val[0] != TypeEncoder::encode_type<boost::dynamic_bitset<>>()) {
+            return false;
+        }
+        t = boost::dynamic_bitset<>(val.substr(1));
+        return true;
+    }
+
+    bool deserialize(std::string& val, aggregate_of_instance::ptr& t) {
+        return false;
+    }
+
+    bool deserialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
+        return false;
+    }
+}
+
+namespace {
+    template<typename T>
+    inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
+    {
+        if (storage_model_ == 0) {
+            return array_.storage_ptr->get<T>(index_);
+        } else {
+            T val;
+            if constexpr (
+                // the following types cannot be directly deserialized from rocksdb, but need to be constructed
+                !std::is_same_v<T, EnumerationReference> &&
+                !std::is_same_v<std::remove_cv_t<std::remove_pointer_t<T>>, IfcUtil::IfcBaseClass>)
+            {
+                std::string str;
+                array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
+                deserialize(str, val);
+            }
+            return val;
+        }
+    }
+
+    template<typename T>
+    inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
+    {
+        if (storage_model_ == 0) {
+            return array_.storage_ptr->has<T>(index_);
+        } else {
+            std::string str;
+            array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
+            return str[0] == TypeEncoder::encode_type<T>();
+        }
+    }
+
+    inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
+    {
+        if (storage_model_ == 0) {
+            return array_.storage_ptr->index(index_);
+        } else {
+            std::string str;
+            array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
+            return (size_t) str[0] - 'A';
+        }
+    }
+
+}
+
 AttributeValue::operator int() const
 {
-    return array_->get<int>(index_);
+    return dispatch_get_<int>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator bool() const
 {
-    return array_->get<bool>(index_);
+    return dispatch_get_<bool>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator double() const
 {
-    return array_->get<double>(index_);
+    return dispatch_get_<double>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator boost::logic::tribool() const
 {
-    if (array_->has<bool>(index_)) {
+    if (dispatch_has_<bool>(array_, storage_model_, instance_name_, index_)) {
-        return array_->get<bool>(index_);
+        return dispatch_get_<bool>(array_, storage_model_, instance_name_, index_);
     }
-    return array_->get<boost::logic::tribool>(index_);
+    return dispatch_get_<boost::logic::tribool>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator std::string() const
 {
-    if (array_->has<EnumerationReference>(index_)) {
+    if (dispatch_has_<EnumerationReference>(array_, storage_model_, instance_name_, index_)) {
         // @todo this is silly, but the way things currently work,
         // @todo also we don't really need to store a reference to the enumeration type, when this same type is already stored on the definition of the entity and no other value can be provided.
-        return array_->get<EnumerationReference>(index_).value();
+        if (storage_model_ == 0) {
+            return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, index_).value();
+        } else {
+            std::string str;
+            array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
+            size_t v;
+            memcpy(&v, str.data() + 1, sizeof(size_t));
+            auto decl = schema_->declarations()[v]->as_enumeration_type();
+            memcpy(&v, str.data() + 5, sizeof(size_t));
+            return decl->lookup_enum_value(v);
+        }
+    }
+    return dispatch_get_<std::string>(array_, storage_model_, instance_name_, index_);
+}
+
+AttributeValue::operator EnumerationReference() const
+{
+    if (storage_model_ == 0) {
+        return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, index_);
+    } else {
+        std::string str;
+        array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
+        size_t v;
+        memcpy(&v, str.data() + 1, sizeof(size_t));
+        auto decl = schema_->declarations()[v]->as_enumeration_type();
+        memcpy(&v, str.data() + 5, sizeof(size_t));
+        return EnumerationReference(decl, v);
     }
-    return array_->get<std::string>(index_);
 }
 
 AttributeValue::operator boost::dynamic_bitset<>() const
 {
-    return array_->get<boost::dynamic_bitset<>>(index_);
+    return dispatch_get_<boost::dynamic_bitset<>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator IfcUtil::IfcBaseClass* () const
 {
-    return array_->get<IfcUtil::IfcBaseClass*>(index_);
+    if (storage_model_ == 0) {
+        return dispatch_get_<IfcUtil::IfcBaseClass*>(array_, storage_model_, instance_name_, index_);
+    } else {
+        std::string str;
+        array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
+        size_t v;
+        memcpy(&v, str.data() + 1, sizeof(size_t));
+        auto decl = schema_->declarations()[v]->as_enumeration_type();
+        memcpy(&v, str.data() + 5, sizeof(size_t));
+        return array_.db_ptr->assert_existance(v);
+    }
 }
 
 AttributeValue::operator std::vector<int>() const
 {
-    return array_->get<std::vector<int>>(index_);
+    return dispatch_get_<std::vector<int>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator std::vector<double>() const
 {
-    return array_->get<std::vector<double>>(index_);
+    return dispatch_get_<std::vector<double>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator std::vector<std::string>() const
 {
-    return array_->get<std::vector<std::string>>(index_);
+    return dispatch_get_<std::vector<std::string>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator std::vector<boost::dynamic_bitset<>>() const
 {
-    return array_->get<std::vector<boost::dynamic_bitset<>>>(index_);
+    return dispatch_get_<std::vector<boost::dynamic_bitset<>>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator boost::shared_ptr<aggregate_of_instance>() const
 {
-    return array_->get<boost::shared_ptr<aggregate_of_instance>>(index_);
+    return dispatch_get_<boost::shared_ptr<aggregate_of_instance>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator std::vector<std::vector<int>>() const
 {
-    return array_->get<std::vector<std::vector<int>>>(index_);
+    return dispatch_get_<std::vector<std::vector<int>>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator std::vector<std::vector<double>>() const
 {
-    return array_->get<std::vector<std::vector<double>>>(index_);
+    return dispatch_get_<std::vector<std::vector<double>>>(array_, storage_model_, instance_name_, index_);
 }
 
 AttributeValue::operator boost::shared_ptr<aggregate_of_aggregate_of_instance>() const
 {
-    return array_->get<boost::shared_ptr<aggregate_of_aggregate_of_instance>>(index_);
+    return dispatch_get_<boost::shared_ptr<aggregate_of_aggregate_of_instance>>(array_, storage_model_, instance_name_, index_);
 }
 
 bool AttributeValue::isNull() const
 {
-    return array_->has<Blank>(index_);
+    return dispatch_has_<Blank>(array_, storage_model_, instance_name_, index_);
 }
 
 unsigned int AttributeValue::size() const
 {
-    return array_->apply_visitor(SizeVisitor{}, index_);
+    // @todo
+    return array_.storage_ptr->apply_visitor(SizeVisitor{}, index_);
 }
 
 IfcUtil::ArgumentType AttributeValue::type() const
 {
-    return static_cast<IfcUtil::ArgumentType>(array_->index(index_));
+    return static_cast<IfcUtil::ArgumentType>(dispatch_index_(array_, storage_model_, instance_name_, index_));
 }

src/ifcparse/IfcEntityInstanceData.h

@@ -25,6 +25,13 @@
 #include "aggregate_of_instance.h"
 #include "IfcSchema.h"
 
+#pragma push_macro("Handle")
+#undef Handle
+
+#include <rocksdb/db.h>
+
+#pragma pop_macro("Handle")
+
 #include <boost/optional.hpp>
 #include <boost/shared_ptr.hpp>
 #include <boost/logic/tribool.hpp>
@@ -36,7 +43,7 @@ private:
     size_t index_;
 public:
 
-    EnumerationReference(const IfcParse::enumeration_type* enumeration, size_t index)
+    EnumerationReference(const IfcParse::enumeration_type* enumeration = nullptr, size_t index = 0)
         : enumeration_(enumeration)
         , index_(index)
     {}
@@ -58,7 +65,12 @@ class Derived {};
 class empty_aggregate_t {};
 class empty_aggregate_of_aggregate_t {};
 
-typedef VariantArray <
+template<typename... Args>
+struct parameter_pack {
+    static constexpr size_t size = sizeof...(Args);
+};
+
+typedef parameter_pack <
     // A null argument, it will always serialize to $
     Blank,
     // @todo Derived is not really necessary anymore, just serialize correctly based on schema
@@ -113,26 +125,75 @@ typedef VariantArray <
     std::vector<std::vector<double>>,
     // An aggregate of an aggregate of entities. E.g. ((#1, #2), (#3))
     aggregate_of_aggregate_of_instance::ptr
-> storage_t;
+> type_variant_parameter_pack;
+
+template<typename Pack>
+struct pack_to_variant_array;
+
+template<typename... Args>
+struct pack_to_variant_array<parameter_pack<Args...>> {
+    using type = VariantArray<Args...>;
+};
+
+using in_memory_attribute_storage = pack_to_variant_array<type_variant_parameter_pack>::type;
+
+template <typename Pack>
+struct TypeEncoder_t;
+
+template <typename... Types>
+struct TypeEncoder_t<parameter_pack<Types...>> {
+    template <typename U>
+    static char encode_type() {
+        return 'A' + ::impl::TypeIndex_v<U, Types...>;
+    }
+};
+
+using TypeEncoder = TypeEncoder_t<type_variant_parameter_pack>;
 
 struct MutableAttributeValue {
     int name_;
     uint8_t index_;
 };
 
+namespace IfcParse {
+    namespace impl {
+        class rocks_db_file_storage;
+    }
+}
+
 // short lived
 struct AttributeValue {
-    const storage_t* array_;
     uint8_t index_;
-
+    uint8_t storage_model_ = 0;
+    size_t instance_name_;
+    // @todo couple with db_ptr;
+    IfcParse::schema_definition* schema_;
+    union pointer_type {
+        const in_memory_attribute_storage* storage_ptr;
+        IfcParse::impl::rocks_db_file_storage* db_ptr;
+        pointer_type(IfcParse::impl::rocks_db_file_storage* db) : db_ptr(db) {}
+        pointer_type(const in_memory_attribute_storage* ims) : storage_ptr(ims) {}
+    };
+    pointer_type array_;
+    
     AttributeValue()
-        : array_(nullptr)
+        : index_(0)
-        , index_(0)
+        , array_((const in_memory_attribute_storage*)nullptr)
+        , storage_model_(0)
     {}
 
-    AttributeValue(const storage_t* arr, uint8_t index)
+    AttributeValue(const in_memory_attribute_storage* arr, uint8_t index)
-        : array_(arr)
+        : index_(index)
-        , index_(index)
+        , array_(arr)
+        , storage_model_(0)
+    {}
+
+    AttributeValue(IfcParse::schema_definition* schema, IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t index)
+        : index_(index)
+        , array_(db)
+        , storage_model_(1)
+        , instance_name_(instance_name)
+        , schema_(schema)
     {}
 
     operator int() const;
@@ -153,17 +214,60 @@ struct AttributeValue {
     operator std::vector<std::vector<double>>() const;
     operator boost::shared_ptr<aggregate_of_aggregate_of_instance>() const;
 
+    operator EnumerationReference() const;
+
     bool isNull() const;
     unsigned int size() const;
 
     IfcUtil::ArgumentType type() const;
 };
 
+struct rocks_db_attribute_storage {
+private:
+    IfcParse::impl::rocks_db_file_storage* fs;
+
+    template<typename Visitor, std::size_t Index>
+    auto apply_visitor_impl(Visitor&& visitor, std::size_t idx, std::integral_constant<std::size_t, Index>) const {
+        return apply_visitor_impl(std::forward<Visitor>(visitor), idx, std::integral_constant<std::size_t, Index - 1>{});
+    }
+
+    template<typename Visitor>
+    void apply_visitor_impl(Visitor&&, std::size_t, std::integral_constant<std::size_t, 0>) const {
+        throw std::runtime_error("Invalid variant index");
+    }
+
+public:
+    size_t size() const {
+        // @todo
+        return 8;
+    }
+
+    template<typename T>
+    void set(std::size_t index, T&& value) {
+        // @todo
+    }
+
+    template<typename T>
+    bool has(std::size_t index) const {
+        // @todo
+        return false;
+    }
+
+    template<typename Visitor>
+    auto apply_visitor(Visitor&& visitor, std::size_t index) const {
+        return apply_visitor_impl(std::forward<Visitor>(visitor), index, std::integral_constant<std::size_t, type_variant_parameter_pack::size>{});
+    }
+};
+
 class IFC_PARSE_API IfcEntityInstanceData {
   public:
-      storage_t storage_;
+      std::variant<in_memory_attribute_storage, rocks_db_attribute_storage> storage_;
 
-      IfcEntityInstanceData(storage_t&& storage)
+      IfcEntityInstanceData(in_memory_attribute_storage&& storage)
+          : storage_(std::move(storage))
+      {}
+
+      IfcEntityInstanceData(rocks_db_attribute_storage&& storage)
           : storage_(std::move(storage))
       {}
 
@@ -182,15 +286,29 @@ class IFC_PARSE_API IfcEntityInstanceData {
 
     AttributeValue get_attribute_value(size_t index) const;
 
-    /*
+    template<typename T>
-    template <typename T>
+    void set_attribute_value(std::size_t index, T&& value) {
-    void set_attribute_value(size_t index, const T& t);
+        std::visit([&index, &value](auto& x) {
+            return x.set(index, value); 
+        }, storage_);
+    }
 
-    void set_attribute_value(size_t index, AttributeValue&, IfcUtil::ArgumentType attr_type = IfcUtil::Argument_UNKNOWN);
+    template<typename T>
-    */
+    bool has_attribute_value(std::size_t index) const {
+        return std::visit([&index](const auto& x) {
+            return x.has<T>(index);
+        }, storage_);
+    }
+
+    template<typename Visitor>
+    auto apply_visitor(Visitor&& visitor, std::size_t index) const {
+        return std::visit([&index, &visitor](const auto& x) {
+            return x.apply_visitor(std::forward<Visitor>(visitor), index);
+        }, storage_);
+    }
 
     size_t size() const {
-        return storage_.size();
+        return std::visit([](const auto& x) { return x.size(); }, storage_);
     }
 
     void toString(std::ostream&, bool upper = false, const IfcParse::entity* ent = nullptr) const;

src/ifcparse/IfcFile.cpp

@@ -259,10 +259,10 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
     }
 
     if (tokens_.empty()) {
-        return IfcEntityInstanceData(storage_t(0));
+        return IfcEntityInstanceData(in_memory_attribute_storage(0));
     }
 
-    storage_t storage(decl != nullptr
+    in_memory_attribute_storage storage(decl != nullptr
         ? (std::min)(parameter_types.size(), tokens_.size())
         : tokens_.size()
     );
@@ -328,3 +328,96 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
 
     return IfcEntityInstanceData(std::move(storage));
 }
+
+IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_iterator::operator*() const {
+    auto it = storage_->byid_.find(*read_id_());
+    if (it != storage_->byid_.end()) {
+        // @todo define an implicit std::to_string() in all map adapters with leading 0s
+        auto jt = storage_->instance_cache_.find(it->second);
+        if (jt != storage_->instance_cache_.end()) {
+            return jt->second;
+        } else {
+            return storage_->assert_existance(it->second);
+        }
+    }
+}
+
+const IfcParse::declaration* IfcParse::impl::rocks_db_file_storage::rocksdb_types_iterator::operator*() const {
+    return storage_->file->schema()->declarations()[*read_id_()];
+}
+
+IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(size_t instanceId) {
+    std::string v;
+    rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instanceId) + "|t", &v);
+    if (s.ok()) {
+        size_t s;
+        memcpy(&s, v.data(), sizeof(size_t));
+        auto decl = file->schema()->declarations()[s];
+        IfcEntityInstanceData data(rocks_db_attribute_storage{});
+        auto inst = file->schema()->instantiate(decl, std::move(data));
+        inst->id_ = instanceId;
+        instance_cache_.insert({ inst->identity(), inst });
+        byid_.insert({ inst->id(), inst->identity() });
+        return inst;
+    }
+    throw std::runtime_error("");
+}
+
+
+#include "rocksdb/merge_operator.h"
+
+namespace {
+
+    class ConcatenateIdMergeOperator : public rocksdb::AssociativeMergeOperator {
+    public:
+        virtual bool Merge(const rocksdb::Slice& key,
+            const rocksdb::Slice* existing_value,
+            const rocksdb::Slice& value,
+            std::string* new_value,
+            rocksdb::Logger* logger) const override {
+            if (existing_value) {
+                new_value->assign(existing_value->data(), existing_value->size());
+                new_value->append(value.data(), value.size());
+            } else {
+                new_value->assign(value.data(), value.size());
+            }
+            return true;
+        }
+
+        virtual const char* Name() const override {
+            return "ConcatenateIdMergeOperator";
+        }
+    };
+
+}
+
+
+// @todo naming
+IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile)
+    : file(ffile)
+    // @todo db is not initialized here yet
+    , byguid_internal_(db, "g|")
+    , byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
+    , byid_(db, "d|")
+    , bytype_(db, "t|")
+{
+    rocksdb::Options options;
+    options.create_if_missing = true;
+    options.merge_operator.reset(new ConcatenateIdMergeOperator());
+    rocksdb::Status status = rocksdb::DB::Open(options, filepath, &db);
+}
+
+IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int id)
+{
+    // @todo rename assert_existance() -> instance_by_id();
+    return assert_existance(id);
+}
+
+IfcUtil::IfcBaseClass* IfcParse::impl::in_memory_file_storage::instance_by_id(int id)
+{
+    auto it = byid_.find(id);
+    if (it == byid_.end()) {
+        throw IfcException("Instance #" + boost::lexical_cast<std::string>(id) + " not found");
+    }
+    return it->second;
+}

src/ifcparse/IfcFile.h

@@ -24,6 +24,9 @@
 #include "IfcParse.h"
 #include "IfcSchema.h"
 #include "IfcSpfHeader.h"
+#include "rocksdb_map_adapter.h"
+#include "map_variant.h"
+#include "map_transformer.h"
 
 #include <boost/multi_index/ordered_index.hpp>
 #include <boost/multi_index/random_access_index.hpp>
@@ -95,94 +98,469 @@ struct parse_context {
     IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size);
 };
 
-/// This class provides several static convenience functions and variables
+#include <variant>
-/// and provide access to the entities in an IFC file
+#include <iterator>
+#include <type_traits>
+#include <iostream>
+#include <vector>
+#include <list>
+
+template <typename... Iterators>
+class variant_iterator {
+public:
+    // The variant type holding one of the underlying iterators.
+    using variant_type = std::variant<Iterators...>;
+
+    // Assuming that all iterator types have the same value_type, difference_type, etc.
+    using value_type = std::common_type_t<typename std::iterator_traits<Iterators>::value_type...>;
+    using difference_type = std::common_type_t<typename std::iterator_traits<Iterators>::difference_type...>;
+    using pointer = value_type*;
+    using reference = value_type&;
+    // For simplicity, we use input_iterator_tag; if all underlying iterators support more,
+    // you could compute the common iterator_category.
+    using iterator_category = std::input_iterator_tag;
+
+    // Default constructor.
+    variant_iterator() = default;
+
+    // Construct from any one of the underlying iterator types.
+    template <typename Iterator>
+    variant_iterator(Iterator it) : it_(it) { }
+
+    // Dereference operator.
+    decltype(auto) operator*() const {
+        return std::visit([](const auto& iter) -> decltype(auto) {
+            return *iter;
+        }, it_);
+    }
+
+    // Arrow operator.
+    decltype(auto) operator->() const {
+        return std::visit([](const auto& iter) -> decltype(auto) {
+            return iter.operator->();
+        }, it_);
+    }
+
+    // Pre-increment operator.
+    variant_iterator& operator++() {
+        std::visit([](auto& iter) { ++iter; }, it_);
+        return *this;
+    }
+
+    // Post-increment operator.
+    variant_iterator operator++(int) {
+        variant_iterator temp(*this);
+        ++(*this);
+        return temp;
+    }
+
+    // Pre-decrement operator.
+    variant_iterator& operator--() {
+        std::visit([](auto& iter) { --iter; }, it_);
+        return *this;
+    }
+
+    // Post-decrement operator.
+    variant_iterator operator--(int) {
+        variant_iterator temp(*this);
+        --(*this);
+        return temp;
+    }
+
+    // Equality comparison.
+    friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) {
+        return lhs.it_ == rhs.it_;
+    }
+
+    // Inequality comparison.
+    friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) {
+        return !(lhs == rhs);
+    }
+
+private:
+    variant_type it_;
+};
+
+namespace impl {
+    struct in_memory_file_storage {
+        IfcParse::IfcSpfLexer* tokens;
+        IfcParse::IfcSpfStream* stream;
+
+        IfcParse::IfcFile* file;
+
+        unresolved_references references_to_resolve;
+
+        typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
+        typedef boost::unordered_map<size_t, size_t> identity_by_id_t;
+        typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_t;
+        typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_by_guid_t;
+        typedef std::tuple<int, short, short> inverse_attr_record;
+        enum INVERSE_ATTR {
+            INSTANCE_ID,
+            INSTANCE_TYPE,
+            ATTRIBUTE_INDEX
+        };
+        typedef std::map<inverse_attr_record, std::vector<int>> entities_by_ref_t;
+        typedef std::map<int, std::vector<int>> entities_by_ref_excl_t;
+        typedef std::map<unsigned int, aggregate_of_instance::ptr> ref_map_t;
+        typedef map_transformer<identity_by_id_t, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
+        typedef entity_by_id_t::iterator iterator;
+
+        identity_by_id_t idenbyid_;
+
+        in_memory_file_storage()
+            : byid_(
+                &idenbyid_, 
+                [this](size_t v) { return byidentity_[v]; }, 
+                [](IfcUtil::IfcBaseClass* inst) { return inst->identity(); }
+            )
+        {}
+
+        class type_iterator : private entities_by_type_t::const_iterator {
+        public:
+            using iterator_category = std::forward_iterator_tag;
+            using value_type = entities_by_type_t::key_type;
+            using difference_type = typename entities_by_type_t::const_iterator::difference_type;
+            using pointer = value_type const*;
+            using reference = value_type const&;
+
+            type_iterator() : entities_by_type_t::const_iterator() {};
+
+            type_iterator(const entities_by_type_t::const_iterator& iter)
+                : entities_by_type_t::const_iterator(iter) {};
+
+            entities_by_type_t::key_type const* operator->() const {
+                return &entities_by_type_t::const_iterator::operator->()->first;
+            }
+
+            entities_by_type_t::key_type const& operator*() const {
+                return entities_by_type_t::const_iterator::operator*().first;
+            }
+
+            type_iterator& operator++() {
+                entities_by_type_t::const_iterator::operator++();
+                return *this;
+            }
+
+            type_iterator operator++(int) {
+                type_iterator tmp(*this);
+                operator++();
+                return tmp;
+            }
+
+            bool operator!=(const type_iterator& other) const {
+                const entities_by_type_t::const_iterator& self_ = *this;
+                const entities_by_type_t::const_iterator& other_ = other;
+                return self_ != other_;
+            }
+        };
+
+
+        static bool guid_map_;
+        static bool guid_map() { return guid_map_; }
+        static void guid_map(bool b) { guid_map_ = b; }
+
+        entity_by_id_t byid_;
+        // this is for simple types
+        entity_by_iden_t byidentity_;
+        // entities_by_type_t bytype_;
+        entities_by_type_t bytype_excl_;
+        // entities_by_ref_t byref_;
+        entities_by_ref_t byref_excl_;
+        entity_by_guid_t byguid_;
+
+        void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
+        void try_read_semicolon() const;
+
+        void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
+        void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
+
+        // @todo is this still used
+        IfcEntityInstanceData read(unsigned int index);
+        void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
+
+        file_open_status good_ = file_open_status::SUCCESS;
+
+        IfcUtil::IfcBaseClass* instance_by_id(int id);
+
+        void add_type_ref(IfcUtil::IfcBaseClass* new_entity) {
+            auto ty = new_entity->declaration().as_entity();
+            if (bytype_excl_.find(ty) == bytype_excl_.end()) {
+                bytype_excl_[ty].reset(new aggregate_of_instance());
+            }
+            bytype_excl_[ty]->push(new_entity);
+        }
+        void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) {
+            // @todo
+        }
+
+        void add_inverse_ref(IfcUtil::IfcBaseClass* new_entity) {
+            auto ty = new_entity->declaration().as_entity();
+            if (bytype_excl_.find(ty) == bytype_excl_.end()) {
+                bytype_excl_[ty].reset(new aggregate_of_instance());
+            }
+            bytype_excl_[ty]->push(new_entity);
+        }
+        void remove_inverse_ref(IfcUtil::IfcBaseClass* new_entity) {
+            // @todo
+        }
+
+        void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
+    };
+
+    struct rocks_db_file_storage {
+        // to make sure that instance pointer are constant during file lifetime
+        // cache instances because we want stable pointers
+        // @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side?
+        typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t;
+        entity_by_iden_cache_t instance_cache_;
+        
+        // lookup id->identity
+        typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
+        identity_by_id_t byid_;
+
+        // typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_identity_t;
+        // storage is now Instance name -> Identity -> Pointer (cached)
+        // entity_by_identity_t byidentity_;
+
+        // index in schema to binary serialized ids
+        typedef rocksdb_map_adapter<size_t, std::string> instance_id_str_by_type_t;
+        instance_id_str_by_type_t bytype_;
+
+        // guid -> id
+        typedef rocksdb_map_adapter<std::string, size_t> instance_id_by_guid_str_t;
+        instance_id_by_guid_str_t byguid_internal_;
+
+        // guid -> id -> instance
+        typedef map_transformer<rocksdb_map_adapter<std::string, size_t>, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_by_guid_t;
+        entity_by_guid_t byguid_;
+
+        rocksdb::DB* db;
+        IfcParse::IfcFile* file;
+
+        // @todo naming
+        rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile);
+
+        bool read_schema(const IfcParse::schema_definition*& schema) {
+            // @todo
+            schema = nullptr;
+            return true;
+        }
+        
+
+        IfcUtil::IfcBaseClass* assert_existance(size_t instanceId);
+
+        // @todo this could be another map_adapter?
+        class rocksdb_instance_iterator {
+        private:
+            rocksdb::Iterator* state_;
+            rocks_db_file_storage* storage_;
+
+            static constexpr char prefix_[] = "a|";
+
+            boost::optional<size_t> read_id_() const {
+                auto sv = state_->key().ToStringView();
+                auto ii = sv.find("|", 2);
+                if (ii != decltype(sv)::npos) {
+                    char* pEnd;
+                    long result = strtol(sv.data() + 2, &pEnd, 10);
+                    if (*pEnd == '|') {
+                        return (size_t)result;
+                    }
+                }
+                return boost::none;
+            }
+        public:
+            rocksdb_instance_iterator()
+                : state_(nullptr)
+                , storage_(nullptr)
+            {}
+            rocksdb_instance_iterator(rocks_db_file_storage* fs) 
+                : storage_(fs)
+            {
+                state_ = fs->db->NewIterator(rocksdb::ReadOptions());
+                state_->Seek(prefix_);
+                if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
+                    delete state_;
+                    state_ = nullptr;
+                }
+            }
+            rocksdb_instance_iterator& operator++() {
+                if (!state_) {
+                    return *this;
+                }
+                auto last_id = read_id_();
+                while (state_->Valid()) {
+                    state_->Next();
+                    // Stop if we've left the prefix range.
+                    if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
+                        delete state_;
+                        state_ = nullptr;
+                        break;
+                    }
+                    if (read_id_() != last_id) {
+                        break;
+                    }
+                }
+                return *this;
+            }
+            rocksdb_instance_iterator operator++(int) {
+                rocksdb_instance_iterator temp = *this;
+                ++(*this);
+                return temp;
+            }
+            bool operator==(const rocksdb_instance_iterator& other) const {
+                if (state_ == nullptr && other.state_ == nullptr) {
+                    return true;
+                } else {
+                    return read_id_() == other.read_id_();
+                }
+            }
+
+            bool operator!=(const rocksdb_instance_iterator& other) const {
+                return !(*this == other);
+            }
+
+            IfcUtil::IfcBaseClass* operator*() const;
+        };
+
+        // @todo merge iterators (template?)
+        class rocksdb_types_iterator {
+        private:
+            rocksdb::Iterator* state_;
+            const rocks_db_file_storage* storage_;
+
+            static constexpr char prefix_[] = "t|";
+
+            boost::optional<size_t> read_id_() const {
+                auto sv = state_->key().ToStringView();
+                auto ii = sv.find("|", 2);
+                if (ii != decltype(sv)::npos) {
+                    char* pEnd;
+                    long result = strtol(sv.data() + 2, &pEnd, 10);
+                    if (*pEnd == '|') {
+                        return (size_t)result;
+                    }
+                }
+                return boost::none;
+            }
+        public:
+            using iterator_category = std::forward_iterator_tag;
+            using value_type = const IfcParse::declaration*;
+            // @todo ?
+            using difference_type = ptrdiff_t;
+            using pointer = value_type const*;
+            using reference = value_type const&;
+
+            rocksdb_types_iterator()
+                : state_(nullptr)
+                , storage_(nullptr)
+            {}
+
+            rocksdb_types_iterator(const rocks_db_file_storage* fs)
+                : storage_(fs)
+            {
+                state_ = fs->db->NewIterator(rocksdb::ReadOptions());
+                state_->Seek(prefix_);
+                if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
+                    delete state_;
+                    state_ = nullptr;
+                }
+            }
+
+            rocksdb_types_iterator& operator++() {
+                if (!state_) {
+                    return *this;
+                }
+                auto last_id = read_id_();
+                while (state_->Valid()) {
+                    state_->Next();
+                    // Stop if we've left the prefix range.
+                    if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
+                        delete state_;
+                        state_ = nullptr;
+                        break;
+                    }
+                    if (read_id_() != last_id) {
+                        break;
+                    }
+                }
+                return *this;
+            }
+
+            rocksdb_types_iterator operator++(int) {
+                rocksdb_types_iterator temp = *this;
+                ++(*this);
+                return temp;
+            }
+
+            bool operator==(const rocksdb_types_iterator& other) const {
+                if (state_ == nullptr && other.state_ == nullptr) {
+                    return true;
+                } else {
+                    return read_id_() == other.read_id_();
+                }
+            }
+
+            bool operator!=(const rocksdb_types_iterator& other) const {
+                return !(*this == other);
+            }
+
+            const IfcParse::declaration* operator*() const;
+        };
+
+        using const_iterator = rocksdb_types_iterator;
+
+        void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
+        void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
+
+        // @todo a bit hard as a map because of value_type being an aggregate
+        void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
+        void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
+
+        IfcUtil::IfcBaseClass* instance_by_id(int id);
+
+        void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
+    };
+}
+
+enum filetype {
+    ifcspf,
+    ifcxml,
+    rocksdb,
+    autodetect
+};
+
+/// This class provides access to the entity instances in an IFC file
+/// The file takes ownership of instances added to this file and deletes them when the file is deleted.
 class IFC_PARSE_API IfcFile {
   public:
-      unresolved_references references_to_resolve;
-
-    typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
-    typedef boost::unordered_map<unsigned int, IfcUtil::IfcBaseClass*> entity_by_id_t;
-    typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_t;
-    typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_by_guid_t;
-    typedef std::tuple<int, short, short> inverse_attr_record;
-    enum INVERSE_ATTR {
-        INSTANCE_ID,
-        INSTANCE_TYPE,
-        ATTRIBUTE_INDEX
-    };
-    typedef std::map<inverse_attr_record, std::vector<int>> entities_by_ref_t;
-    typedef std::map<int, std::vector<int>> entities_by_ref_excl_t;
-    typedef std::map<unsigned int, aggregate_of_instance::ptr> ref_map_t;
-    typedef entity_by_id_t::const_iterator const_iterator;
-
-    class type_iterator : private entities_by_type_t::const_iterator {
-      public:
-        type_iterator() : entities_by_type_t::const_iterator(){};
-
-        type_iterator(const entities_by_type_t::const_iterator& iter)
-            : entities_by_type_t::const_iterator(iter){};
-
-        entities_by_type_t::key_type const* operator->() const {
-            return &entities_by_type_t::const_iterator::operator->()->first;
-        }
-
-        entities_by_type_t::key_type const& operator*() const {
-            return entities_by_type_t::const_iterator::operator*().first;
-        }
-
-        type_iterator& operator++() {
-            entities_by_type_t::const_iterator::operator++();
-            return *this;
-        }
-
-        type_iterator operator++(int) {
-            type_iterator tmp(*this);
-            operator++();
-            return tmp;
-        }
-
-        bool operator!=(const type_iterator& other) const {
-            const entities_by_type_t::const_iterator& self_ = *this;
-            const entities_by_type_t::const_iterator& other_ = other;
-            return self_ != other_;
-        }
-    };
-
-    static bool guid_map_;
-    static bool guid_map() { return guid_map_; }
-    static void guid_map(bool b) { guid_map_ = b; }
 
   private:
     typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_entity_map_t;
 
+    // @todo determine the constness of things (probably needs to be all const, we don't want to overwrite)
+    // @todo we have variant_iterator and MapVariant, we probably need to retain only one?
+public:
+    using const_iterator = variant_iterator<impl::in_memory_file_storage::iterator, impl::rocks_db_file_storage::const_iterator>;
+    using type_iterator = variant_iterator<impl::in_memory_file_storage::type_iterator, impl::rocks_db_file_storage::rocksdb_types_iterator>;
+    using storage_t = std::variant<std::monostate, impl::in_memory_file_storage, impl::rocks_db_file_storage>;
+    // @todo temporarily public for header
+    storage_t storage_;
+private:
     file_open_status good_ = file_open_status::SUCCESS;
 
     const IfcParse::schema_definition* schema_;
     const IfcParse::declaration* ifcroot_type_;
 
-    // std::vector<Argument*> internal_attribute_vector_, internal_attribute_vector_simple_type_;
-
-    entity_by_id_t byid_;
-    // this is for simple types
-    entity_by_iden_t byidentity_;
-    // entities_by_type_t bytype_;
-    entities_by_type_t bytype_excl_;
-    // entities_by_ref_t byref_;
-    entities_by_ref_t byref_excl_;
-    entity_by_guid_t byguid_;
     entity_entity_map_t entity_file_map_;
 
-    unsigned int MaxId;
+    unsigned int max_id_;
 
     IfcSpfHeader _header;
 
     void setDefaultHeaderValues();
 
-    void initialize_(IfcParse::IfcSpfStream* stream);
-
-    void build_inverses_(IfcUtil::IfcBaseClass*);
-
     typedef boost::multi_index_container<
         int,
         boost::multi_index::indexed_by<
@@ -195,37 +573,39 @@ class IFC_PARSE_API IfcFile {
     void process_deletion_();
 
   public:
-    IfcParse::IfcSpfLexer* tokens;
-    IfcParse::IfcSpfStream* stream;
-
 #ifdef USE_MMAP
     IfcFile(const std::string& path, bool mmap = false);
 #else
-    IfcFile(const std::string& path);
+    IfcFile(const std::string& path, filetype ty=ifcspf);
 #endif
     IfcFile(std::istream& stream, int length);
     IfcFile(void* data, int length);
     IfcFile(IfcParse::IfcSpfStream* stream);
     IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"));
 
-    /// Deleting the file will also delete all new instances that were added to the file (via memory allocation)
+    ~IfcFile() {
-    virtual ~IfcFile();
+        for (const auto& p : byidentity_) {
+            delete p.second;
+        }
+    }
 
     file_open_status good() const { return good_; }
 
-    /// Returns the first entity in the file, this probably is the entity
+    /// Returns the first entity in the range of instances contained in the model,
-    /// with the lowest id (EXPRESS ENTITY_INSTANCE_NAME)
+    /// in arbitrary order
-    const_iterator begin() const;
+    auto begin() const {
-    /// Returns the last entity in the file, this probably is the entity
+        return byid_.begin();
-    /// with the highest id (EXPRESS ENTITY_INSTANCE_NAME)
+    }
-    const_iterator end() const;
+
+    /// Returns the first entity in the range of instances contained in the model,
+    /// in arbitrary order
+    auto end() const {
+        return byid_.end();
+    }
 
     type_iterator types_begin() const;
     type_iterator types_end() const;
 
-    // type_iterator types_incl_super_begin() const;
-    // type_iterator types_incl_super_end() const;
-
     /// Returns all entities in the file that match the template argument.
     /// NOTE: This also returns subtypes of the requested type, for example:
     /// IfcWall will also return IfcWallStandardCase entities
@@ -294,9 +674,9 @@ class IFC_PARSE_API IfcFile {
 
     size_t getTotalInverses(int instance_id);
 
-    unsigned int FreshId() { return ++MaxId; }
+    unsigned int FreshId() { return ++max_id_; }
 
-    unsigned int getMaxId() const { return MaxId; }
+    unsigned int getMaxId() const { return max_id_; }
 
     const IfcParse::declaration* ifcroot_type() const { return ifcroot_type_; }
 
@@ -305,12 +685,6 @@ class IFC_PARSE_API IfcFile {
     IfcUtil::IfcBaseClass* addEntity(IfcUtil::IfcBaseClass* entity, int id = -1);
     void addEntities(aggregate_of_instance::ptr entities);
 
-    void batch() { batch_mode_ = true; }
-    void unbatch() {
-        process_deletion_();
-        batch_mode_ = false;
-    }
-
     /// Removes entity instance from file and unsets references.
     ///
     /// Attention when running removeEntity inside a loop over a list of entities to be removed.
@@ -327,20 +701,31 @@ class IFC_PARSE_API IfcFile {
 
     static std::string createTimestamp() ;
 
-    void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
-    void try_read_semicolon() const;
-
-    void register_inverse(unsigned, const IfcParse::entity* from_entity, Token, int attribute_index);
-    void register_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
-    void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
-
     const IfcParse::schema_definition* schema() const { return schema_; }
 
     std::pair<IfcUtil::IfcBaseClass*, double> getUnit(const std::string& unit_type);
 
     void build_inverses();
 
-    entity_by_guid_t& internal_guid_map() { return byguid_; };
+    // @todo variant apply_visitor
+    void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
+    void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
+
+    typedef VariantMap<impl::in_memory_file_storage::entity_by_guid_t, impl::rocks_db_file_storage::entity_by_guid_t> entity_by_guid_t;
+    entity_by_guid_t byguid_;
+    typedef VariantMap<impl::in_memory_file_storage::identity_by_id_t, impl::rocks_db_file_storage::identity_by_id_t> identity_by_id_t;
+    identity_by_id_t byid_;
+    typedef VariantMap<impl::in_memory_file_storage::entity_by_iden_t, impl::rocks_db_file_storage::entity_by_iden_cache_t> entity_by_iden_t;
+    entity_by_iden_t byidentity_;
+
+    // @todo
+    entity_by_guid_t internal_guid_map() { return byguid_; };
+
+    void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
+    void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
+    void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
+
+    void build_inverses_(IfcUtil::IfcBaseClass*);
 };
 
 #ifdef WITH_IFCXML

src/ifcparse/IfcHierarchyHelper.h

@@ -434,17 +434,17 @@ class IFC_PARSE_API IfcHierarchyHelper : public IfcParse::IfcFile {
             aggregate_of_instance::ptr related_objects(new aggregate_of_instance);
             related_objects->push(related_object);
 
-            IfcEntityInstanceData data = IfcEntityInstanceData(storage_t(T::Class().attribute_count()));
+            IfcEntityInstanceData data = IfcEntityInstanceData(in_memory_attribute_storage(T::Class().attribute_count()));
-            data.storage_.set(0, (std::string)IfcParse::IfcGlobalId());
+            data.set_attribute_value(0, (std::string)IfcParse::IfcGlobalId());
-            data.storage_.set(1, owner_hist);
+            data.set_attribute_value(1, owner_hist);
             int relating_index = 4;
             int related_index = 5;
             if (T::Class().name() == "IfcRelContainedInSpatialStructure" || std::is_base_of<typename Schema::IfcRelDefines, T>::value) {
                 // some classes have attributes reversed.
                 std::swap(relating_index, related_index);
             }
-            data.storage_.set(relating_index, relating_object);
+            data.set_attribute_value(relating_index, relating_object);
-            data.storage_.set(related_index, related_objects);
+            data.set_attribute_value(related_index, related_objects);
 
             T* t = (T*)Schema::get_schema().instantiate(&T::Class(), std::move(data));
             addEntity(t);

src/ifcparse/IfcParse.h

@@ -164,8 +164,6 @@ class IFC_PARSE_API IfcSpfLexer {
     void TokenString(unsigned int offset, std::string& result);
 };
 
-IFC_PARSE_API IfcEntityInstanceData read(unsigned int index, IfcFile* file);
-
 IFC_PARSE_API aggregate_of_instance::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level = -1);
 
 IFC_PARSE_API aggregate_of_instance::ptr traverse_breadth_first(IfcUtil::IfcBaseClass* instance, int max_level = -1);

src/ifcparse/IfcSpfHeader.cpp

@@ -37,44 +37,69 @@ static const char* const DATA = "DATA";
 using namespace IfcParse;
 
 namespace {
-    IfcEntityInstanceData read_from_file(IfcFile* f, size_t s) {
+    IfcEntityInstanceData read_from_spf_file(IfcFile* f, size_t s) {
-        parse_context pc;
+        return std::visit([f, s](auto& m) {
-        f->tokens->Next();
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
-        f->load(-1, nullptr, pc, -1);
+                parse_context pc;
-        return pc.construct(-1, f->references_to_resolve, nullptr, s);
+                m.tokens->Next();
+                m.load(-1, nullptr, pc, -1);
+                return pc.construct(-1, m.references_to_resolve, nullptr, s);
+            } else {
+                // std::unreachable();
+                return IfcEntityInstanceData(in_memory_attribute_storage(10));
+            }
+        }, f->storage_);
     }
 }
 
 HeaderEntity::HeaderEntity(const char* const datatype, size_t size, IfcFile* file)
     : datatype_(datatype)
     , file_(file)
-    , data_(file ? read_from_file(file, size) : IfcEntityInstanceData(storage_t(size)))
+    , data_(file ? read_from_spf_file(file, size) : IfcEntityInstanceData(in_memory_attribute_storage(size)))
 {}
 
 HeaderEntity::~HeaderEntity() {
 }
 
 void IfcSpfHeader::readSemicolon() {
-    if (!TokenFunc::isOperator(file_->tokens->Next(), ';')) {
+    std::visit([](auto& m) {
-        throw IfcException(std::string("Expected ;"));
+        if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
-    }
+            if (!TokenFunc::isOperator(m.tokens->Next(), ';')) {
+                throw IfcException(std::string("Expected ;"));
+            }
+        } else {
+            // std::unreachable();
+        }
+    }, file_->storage_);
 }
 
 void IfcSpfHeader::readParen() {
-    if (!TokenFunc::isOperator(file_->tokens->Next(), '(')) {
+    std::visit([](auto& m) {
-        throw IfcException(std::string("Expected ("));
+        if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
-    }
+            if (!TokenFunc::isOperator(m.tokens->Next(), '(')) {
+                throw IfcException(std::string("Expected ("));
+            }
+        } else {
+            // std::unreachable();
+        }
+    }, file_->storage_);
 }
 
 void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) {
-    if (TokenFunc::asStringRef(file_->tokens->Next()) != term) {
+    std::visit([this, term, trail](auto& m) {
-        throw IfcException(std::string("Expected " + term));
+        if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
-    }
+            if (TokenFunc::asStringRef(m.tokens->Next()) != term) {
-    if (trail == TRAILING_SEMICOLON) {
+                throw IfcException(std::string("Expected " + term));
-        readSemicolon();
+            }
-    } else if (trail == TRAILING_PAREN) {
+            if (trail == TRAILING_SEMICOLON) {
-        readParen();
+                readSemicolon();
-    }
+            } else if (trail == TRAILING_PAREN) {
+                readParen();
+            }
+        } else {
+            // std::unreachable();
+        }
+    }, file_->storage_);    
 }
 
 void IfcSpfHeader::read() {

src/ifcparse/IfcSpfHeader.h

@@ -39,14 +39,6 @@ class IFC_PARSE_API HeaderEntity {
     HeaderEntity(const char* const datatype, size_t size, IfcParse::IfcFile* file);
     virtual ~HeaderEntity();
 
-    void setValue(unsigned int index, const std::string& string) {
-        data_.storage_.set(index, string);
-    }
-
-    void setValue(unsigned int index, const std::vector<std::string>& strings) {
-        data_.storage_.set(index, strings);
-    }
-
   public:
     virtual size_t getArgumentCount() const {
         return data_.size();
@@ -71,8 +63,8 @@ class IFC_PARSE_API FileDescription : public HeaderEntity {
     std::vector<std::string> description() const { return data_.get_attribute_value(0); }
     std::string implementation_level() const { return data_.get_attribute_value(1); }
 
-    void description(const std::vector<std::string>& value) { setValue(0, value); }
+    void description(const std::vector<std::string>& value) { data_.set_attribute_value(0, value); }
-    void implementation_level(const std::string& value) { setValue(1, value); }
+    void implementation_level(const std::string& value) { data_.set_attribute_value(1, value); }
 };
 
 class IFC_PARSE_API FileName : public HeaderEntity {
@@ -87,13 +79,13 @@ class IFC_PARSE_API FileName : public HeaderEntity {
     std::string originating_system() const { return data_.get_attribute_value(5); }
     std::string authorization() const { return data_.get_attribute_value(6); }
 
-    void name(const std::string& value) { setValue(0, value); }
+    void name(const std::string& value) { data_.set_attribute_value(0, value); }
-    void time_stamp(const std::string& value) { setValue(1, value); }
+    void time_stamp(const std::string& value) { data_.set_attribute_value(1, value); }
-    void author(const std::vector<std::string>& value) { setValue(2, value); }
+    void author(const std::vector<std::string>& value) { data_.set_attribute_value(2, value); }
-    void organization(const std::vector<std::string>& value) { setValue(3, value); }
+    void organization(const std::vector<std::string>& value) { data_.set_attribute_value(3, value); }
-    void preprocessor_version(const std::string& value) { setValue(4, value); }
+    void preprocessor_version(const std::string& value) { data_.set_attribute_value(4, value); }
-    void originating_system(const std::string& value) { setValue(5, value); }
+    void originating_system(const std::string& value) { data_.set_attribute_value(5, value); }
-    void authorization(const std::string& value) { setValue(6, value); }
+    void authorization(const std::string& value) { data_.set_attribute_value(6, value); }
 };
 
 class IFC_PARSE_API FileSchema : public HeaderEntity {
@@ -102,7 +94,7 @@ class IFC_PARSE_API FileSchema : public HeaderEntity {
 
     std::vector<std::string> schema_identifiers() const { return data_.get_attribute_value(0); }
 
-    void schema_identifiers(const std::vector<std::string>& value) { setValue(0, value); }
+    void schema_identifiers(const std::vector<std::string>& value) { data_.set_attribute_value(0, value); }
 };
 
 class IFC_PARSE_API IfcSpfHeader {

src/ifcparse/IfcUtil.cpp

@@ -209,7 +209,7 @@ IfcUtil::IfcBaseEntity::IfcBaseEntity(IfcEntityInstanceData&& data)
 void IfcUtil::IfcBaseEntity::populate_derived() {
     for (auto it = declaration().as_entity()->derived().begin(); it != declaration().as_entity()->derived().end(); ++it) {
         if (*it) {
-            this->data().storage_.set(
+            this->data().set_attribute_value(
                 std::distance(declaration().as_entity()->derived().begin(), it),
                 Derived{}
             );

src/ifcparse/map_transformer.h

@@ -0,0 +1,120 @@
+/********************************************************************************
+*                                                                              *
+* This file is part of IfcOpenShell.                                           *
+*                                                                              *
+* IfcOpenShell is free software: you can redistribute it and/or modify         *
+* it under the terms of the Lesser GNU General Public License as published by  *
+* the Free Software Foundation, either version 3.0 of the License, or          *
+* (at your option) any later version.                                          *
+*                                                                              *
+* IfcOpenShell is distributed in the hope that it will be useful,              *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of               *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
+* Lesser GNU General Public License for more details.                          *
+*                                                                              *
+* You should have received a copy of the Lesser GNU General Public License     *
+* along with this program. If not, see <http://www.gnu.org/licenses/>.         *
+*                                                                              *
+********************************************************************************/
+
+#include <iterator>
+#include <type_traits>
+#include <utility>
+#include <functional>
+
+// map_transformer: wraps a map-like construct so that its iterator returns
+// a value_type where the mapped element is transformed via a function.
+template <typename BaseMap, typename Transform, typename TransformBack>
+class map_transformer {
+public:
+    using key_type = typename BaseMap::key_type;
+    using base_mapped_type = typename BaseMap::mapped_type;
+    using transformed_mapped_type = std::invoke_result_t<Transform, base_mapped_type>;
+    using value_type = std::pair<key_type, transformed_mapped_type>;
+
+private:
+    BaseMap* base_map_;
+    Transform transform_;
+
+    TransformBack transform_back_;
+
+public:
+    map_transformer(BaseMap* map, Transform transform, TransformBack transform_back)
+        : base_map_(map), transform_(transform), transform_back_(transform_back){}
+
+    class iterator {
+    public:
+        using base_iterator = typename BaseMap::iterator;
+        using iterator_category = std::forward_iterator_tag;
+        using difference_type = typename std::iterator_traits<base_iterator>::difference_type;
+        using key_type = typename BaseMap::key_type;
+        using transformed_mapped_type = std::invoke_result_t<Transform, typename BaseMap::mapped_type>;
+        using value_type = std::pair<key_type, transformed_mapped_type>;
+
+    private:
+        base_iterator base_it_;
+        Transform* transform_ptr_;
+
+        mutable value_type cached_value_;
+
+    public:
+        iterator() : base_it_(), transform_ptr_(nullptr) {}
+        iterator(base_iterator base_it, Transform* transform_ptr)
+            : base_it_(base_it), transform_ptr_(transform_ptr) {}
+
+        // On dereference, return a pair where the key is unchanged and the mapped value
+        // is the result of applying the transform to the underlying mapped value.
+        // @todo should these also all be const references so that key/value can be non-copyable (i.e unique_ptr)
+        value_type operator*() const {
+            auto base_val = *base_it_;
+            return { base_val.first, (*transform_ptr_)(base_val.second) };
+        }
+
+        // operator-> uses a mutable cache to return a pointer to the current value.
+        value_type* operator->() const {
+            cached_value_ = **this;
+            return &cached_value_;
+        }
+
+        iterator& operator++() {
+            ++base_it_;
+            return *this;
+        }
+
+        iterator operator++(int) {
+            iterator tmp(*this);
+            ++(*this);
+            return tmp;
+        }
+
+        bool operator==(const iterator& other) const {
+            return base_it_ == other.base_it_;
+        }
+
+        bool operator!=(const iterator& other) const {
+            return !(*this == other);
+        }
+    };
+
+    iterator begin() {
+        return iterator(base_map_->begin(), &transform_);
+    }
+
+    iterator end() {
+        return iterator(base_map_->end(), &transform_);
+    }
+
+    iterator find(const key_type& k) {
+        return iterator(base_map_->find(k), &transform_);
+    }
+
+    // @todo still not sure if this is a good idea, do we want to insert into the transformed map?
+    std::pair<iterator, bool> insert(const value_type& val) {
+        auto p = base_map_->insert({ val.first, transform_back_(val.second) });
+        return { iterator(p.first, &transform_), p.second };
+    }
+
+    size_t erase(const key_type& key) {
+        return base_map_->erase(key);
+    }
+};

src/ifcparse/map_variant.h

@@ -0,0 +1,174 @@
+/********************************************************************************
+*                                                                              *
+* This file is part of IfcOpenShell.                                           *
+*                                                                              *
+* IfcOpenShell is free software: you can redistribute it and/or modify         *
+* it under the terms of the Lesser GNU General Public License as published by  *
+* the Free Software Foundation, either version 3.0 of the License, or          *
+* (at your option) any later version.                                          *
+*                                                                              *
+* IfcOpenShell is distributed in the hope that it will be useful,              *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of               *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
+* Lesser GNU General Public License for more details.                          *
+*                                                                              *
+* You should have received a copy of the Lesser GNU General Public License     *
+* along with this program. If not, see <http://www.gnu.org/licenses/>.         *
+*                                                                              *
+********************************************************************************/
+
+#include <map>
+#include <variant>
+#include <tuple>
+#include <utility>
+#include <cstddef>
+#include <string>
+#include <iostream>
+
+// VariantMap: A map interface that delegates to one of several map types.
+// The underlying maps are referenced by pointers (not moved into the variant).
+// All map types must share the same key_type, mapped_type, and value_type.
+template <typename... Maps>
+class VariantMap {
+public:
+    // The variant holds a pointer to the map
+    using variant_type = std::variant<std::monostate, Maps*...>;
+    variant_type map_;
+
+    // Deduce common types from the first map type.
+    // @todo these are not common types, but just the 1st
+    using key_type = typename std::tuple_element<0, std::tuple<Maps...>>::type::key_type;
+    using mapped_type = typename std::tuple_element<0, std::tuple<Maps...>>::type::mapped_type;
+    using value_type = typename std::tuple_element<0, std::tuple<Maps...>>::type::value_type;
+
+    using underlying_iterator_variant = std::variant<typename Maps::iterator...>;
+
+    class iterator {
+    public:
+        using value_type = VariantMap::value_type;
+        using difference_type = std::ptrdiff_t;
+        using pointer = value_type*;
+        using reference = value_type;
+        using iterator_category = std::forward_iterator_tag;
+
+        underlying_iterator_variant it_var;
+
+        // mutable cache to support operator-> (so that it->second works)
+        mutable std::unique_ptr<value_type> cached_value_ptr_;
+
+        iterator() = default;
+
+        explicit iterator(underlying_iterator_variant v)
+            : it_var(std::move(v)) {}
+
+        iterator(const iterator& other)
+            : it_var(other.it_var), cached_value_ptr_(nullptr) {}
+
+        iterator& operator=(const iterator& other) {
+            if (this != &other) {
+                it_var = other.it_var;
+                cached_value_ptr_.reset(); // clear the cache
+            }
+            return *this;
+        }
+
+        value_type operator*() const {
+            return std::visit([](auto& it) -> value_type { return *it; }, it_var);
+        }
+
+        value_type* operator->() const {
+            // @todo we need to make a copy here (stored in unique_ptr) because the
+            // value_type appears to be pair<const K, V> instead of <K, V> or something
+            // related...
+            cached_value_ptr_ = std::make_unique<value_type>(**this);
+            return cached_value_ptr_.get();
+        }
+
+        iterator& operator++() {
+            std::visit([](auto& it) { ++it; }, it_var);
+            return *this;
+        }
+
+        iterator operator++(int) {
+            iterator tmp(*this);
+            ++(*this);
+            return tmp;
+        }
+
+        bool operator==(const iterator& other) const {
+            return it_var == other.it_var;
+        }
+
+        bool operator!=(const iterator& other) const {
+            return !(*this == other);
+        }
+    };
+
+    VariantMap() {}
+
+    template <typename MapT>
+    VariantMap(MapT* m) : map_(m) {}
+
+    iterator begin() const{
+        return std::visit([](auto m) -> iterator {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
+                return iterator{};
+            } else {
+                return iterator(m->begin());
+            }
+        }, map_);
+    }
+
+    iterator end() const {
+        return std::visit([](auto m) -> iterator {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
+                return iterator{};
+            } else {
+                return iterator(m->end());
+            }
+        }, map_);
+    }
+
+    iterator find(const key_type& key) const {
+        return std::visit([&key](auto m) -> iterator {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
+                return iterator{};
+            } else {
+                return iterator(m->find(key));
+            }
+        }, map_);
+    }
+
+    size_t erase(const key_type& key) {
+        return std::visit([&key](auto m) -> size_t {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
+                return size_t(0);
+            } else {
+                return m->erase(key);
+            }
+        }, map_);
+    }
+
+    size_t erase(const iterator& it) {
+        return std::visit([&it](auto m) -> size_t {
+            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
+                return size_t(0);
+            } else {
+                // @todo erasing by iterator would be more efficient
+                return m->erase(it->first);
+            }
+        }, map_);
+    }
+
+    std::pair<iterator, bool> insert(const value_type& val) {
+        return std::visit([this, &val](auto m) -> std::pair<iterator, bool> {
+            // @todo is monostate still necessary here?
+            if constexpr (!std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
+                auto result = m->insert(val);
+                return { iterator(result.first), result.second };
+            } else {
+                return { end(), false };
+            }
+        }, map_);
+    }
+};

src/ifcparse/parse_ifcxml.cpp

@@ -279,7 +279,7 @@ static void end_element(void* user, const xmlChar* tag) {
         }
         */
         // @todo
-        // back.inst()->data().storage_.set(back.idx(), elems);
+        // back.inst()->data().set_attribute_value(back.idx(), elems);
     }
 
     if (state->dialect == ifcxml_dialect_ifc2x3 && state->stack.back().ntype() == stack_node::node_instance) {
@@ -325,7 +325,7 @@ static void process_characters(void* user, const xmlChar* character, int len) {
         if (!val.empty()) {
             // type declaration always at idx 0
             visit_any([&state](auto& v) {
-                state->stack.back().inst()->data().storage_.set(0, v);
+                state->stack.back().inst()->data().set_attribute_value(0, v);
             }, val);
         }
     } else if (state_type == stack_node::node_header_entry) {
@@ -479,7 +479,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
                 }
             }
 
-            auto untyped = IfcEntityInstanceData(storage_t(decl->as_entity() != nullptr ? decl->as_entity()->attribute_count() : 1));
+            auto untyped = IfcEntityInstanceData(in_memory_attribute_storage(decl->as_entity() != nullptr ? decl->as_entity()->attribute_count() : 1));
 
             const IfcParse::entity* entity = decl->as_entity();
             if (entity != nullptr) {
@@ -494,7 +494,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
                         auto val = parse_attribute_value(attr->type_of_attribute(), pair.second);
                         if (!val.empty()) {
                             visit_any([&untyped, idx](auto& v) {
-                                untyped.storage_.set(idx, v);
+                                untyped.set_attribute_value(idx, v);
                             }, val);
                         }
                     } else {
@@ -531,7 +531,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
             IfcUtil::IfcBaseClass* inst;
             auto inst_ = create_instance(decl);
             instance_to_attribute(inst_, state->stack.back().idx(), inst);
-            // state->stack.back().inst()->data().storage_.set(state->stack.back().idx(), attr);
+            // state->stack.back().inst()->data().set_attribute_value(state->stack.back().idx(), attr);
             state->stack.push_back(stack_node::instance(id, inst));
         } else if (state_type == stack_node::node_aggregate) {
 
@@ -592,7 +592,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
                             if (inst != nullptr) {
                                 int idx = (*found)->entity_reference()->attribute_index(
                                     (*found)->attribute_reference());
-                                inst->data().storage_.set(idx, state->stack.back().inst());
+                                inst->data().set_attribute_value(idx, state->stack.back().inst());
                                 state->stack.push_back(stack_node::instance(id, inst));
                             } else {
                                 Logger::Error("Unknown attribute " + tagname);
@@ -663,12 +663,12 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
                     int idx = state->stack.back().inv_attr()->entity_reference()->attribute_index(
                         state->stack.back().inv_attr()->attribute_reference());
                     if (inst != nullptr) {
-                        inst->data().storage_.set(idx, state->stack.back().inst());
+                        inst->data().set_attribute_value(idx, state->stack.back().inst());
                     } else {
                         Logger::Error("Internal error, inverse attribute not processed");
                     }
                 } else if (state_type == stack_node::node_instance_attribute) {
-                    state->stack.back().inst()->data().storage_.set(state->stack.back().idx(), inst);
+                    state->stack.back().inst()->data().set_attribute_value(state->stack.back().idx(), inst);
                 }
 
                 if (entity == nullptr) {

src/ifcparse/rocksdb_map_adapter.h

@@ -0,0 +1,251 @@
+/********************************************************************************
+*                                                                              *
+* This file is part of IfcOpenShell.                                           *
+*                                                                              *
+* IfcOpenShell is free software: you can redistribute it and/or modify         *
+* it under the terms of the Lesser GNU General Public License as published by  *
+* the Free Software Foundation, either version 3.0 of the License, or          *
+* (at your option) any later version.                                          *
+*                                                                              *
+* IfcOpenShell is distributed in the hope that it will be useful,              *
+* but WITHOUT ANY WARRANTY; without even the implied warranty of               *
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
+* Lesser GNU General Public License for more details.                          *
+*                                                                              *
+* You should have received a copy of the Lesser GNU General Public License     *
+* along with this program. If not, see <http://www.gnu.org/licenses/>.         *
+*                                                                              *
+********************************************************************************/
+
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+#include <memory>
+#include <string>
+#include <utility>
+#include <iterator>
+#include <cstddef>
+
+// Serialization and deserialization primitives
+template <typename T>
+struct DefaultCodec;
+
+// Specialization for size_t.
+template <>
+struct DefaultCodec<size_t> {
+    std::string encode(const size_t& v) const {
+        std::string s(sizeof(v), 0);
+        size_t temp = v;
+        for (size_t i = 0; i < sizeof(v); i++) {
+            s[sizeof(v) - i - 1] = static_cast<char>(temp & 0xFF);
+            temp >>= 8;
+        }
+        return s;
+    }
+    size_t decode(const std::string& s) const {
+        size_t v = 0;
+        for (size_t i = 0; i < s.size(); i++) {
+            v = (v << 8) | static_cast<unsigned char>(s[i]);
+        }
+        return v;
+    }
+};
+
+// Specialization for std::string (identity)
+template <>
+struct DefaultCodec<std::string> {
+    std::string encode(const std::string& v) const {
+        return v;
+    }
+    std::string decode(const std::string& s) const {
+        return s;
+    }
+};
+
+template <typename KeyT>
+std::string key_to_string(const KeyT& key) {
+    if constexpr (std::is_same_v<KeyT, std::string>) {
+        return key;
+    } else {
+        return std::to_string(key);
+    }
+}
+
+// Convert from a string to a key. For non-string types, we assume numeric keys.
+template <typename KeyT>
+KeyT key_from_string(const std::string& s) {
+    // @todo tuples
+    if constexpr (std::is_same_v<KeyT, std::string>) {
+        return s;
+    } else if constexpr (std::is_integral_v<KeyT>) {
+        return static_cast<KeyT>(std::stoll(s));
+    } else {
+        static_assert(sizeof(KeyT) == 0, "key_from_string not implemented for this type");
+    }
+}
+
+// rocksdb_map_adapter: a std::map-like interface on a RocksDB keyspace with a given prefix.
+// The mapped_type is templated and encoded/decoded via Codec.
+template <typename KeyT, typename MappedT, typename Codec = DefaultCodec<MappedT>>
+class rocksdb_map_adapter {
+public:
+    using key_type = KeyT;
+    using mapped_type = MappedT;
+    using value_type = std::pair<key_type, mapped_type>;
+
+private:
+    rocksdb::DB* db_;
+    std::string prefix_;
+    Codec codec_;
+
+public:
+    rocksdb_map_adapter(rocksdb::DB* db, const std::string& prefix)
+        : db_(db), prefix_(prefix), codec_(Codec{}) {}
+
+    class iterator {
+    public:
+        using value_type = std::pair<key_type, mapped_type>;
+        using difference_type = std::ptrdiff_t;
+        using iterator_category = std::forward_iterator_tag;
+        using pointer = value_type*;
+        using reference = value_type&;
+
+    private:
+        rocksdb::DB* db_;
+        std::string prefix_;
+        Codec codec_;
+        // When it_ is nullptr, this iterator is at end.
+        std::unique_ptr<rocksdb::Iterator> it_;
+        mutable value_type cached_value_;
+
+        void check_valid() {
+            if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_)) {
+                it_.reset();
+            }
+        }
+
+    public:
+        iterator() : db_(nullptr), prefix_(), codec_(Codec{}), it_(nullptr) {}
+
+        iterator(rocksdb::DB* db, const std::string& prefix,
+            std::unique_ptr<rocksdb::Iterator> iter, Codec codec = Codec{})
+            : db_(db), prefix_(prefix), codec_(codec), it_(std::move(iter))
+        {
+            check_valid();
+        }
+
+        iterator(const iterator& other)
+            : db_(other.db_), prefix_(other.prefix_), codec_(other.codec_)
+        {
+            if (other.it_) {
+                std::string curr = other.it_->key().ToString();
+                it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
+                it_->Seek(curr);
+                if (!it_->Valid() || it_->key().ToString() != curr)
+                    it_.reset();
+            }
+        }
+
+        iterator& operator=(const iterator& other) {
+            if (this != &other) {
+                db_ = other.db_;
+                prefix_ = other.prefix_;
+                codec_ = other.codec_;
+                if (other.it_) {
+                    std::string curr = other.it_->key().ToString();
+                    it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
+                    it_->Seek(curr);
+                    if (!it_->Valid() || it_->key().ToString() != curr)
+                        it_.reset();
+                } else {
+                    it_.reset();
+                }
+            }
+            return *this;
+        }
+
+        value_type operator*() const {
+            std::string full_key = it_->key().ToString();
+            std::string key_without_prefix = full_key.substr(prefix_.size());
+            std::string value_str = it_->value().ToString();
+            return { key_from_string<key_type>(key_without_prefix), codec_.decode(value_str) };
+        }
+
+        // operator-> uses a mutable cache to return a pointer to the current value.
+        value_type* operator->() const {
+            cached_value_ = **this;
+            return &cached_value_;
+        }
+
+        iterator& operator++() {
+            if (it_) {
+                it_->Next();
+                check_valid();
+            }
+            return *this;
+        }
+
+        iterator operator++(int) {
+            iterator tmp(*this);
+            ++(*this);
+            return tmp;
+        }
+
+        bool operator==(const iterator& other) const {
+            if (!it_ && !other.it_) return true;
+            if (it_ && other.it_)
+                return it_->key().ToString() == other.it_->key().ToString();
+            return false;
+        }
+
+        bool operator!=(const iterator& other) const {
+            return !(*this == other);
+        }
+    };
+
+    iterator begin() const {
+        auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
+        iter->Seek(prefix_);
+        if (iter->Valid() && iter->key().starts_with(prefix_)) {
+            return iterator(db_, prefix_, std::move(iter), codec_);
+        }
+        return end();
+    }
+
+    iterator end() const {
+        return iterator();
+    }
+
+    iterator find(const key_type& key) const {
+        std::string key_str = key_to_string(key);
+        std::string full_key = prefix_ + key_str;
+        auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
+        iter->Seek(full_key);
+        if (iter->Valid() && iter->key().ToString() == full_key)
+            return iterator(db_, prefix_, std::move(iter), codec_);
+        return end();
+    }
+
+    size_t erase(const key_type& key) {
+        std::string key_str = key_to_string(key);
+        std::string full_key = prefix_ + key_str;
+        rocksdb::Status s = db_->Delete(rocksdb::WriteOptions{}, full_key);
+        return s.ok() ? 1 : 0;
+    }
+
+    std::pair<iterator, bool> insert(const value_type& val) {
+        std::string key_str = key_to_string(val.first);
+        std::string full_key = prefix_ + key_str;
+        std::string existing;
+        rocksdb::Status s = db_->Get(rocksdb::ReadOptions{}, full_key, &existing);
+        if (s.ok()) {
+            // Key already exists.
+            return { find(val.first), false };
+        }
+        std::string encoded = codec_.encode(val.second);
+        s = db_->Put(rocksdb::WriteOptions{}, full_key, encoded);
+        if (!s.ok()) {
+            return { end(), false };
+        }
+        return { find(val.first), true };
+    }
+};

src/ifcparse/variantarray.h

@@ -42,18 +42,6 @@ namespace impl {
     template<class T, typename... Args>
     struct is_unique_ptr<std::unique_ptr<T, Args...>> : std::true_type {};
 
-    /*
-    // Trait to find index of type in parameter pack
-    template <typename T, typename... Ts>
-    struct TypeIndex;
-    template <typename T, typename... Ts>
-    struct TypeIndex<T, T, Ts...> : std::integral_constant<std::size_t, 0> {};
-    template <typename T, typename U, typename... Ts>
-    struct TypeIndex<T, U, Ts...> : std::integral_constant<std::size_t, 1 + TypeIndex<T, Ts...>::value> {};
-    template <typename T, typename... Ts>
-    constexpr std::size_t TypeIndex_v = TypeIndex<T, Ts...>::value;
-    */
-
     // Trait to find index of type in parameter pack considering inheritance
     template <typename T, typename... Ts>
     struct TypeIndex;
@@ -239,7 +227,7 @@ public:
         return apply_visitor_impl(std::forward<Visitor>(visitor), index, std::integral_constant<std::size_t, sizeof...(Types)>{});
     }
 
-    auto size() const {
+    size_t size() const {
         return size_and_indices_ ? size_and_indices_[0] : 0;
     }