From 9f49ad5f71f9b85adf08522f611ecfd6d4319b07 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 9 Mar 2025 21:20:16 +0100 Subject: [PATCH] Non-streaming rocksdb serializer nearing completion --- src/ifcparse/IfcBaseClass.h | 7 +- src/ifcparse/IfcEntityInstanceData.cpp | 95 +++++------ src/ifcparse/IfcEntityInstanceData.h | 7 +- src/ifcparse/IfcFile.cpp | 58 ++++--- src/ifcparse/IfcFile.h | 135 ++++++++-------- src/ifcparse/IfcParse.cpp | 208 ++++++++++++++++--------- src/ifcparse/IfcSIPrefix.cpp | 3 +- src/ifcparse/rocksdb_map_adapter.h | 87 ++++++++++- src/ifcparse/rocksdb_set_view.h | 195 +++++++++++++++++++++++ src/ifcparse/set_to_map_transformer.h | 113 ++++++++++++++ win/build-deps.cmd | 2 + 11 files changed, 672 insertions(+), 238 deletions(-) create mode 100644 src/ifcparse/rocksdb_set_view.h create mode 100644 src/ifcparse/set_to_map_transformer.h diff --git a/src/ifcparse/IfcBaseClass.h b/src/ifcparse/IfcBaseClass.h index 4a1a383598..9ecab3e669 100644 --- a/src/ifcparse/IfcBaseClass.h +++ b/src/ifcparse/IfcBaseClass.h @@ -97,12 +97,7 @@ protected: IfcEntityInstanceData data_; public: - IfcBaseClass(IfcEntityInstanceData&& data) - : identity_(counter_++) - , id_(0) - , file_(nullptr) - , data_(std::move(data)) - {} + IfcBaseClass(IfcEntityInstanceData&& data); const IfcEntityInstanceData& data() const { return data_; } IfcEntityInstanceData& data() { return data_; } diff --git a/src/ifcparse/IfcEntityInstanceData.cpp b/src/ifcparse/IfcEntityInstanceData.cpp index 793e8fda07..5a1c0905bf 100644 --- a/src/ifcparse/IfcEntityInstanceData.cpp +++ b/src/ifcparse/IfcEntityInstanceData.cpp @@ -31,7 +31,7 @@ public: namespace { template - inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_) + inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, bool is_entity, uint8_t index_) { if (storage_model_ == 0) { return array_.storage_ptr->get(index_); @@ -43,7 +43,7 @@ namespace { !std::is_same_v>, IfcUtil::IfcBaseClass>) { std::string str; - array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); impl::deserialize(array_.db_ptr, str, val); } return val; @@ -51,24 +51,29 @@ namespace { } template - inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_) + inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, bool is_entity, uint8_t index_) { if (storage_model_ == 0) { return array_.storage_ptr->has(index_); } else { std::string str; - array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + if constexpr (std::is_same_v) { + if (str.size() == 0) { + return true; + } + } return str[0] == TypeEncoder::encode_type(); } } - inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_) + inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, bool is_entity, uint8_t index_) { if (storage_model_ == 0) { return array_.storage_ptr->index(index_); } else { std::string str; - array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); return (size_t) str[0] - 'A'; } } @@ -77,57 +82,57 @@ namespace { AttributeValue::operator int() const { - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator bool() const { - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator double() const { - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator boost::logic::tribool() const { - if (dispatch_has_(array_, storage_model_, instance_name_, index_)) { - return dispatch_get_(array_, storage_model_, instance_name_, index_); + if (dispatch_has_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_)) { + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator std::string() const { - if (dispatch_has_(array_, storage_model_, instance_name_, index_)) { + if (dispatch_has_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_)) { // @todo this is silly, but the way things currently work, // @todo also we don't really need to store a reference to the enumeration type, when this same type is already stored on the definition of the entity and no other value can be provided. if (storage_model_ == 0) { - return dispatch_get_(array_, storage_model_, instance_name_, index_).value(); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_).value(); } else { std::string str; - array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); size_t v; memcpy(&v, str.data() + 1, sizeof(size_t)); - auto decl = schema_->declarations()[v]->as_enumeration_type(); + auto decl = array_.db_ptr->file->schema()->declarations()[v]->as_enumeration_type(); memcpy(&v, str.data() + 1 + sizeof(size_t), sizeof(size_t)); return decl->lookup_enum_value(v); } } - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator EnumerationReference() const { if (storage_model_ == 0) { - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } else { std::string str; - array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); size_t v; memcpy(&v, str.data() + 1, sizeof(size_t)); - auto decl = schema_->declarations()[v]->as_enumeration_type(); + auto decl = array_.db_ptr->file->schema()->declarations()[v]->as_enumeration_type(); memcpy(&v, str.data() + 5, sizeof(size_t)); return EnumerationReference(decl, v); } @@ -135,24 +140,24 @@ AttributeValue::operator EnumerationReference() const AttributeValue::operator boost::dynamic_bitset<>() const { - return dispatch_get_>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator IfcUtil::IfcBaseClass* () const { if (storage_model_ == 0) { - return dispatch_get_(array_, storage_model_, instance_name_, index_); + return dispatch_get_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } else { std::string str; - array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); size_t v; memcpy(&v, str.data() + 2, sizeof(size_t)); - if (str[1] == 'e') { + if (str[1] == 'i') { // entity reference, by #Name - return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_name); + return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::entityinstance_ref); } else if (str[1] == 't') { // type reference by Identity - return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_identity); + return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::typedecl_ref); } } @@ -160,47 +165,47 @@ AttributeValue::operator IfcUtil::IfcBaseClass* () const AttributeValue::operator std::vector() const { - return dispatch_get_>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator std::vector() const { - return dispatch_get_>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator std::vector() const { - return dispatch_get_>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator std::vector>() const { - return dispatch_get_>>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator boost::shared_ptr() const { - return dispatch_get_>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator std::vector>() const { - return dispatch_get_>>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator std::vector>() const { - return dispatch_get_>>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } AttributeValue::operator boost::shared_ptr() const { - return dispatch_get_>(array_, storage_model_, instance_name_, index_); + return dispatch_get_>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } bool AttributeValue::isNull() const { - return dispatch_has_(array_, storage_model_, instance_name_, index_); + return dispatch_has_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_); } unsigned int AttributeValue::size() const @@ -211,7 +216,7 @@ unsigned int AttributeValue::size() const IfcUtil::ArgumentType AttributeValue::type() const { - return static_cast(dispatch_index_(array_, storage_model_, instance_name_, index_)); + return static_cast(dispatch_index_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_)); } bool impl::serialize(std::string& val, const IfcUtil::IfcBaseClass* t) @@ -221,8 +226,8 @@ bool impl::serialize(std::string& val, const IfcUtil::IfcBaseClass* t) val[0] = TypeEncoder::encode_type(); // 1 = entity - stored by id (entity name) // 2 = type - stored by identity (internal counter in class) - val[1] = t->declaration().as_entity() ? 'e' : 't'; - size_t iden = t->declaration().as_entity() ? t->id() : t->identity(); + val[1] = t->declaration().as_entity() ? 'i' : 't'; + size_t iden = t->id() ? t->id() : t->identity(); memcpy(val.data() + 2, &iden, s); return true; } @@ -243,12 +248,12 @@ bool impl::serialize(std::string& val, const aggregate_of_instance::ptr& t) { // no attempt at alignment val.resize(t->size() * (sizeof(size_t) + 1) + 1); - val[0] = TypeEncoder::encode_type(); + val[0] = TypeEncoder::encode_type(); char* ptr = val.data() + 1; for (auto it = t->begin(); it != t->end(); ++it) { - *ptr = (*it)->declaration().as_entity() ? 'e' : 't'; + *ptr = (*it)->declaration().as_entity() ? 'i' : 't'; ptr++; - size_t iden = (*it)->declaration().as_entity() ? (*it)->id() : (*it)->identity(); + size_t iden = (*it)->id() ? (*it)->id() : (*it)->identity(); memcpy(ptr, &iden, sizeof(size_t)); ptr += sizeof(size_t); } @@ -328,10 +333,10 @@ bool impl::deserialize(IfcParse::impl::rocks_db_file_storage* storage, const std ptr++; size_t v; memcpy(&v, ptr, sizeof(size_t)); - if (tt == 'e') { - t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_name)); + if (tt == 'i') { + t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::entityinstance_ref)); } else if (tt == 't') { - t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_identity)); + t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::typedecl_ref)); } else { throw std::runtime_error(""); } @@ -352,7 +357,7 @@ void rocks_db_attribute_storage::set(void* storage, const IfcParse::declaration* impl::serialize(v, value); rdb_storage->db->Put( rocksdb::WriteOptions{}, - (is_header ? "h|" : "i|") + + (is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) + (is_header ? decl->name() : std::to_string(identity)) + "|" + std::to_string(index), v); } diff --git a/src/ifcparse/IfcEntityInstanceData.h b/src/ifcparse/IfcEntityInstanceData.h index afe61b8238..e4d13eb2b2 100644 --- a/src/ifcparse/IfcEntityInstanceData.h +++ b/src/ifcparse/IfcEntityInstanceData.h @@ -280,9 +280,8 @@ namespace impl { struct AttributeValue { uint8_t index_; uint8_t storage_model_ = 0; + uint8_t entity_or_type_ = 0; size_t instance_name_; - // @todo couple with db_ptr; - const IfcParse::schema_definition* schema_; union pointer_type { const in_memory_attribute_storage* storage_ptr; IfcParse::impl::rocks_db_file_storage* db_ptr; @@ -303,12 +302,12 @@ struct AttributeValue { , storage_model_(0) {} - AttributeValue(const IfcParse::schema_definition* schema, IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t index) + AttributeValue(IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t entity_or_type, uint8_t index) : index_(index) , array_(db) , storage_model_(1) , instance_name_(instance_name) - , schema_(schema) + , entity_or_type_(entity_or_type) {} operator int() const; diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index 8941e970e7..af585d1499 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -329,6 +329,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re return IfcEntityInstanceData(std::move(storage)); } +/* IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_iterator::operator*() const { auto it = storage_->byid_.find(*read_id_()); if (it != storage_->byid_.end()) { @@ -341,32 +342,22 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_i } } } +*/ const IfcParse::declaration* IfcParse::impl::rocks_db_file_storage::rocksdb_types_iterator::operator*() const { return storage_->file->schema()->declarations()[*read_id_()]; } IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(size_t number, instance_ref r) { - size_t name, identity; - std::string v; - if (r == by_identity) { - name = 0; - identity = number; - } else { - name = number; - auto it = byid_.find(name); - if (it == byid_.end()) { - throw std::runtime_error("Unable to lookup identity of name: #" + std::to_string(number)); - } - identity = it->second; - } - - decltype(instance_cache_)::const_iterator it = instance_cache_.find(identity); + decltype(instance_cache_)::const_iterator it = instance_cache_.find({ r, number }); if (it != instance_cache_.end()) { return it->second; } + + std::string v; - rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(identity) + "|t", &v); + // @todo should always be name, as we can/should not assign to identity + rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, (r == entityinstance_ref ? "i|" : "t|") + std::to_string(number) + "|_", &v); if (s.ok()) { size_t s; memcpy(&s, v.data(), sizeof(size_t)); @@ -375,17 +366,14 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s } auto decl = file->schema()->declarations()[s]; bool is_entity = decl->as_entity() != nullptr; - if (is_entity != (r == by_name)) { + if (is_entity != (r == entityinstance_ref)) { throw std::runtime_error("Incorrect reference"); } IfcEntityInstanceData data(rocks_db_attribute_storage{}); auto inst = file->schema()->instantiate(decl, std::move(data)); - inst->id_ = name; + inst->id_ = number; inst->file_ = file; - instance_cache_.insert({ inst->identity(), inst }); - if (is_entity) { - byid_.insert({ inst->id(), inst->identity() }); - } + instance_cache_.insert({ {r, number}, inst }); return inst; } throw std::runtime_error(""); @@ -410,11 +398,13 @@ IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& : file(ffile) , db(init_db(filepath)) , byguid_internal_(db, "g|") - , byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, by_name); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); }) - , byid_(db, "d|") + , byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, entityinstance_ref); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); }) + , instance_ids_(db, "i|") + , instance_by_name_(&instance_ids_, [this](size_t v) { return assert_existance(v, entityinstance_ref); }) , bytype_(db, "t|") + , byref_excl_(db, "v|") // @todo by_identity is probably not correct here, this mapping is Name -> Identity, so Fn should have access to full pair? - , byidentity_(&byid_, [this](size_t v) { return assert_existance(v, by_identity); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); }) + // , byidentity_(&byid_, [this](size_t v) { return assert_existance(v, by_identity); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); }) {} IfcParse::impl::rocks_db_file_storage::~rocks_db_file_storage() @@ -433,7 +423,8 @@ IfcParse::impl::rocks_db_file_storage::~rocks_db_file_storage() IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int id) { // @todo rename assert_existance() -> instance_by_id(); - return assert_existance(id, by_name); + // - no cannot be done, because it needs to differentiate between entity instances and typedecls + return assert_existance(id, entityinstance_ref); } void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::IfcBaseClass* inst) @@ -441,8 +432,8 @@ void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::If auto id = inst->id(); { - // compute next prefix that does not start with v|{id} - auto prefix = "v|" + id; + // compute next prefix that does not start with v|{id}| + auto prefix = "v|" + std::to_string(id) + "|"; auto it = db->NewIterator(rocksdb::ReadOptions()); it->Seek(prefix); while (it->Valid()) { @@ -472,7 +463,7 @@ void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::If // and update inverses from entity into other { - auto prefix = "v|" + name; + auto prefix = "v|" + std::to_string(name) + "|"; auto it = db->NewIterator(rocksdb::ReadOptions()); it->Seek(prefix); while (it->Valid() && it->key().starts_with(prefix)) { @@ -498,4 +489,11 @@ IfcUtil::IfcBaseClass* IfcParse::impl::in_memory_file_storage::instance_by_id(in throw IfcException("Instance #" + boost::lexical_cast(id) + " not found"); } return it->second; -} \ No newline at end of file +} + +IfcParse::IfcFile::~IfcFile() { + // @todo this does not make sense for rocksdb, because it would assert existance for the entire lazy model only to free the instances again + for (const auto& p : byid_) { + delete p.second; + } +} diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index 3ec8afdee4..14085046b6 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -25,8 +25,10 @@ #include "IfcSchema.h" #include "IfcSpfHeader.h" #include "rocksdb_map_adapter.h" +#include "rocksdb_set_view.h" #include "map_variant.h" #include "map_transformer.h" +#include "set_to_map_transformer.h" #include #include @@ -213,35 +215,24 @@ namespace impl { struct in_memory_file_storage { IfcParse::IfcSpfLexer* tokens; IfcParse::IfcSpfStream* stream; - IfcParse::IfcFile* file; unresolved_references references_to_resolve; typedef std::map entities_by_type_t; - typedef boost::unordered_map identity_by_id_t; - typedef boost::unordered_map entity_by_iden_t; - typedef std::map entity_by_guid_t; + typedef boost::unordered_map entity_instance_by_name_t; + typedef boost::unordered_map type_instance_by_name_t; + typedef std::map entity_instance_by_guid_t; typedef std::tuple inverse_attr_record; enum INVERSE_ATTR { INSTANCE_ID, INSTANCE_TYPE, ATTRIBUTE_INDEX }; - typedef std::map> entities_by_ref_t; - typedef std::map> entities_by_ref_excl_t; - typedef std::map ref_map_t; - typedef map_transformer, std::function> entity_by_id_t; - typedef entity_by_id_t::iterator iterator; - - in_memory_file_storage() - : byid_( - &idenbyid_, - [this](size_t v) { return byidentity_[v]; }, - [](IfcUtil::IfcBaseClass* inst) { return inst->identity(); } - ) - {} + typedef std::map> entities_by_ref_t; + typedef entity_instance_by_name_t::iterator iterator; + in_memory_file_storage() : tokens(nullptr), stream(nullptr), file(nullptr) {} in_memory_file_storage(const in_memory_file_storage&) = delete; in_memory_file_storage(const in_memory_file_storage&&) = delete; @@ -289,15 +280,11 @@ namespace impl { static bool guid_map() { return guid_map_; } static void guid_map(bool b) { guid_map_ = b; } - // this is for simple types - entity_by_iden_t byidentity_; - // entities_by_type_t bytype_; + entity_instance_by_name_t byid_; + type_instance_by_name_t tbyid_; entities_by_type_t bytype_excl_; - // entities_by_ref_t byref_; entities_by_ref_t byref_excl_; - entity_by_guid_t byguid_; - identity_by_id_t idenbyid_; - entity_by_id_t byid_; + entity_instance_by_guid_t byguid_; void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); void try_read_semicolon() const; @@ -315,33 +302,26 @@ namespace impl { void add_type_ref(IfcUtil::IfcBaseClass* new_entity) { auto ty = new_entity->declaration().as_entity(); - if (bytype_excl_.find(ty) == bytype_excl_.end()) { - bytype_excl_[ty].reset(new aggregate_of_instance()); + if (ty) { + if (bytype_excl_.find(ty) == bytype_excl_.end()) { + bytype_excl_[ty].reset(new aggregate_of_instance()); + } + bytype_excl_[ty]->push(new_entity); } - bytype_excl_[ty]->push(new_entity); } void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) { auto ty = new_entity->declaration().as_entity(); - auto it = bytype_excl_.find(ty); - if (it != bytype_excl_.end()) { - it->second->remove(new_entity); - if (it->second->size() == 0) { - bytype_excl_.erase(ty); + if (ty) { + auto it = bytype_excl_.find(ty); + if (it != bytype_excl_.end()) { + it->second->remove(new_entity); + if (it->second->size() == 0) { + bytype_excl_.erase(ty); + } } } } - void add_inverse_ref(IfcUtil::IfcBaseClass* new_entity) { - auto ty = new_entity->declaration().as_entity(); - if (bytype_excl_.find(ty) == bytype_excl_.end()) { - bytype_excl_[ty].reset(new aggregate_of_instance()); - } - bytype_excl_[ty]->push(new_entity); - } - void remove_inverse_ref(IfcUtil::IfcBaseClass* new_entity) { - // @todo - } - void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); template @@ -355,19 +335,30 @@ namespace impl { rocksdb::DB* db; IfcParse::IfcFile* file; + enum instance_ref { + typedecl_ref, + entityinstance_ref + }; + // to make sure that instance pointer are constant during file lifetime // cache instances because we want stable pointers // @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side? - typedef std::map entity_by_iden_cache_t; + typedef std::map, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t; entity_by_iden_cache_t instance_cache_; - - // lookup id->identity - typedef rocksdb_map_adapter identity_by_id_t; - identity_by_id_t byid_; + + // @todo all these size_ts should probably be uint32_t for consistency with in-mem storage - typedef map_transformer, std::function, std::function> entity_by_id_t; + // lookup id->identity + // typedef rocksdb_map_adapter identity_by_id_t; + // identity_by_id_t byid_; + typedef rocksdb_set_view instance_name_view_t; + instance_name_view_t instance_ids_; + typedef set_to_map_transformer> entity_instance_by_name_t; + entity_instance_by_name_t instance_by_name_; + + // typedef map_transformer, std::function, std::function> entity_by_id_t; // storage is now Instance name -> Identity -> Pointer (cached) - entity_by_id_t byidentity_; + // entity_by_id_t byidentity_; // index in schema to binary serialized ids typedef rocksdb_map_adapter instance_id_str_by_type_t; @@ -378,23 +369,28 @@ namespace impl { instance_id_by_guid_str_t byguid_internal_; // guid -> id -> instance - typedef map_transformer, std::function, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_by_guid_t; - entity_by_guid_t byguid_; + typedef map_transformer, std::function, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_instance_by_guid_t; + entity_instance_by_guid_t byguid_; + + typedef std::tuple inverse_attr_record; + enum INVERSE_ATTR { + INSTANCE_ID, + INSTANCE_TYPE, + ATTRIBUTE_INDEX + }; + typedef rocksdb_map_adapter> entities_by_ref_t; + entities_by_ref_t byref_excl_; // @todo naming rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file); ~rocks_db_file_storage(); bool read_schema(const IfcParse::schema_definition*& schema); - - enum instance_ref { - by_name, - by_identity - }; IfcUtil::IfcBaseClass* assert_existance(size_t instanceId, instance_ref r); // @todo this could be another map_adapter? + /* class rocksdb_instance_iterator { private: rocksdb::Iterator* state_; @@ -467,6 +463,7 @@ namespace impl { IfcUtil::IfcBaseClass* operator*() const; }; + */ // @todo merge iterators (template?) class rocksdb_types_iterator { @@ -554,7 +551,7 @@ namespace impl { }; // @todo rocksdb_instance_iterator? - using const_iterator = rocksdb_types_iterator; + using const_iterator = entity_instance_by_name_t::iterator; void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); @@ -634,11 +631,7 @@ private: // @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = ifcspf, const std::string& path = ""); - ~IfcFile() { - for (const auto& p : byidentity_) { - delete p.second; - } - } + ~IfcFile(); file_open_status good() const { return good_; } @@ -761,18 +754,14 @@ private: void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); - typedef VariantMap entity_by_guid_t; - entity_by_guid_t byguid_; - typedef VariantMap entity_by_id_t; + typedef VariantMap entity_instance_by_guid_t; + entity_instance_by_guid_t byguid_; + typedef VariantMap entity_by_id_t; entity_by_id_t byid_; - typedef VariantMap entity_by_iden_t; - entity_by_iden_t byidentity_; - typedef VariantMap identity_by_id_t; - identity_by_id_t idenbyid_; + typedef VariantMap entities_by_ref_t; + entities_by_ref_t byref_excl_; - - // @todo - entity_by_guid_t internal_guid_map() { return byguid_; }; + entity_instance_by_guid_t internal_guid_map() { return byguid_; }; void add_type_ref(IfcUtil::IfcBaseClass* new_entity); void remove_type_ref(IfcUtil::IfcBaseClass* new_entity); diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index 4c6848cb23..a09e6b7791 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -767,8 +767,8 @@ void IfcParse::impl::in_memory_file_storage::register_inverse(unsigned id_from, } void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) { - std::vector& ids = byref_excl_[{inst->id(), from_entity->index_in_schema(), attribute_index}]; - std::vector::iterator iter = std::find(ids.begin(), ids.end(), id_from); + auto& ids = byref_excl_[{inst->id(), from_entity->index_in_schema(), attribute_index}]; + auto iter = std::find(ids.begin(), ids.end(), id_from); if (iter == ids.end()) { // @todo inverses also need to be populated when multiple instances are added to a new file. // throw IfcParse::IfcException("Instance not found among inverses"); @@ -780,8 +780,9 @@ void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from namespace { template std::string to_string_fixed_width(const T& t, size_t w) { + // @todo currently inactive std::ostringstream oss; - oss << std::setfill('0') << std::setw(w) << t; + oss << /*std::setfill('0') << std::setw(w) <<*/ t; return oss.str(); } } @@ -824,44 +825,47 @@ void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from, void IfcParse::impl::rocks_db_file_storage::add_type_ref(IfcUtil::IfcBaseClass* new_entity) { - if (!new_entity->declaration().as_entity()) { - throw std::runtime_error("Type refs are only supposed to be used for entities"); - } - - size_t v = new_entity->id(); + size_t v; std::string s(sizeof(size_t), ' '); - memcpy(s.data(), &v, sizeof(size_t)); - // no merges yet, because the python client doesn't support them - // db->Merge(rocksdb::WriteOptions{}, "t|" + std::to_string(new_entity->declaration().index_in_schema()), s); - { - std::string current; - auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema()); - db->Get(rocksdb::ReadOptions{}, key, ¤t); - auto new_val = current + s; - db->Put(rocksdb::WriteOptions{}, key, new_val); + if (new_entity->declaration().as_entity()) { + v = new_entity->id(); + memcpy(s.data(), &v, sizeof(size_t)); + + // no merges yet, because the python client doesn't support them + // db->Merge(rocksdb::WriteOptions{}, "t|" + std::to_string(new_entity->declaration().index_in_schema()), s); + { + std::string current; + // @todo this uses the same key-namespace as typedecl instances, not a direct conflict, but also not very clear + auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema()); + db->Get(rocksdb::ReadOptions{}, key, ¤t); + auto new_val = current + s; + db->Put(rocksdb::WriteOptions{}, key, new_val); + } } // not only mapping also register type v = new_entity->declaration().index_in_schema(); memcpy(s.data(), &v, sizeof(size_t)); - db->Put(rocksdb::WriteOptions{}, "i|" + std::to_string(new_entity->identity()) + "|t", s); + db->Put(rocksdb::WriteOptions{}, (new_entity->declaration().as_entity() ? "i|" : "t|") + std::to_string(new_entity->id() ? new_entity->id() : new_entity->identity()) + "|_", s); } void IfcParse::impl::rocks_db_file_storage::remove_type_ref(IfcUtil::IfcBaseClass* new_entity) { - std::string s; - auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema()); - if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) { - std::vector vals(s.size() / sizeof(size_t)); - memcpy(vals.data(), s.data(), s.size()); - vals.erase(std::find(vals.begin(), vals.end(), (size_t)new_entity->id())); - s.resize(vals.size() * sizeof(size_t)); - memcpy(s.data(), vals.data(), s.size()); - db->Put(rocksdb::WriteOptions{}, key, s); + if (new_entity->declaration().as_entity()) { + std::string s; + auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema()); + if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) { + std::vector vals(s.size() / sizeof(size_t)); + memcpy(vals.data(), s.data(), s.size()); + vals.erase(std::find(vals.begin(), vals.end(), (size_t)new_entity->id())); + s.resize(vals.size() * sizeof(size_t)); + memcpy(s.data(), vals.data(), s.size()); + db->Put(rocksdb::WriteOptions{}, key, s); + } } - db->Delete(rocksdb::WriteOptions{}, "i|" + std::to_string(new_entity->identity()) + "|t"); + db->Delete(rocksdb::WriteOptions{}, (new_entity->declaration().as_entity() ? "i|" : "t|") + std::to_string(new_entity->id() ? new_entity->id() : new_entity->identity()) + "|_"); } namespace { @@ -1242,12 +1246,12 @@ void IfcUtil::IfcBaseClass::set_attribute_value(size_t i, const T& t) { void* const storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr; if constexpr (std::is_pointer_v) { if (t) { - data_.set_attribute_value(storage, &declaration(), identity(), i, t); + data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(), i, t); } else { - data_.set_attribute_value(storage, &declaration(), identity(), i, Blank{}); + data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(), i, Blank{}); } } else { - data_.set_attribute_value(storage, &declaration(), identity(),i, t); + data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(),i, t); } } auto new_attribute = get_attribute_value(i); @@ -1301,16 +1305,16 @@ IfcFile::IfcFile(const std::string& path, filetype ty) { // @todo unify these names, it's already confusing enough as it stands byid_ = decltype(byid_)(&std::get(storage_).byid_); - idenbyid_ = decltype(idenbyid_)(&std::get(storage_).idenbyid_); - byidentity_ = decltype(byidentity_)(&std::get(storage_).byidentity_); + byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); + // byidentity_ = decltype(byidentity_)(&std::get(storage_).byidentity_); } else { // @todo this can only be used for databases that already exist, because otherwise there is no way to specify the schema storage_.emplace<2>(path, this); std::get(storage_).read_schema(schema_); - byid_ = decltype(byid_)(&std::get(storage_).byidentity_); - idenbyid_ = decltype(idenbyid_)(&std::get(storage_).byid_); - byidentity_ = decltype(byidentity_)(&std::get(storage_).instance_cache_); + byid_ = decltype(byid_)(&std::get(storage_).instance_by_name_); + byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); + // byidentity_ = decltype(byidentity_)(&std::get(storage_).instance_cache_); } ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); } @@ -1347,14 +1351,14 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s std::get(storage_).file = this; byid_ = decltype(byid_)(&std::get(storage_).byid_); - idenbyid_ = decltype(idenbyid_)(&std::get(storage_).idenbyid_); - byidentity_ = decltype(byidentity_)(&std::get(storage_).byidentity_); + byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); + // byidentity_ = decltype(byidentity_)(&std::get(storage_).byidentity_); } else { storage_.emplace<2>(path, this); - byid_ = decltype(byid_)(&std::get(storage_).byidentity_); - idenbyid_ = decltype(idenbyid_)(&std::get(storage_).byid_); - byidentity_ = decltype(byidentity_)(&std::get(storage_).instance_cache_); + byid_ = decltype(byid_)(&std::get(storage_).instance_by_name_); + byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); + // byidentity_ = decltype(byidentity_)(&std::get(storage_).instance_cache_); } setDefaultHeaderValues(); } @@ -1492,8 +1496,8 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt Logger::Message(Logger::LOG_WARNING, ss.str()); } - idenbyid_[current_id] = instance->identity(); - byidentity_[instance->identity()] = instance; + // byidentity_[instance->identity()] = instance; + byid_.insert({ current_id, instance }); // @nb cannot assign to byid_; // byid_[current_id] = instance; @@ -1548,10 +1552,10 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found"); } else { - byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, it->second); + byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, it->second); } } else if (auto* inst = boost::get(v)) { - byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst); + byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst); } } else if (auto* v = boost::get>(&p.second)) { aggregate_of_instance::ptr instances(new aggregate_of_instance); @@ -1568,7 +1572,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt instances->push(*inst); } } - byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances); + byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances); } else if (auto* v = boost::get>>(&p.second)) { aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance); for (const auto& vi : *v) { @@ -1587,7 +1591,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt } instances->push(inner); } - byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances); + byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances); } } @@ -1760,7 +1764,12 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) if (entity->declaration().as_entity() == nullptr) { // While not a mapping that can be queried, we do need to free the instance later on // @todo. why (over?)write this when adding from the same file? - byidentity_.insert({ new_entity->identity(), new_entity }); + std::visit([new_entity](auto& m) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + // @todo not freed yet + m.tbyid_.insert({ new_entity->identity(), new_entity }); + } + }, storage_); } // If it is part of this file @@ -1786,10 +1795,23 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) } new_entity->file_ = this; + // A new entity instance name is generated and + // the instance is pointed to this file. + if (new_entity->declaration().as_entity() != nullptr) { + if (id == -1) { + new_entity->as()->set_id(FreshId()); + } else { + new_entity->as()->set_id((unsigned int)id); + if ((unsigned)id > max_id_) { + max_id_ = (unsigned)id; + } + } + } + void* own_storage = std::visit([](const auto& m) { return (void*)&m; }, storage_); void* other_storage = std::visit([](const auto& m) { return (void*)&m; }, other_file->storage_); for (size_t i = 0; i < (entity->declaration().as_entity() ? entity->declaration().as_entity()->attribute_count() : 1); ++i) { - entity->data().apply_visitor(other_storage, decl, entity->identity(), [this, i, decl, new_entity, own_storage](const auto& v) { + entity->data().apply_visitor(other_storage, decl, entity->id() ? entity->id() : entity->identity(), [this, i, decl, new_entity, own_storage](const auto& v) { using U = std::decay_t; // only need to copy non-instance attribute values, others are assigned below after mapping if constexpr (std::is_same_v) { @@ -1895,19 +1917,6 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) } } - // A new entity instance name is generated and - // the instance is pointed to this file. - if (new_entity->declaration().as_entity() != nullptr) { - if (id == -1) { - new_entity->as()->set_id(FreshId()); - } else { - new_entity->as()->set_id((unsigned int)id); - if ((unsigned)id > max_id_) { - max_id_ = (unsigned)id; - } - } - } - entity_file_map_.insert(entity_entity_map_t::value_type(entity->identity(), new_entity)); } @@ -1929,9 +1938,10 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) // The mapping by entity type is updated. const IfcParse::declaration* ty = &new_entity->declaration(); - if (ty->as_entity() != nullptr) { + // @nb happens always because this also registers the type of the instance in rocksdb + // if (ty->as_entity() != nullptr) { add_type_ref(new_entity); - } + // } if (ty->as_entity() != nullptr) { int new_id = -1; @@ -1956,17 +1966,27 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) ss << "Overwriting entity with id " << new_id; Logger::Message(Logger::LOG_WARNING, ss.str()); } - // The mapping by entity instance name is updated. - idenbyid_.insert({ new_id, new_entity->identity() }); - byidentity_.insert({ new_entity->identity(), new_entity }); + + // rocksdb instances are assumed to be create with file.create(); + std::visit([new_entity](auto& m) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + // @todo not freed yet + m.byid_.insert({ new_entity->identity(), new_entity }); + } + }, storage_); } else if (new_entity->file_ == nullptr) { // For non-entity instances, no mappings are updated, but the file // pointer has to be set, so that actual copies are created in subsequent // times. new_entity->file_ = this; - // While not a mapping that can be queried, we do need to free the instance - byidentity_.insert({ new_entity->identity(), new_entity }); + // rocksdb instances are assumed to be create with file.create(); + std::visit([new_entity](auto& m) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + // @todo not freed yet + m.tbyid_.insert({ new_entity->identity(), new_entity }); + } + }, storage_); } /* @@ -2077,7 +2097,8 @@ void IfcFile::removeEntity(IfcUtil::IfcBaseClass* entity) { } } - byid_.erase(byid_.find(id)); + //byid_.erase(byid_.find(id)); + byid_.erase(id); const IfcParse::declaration* ty = &entity->declaration(); @@ -2183,7 +2204,7 @@ aggregate_of_instance::ptr IfcFile::instances_by_type_excl_subtypes(const IfcPar std::vector vals(s.size() / sizeof(size_t)); memcpy(vals.data(), s.data(), s.size()); for (auto& v : vals) { - ret->push(x.assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_name)); + ret->push(x.assert_existance(v, IfcParse::impl::rocks_db_file_storage::entityinstance_ref)); } } return ret; @@ -2413,7 +2434,26 @@ aggregate_of_instance::ptr IfcFile::getInverse(int instance_id, const IfcParse:: } } } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { - // @todo + if (attribute_index == -1) { + // @todo no lower/upper_bounds() implemented yet + auto prefix = "v|" + std::to_string(instance_id) + "|"; + auto it = x.db->NewIterator(rocksdb::ReadOptions()); + it->Seek(prefix); + while (it->Valid() && it->key().starts_with(prefix)) { + std::vector vals(it->value().size() / sizeof(size_t)); + memcpy(vals.data(), it->value().data(), it->value().size()); + for (auto& v : vals) { + return_value->push(instance_by_id(v)); + } + } + } else { + auto it = x.byref_excl_.find({ instance_id, ent->index_in_schema(), attribute_index }); + if (it != x.byref_excl_.end()) { + for (auto& i : it->second) { + return_value->push(instance_by_id(i)); + } + } + } } }, storage_); }); @@ -2586,12 +2626,12 @@ std::atomic_uint32_t IfcUtil::IfcBaseClass::counter_(0); void IfcUtil::IfcBaseClass::unset_attribute_value(size_t index) { void* storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr; - data_.set_attribute_value(storage, &declaration(), identity(), index, Blank{}); + data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(), index, Blank{}); } AttributeValue IfcUtil::IfcBaseClass::get_attribute_value(size_t index) const { void* storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr; - return data_.get_attribute_value(storage, &declaration(), identity(), index); + return data_.get_attribute_value(storage, &declaration(), id() ? id() : identity(), index); } void IfcUtil::IfcBaseClass::toString(std::ostream& out, bool upper) const @@ -2606,7 +2646,7 @@ void IfcUtil::IfcBaseClass::toString(std::ostream& out, bool upper) const out << declaration().name(); } void* storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr; - data().toString(storage, &declaration(), identity(), out, upper); + data().toString(storage, &declaration(), id() ? id() : identity(), out, upper); } /* @@ -2623,8 +2663,7 @@ AttributeValue IfcEntityInstanceData::get_attribute_value(void* storage, const I if constexpr (std::is_same_v, in_memory_attribute_storage>) { return AttributeValue(&x, (uint8_t)index); } else if constexpr (std::is_same_v, rocks_db_attribute_storage>) { - // @todo - return AttributeValue(decl->schema(), (IfcParse::impl::rocks_db_file_storage*) storage, identity, index); + return AttributeValue((IfcParse::impl::rocks_db_file_storage*) storage, identity, decl->as_entity() ? 1 : 0, index); } else { return AttributeValue{}; } @@ -2643,6 +2682,21 @@ bool IfcParse::impl::rocks_db_file_storage::read_schema(const IfcParse::schema_d return false; } +IfcUtil::IfcBaseClass::IfcBaseClass(IfcEntityInstanceData&& data) + : identity_(counter_++) + , id_(0) + , file_(nullptr) + , data_(std::move(data)) +{ + /* + * @todo this is not allowed cannot call virtual func in constructor + if (!declaration().as_entity()) { + // @nb from v0.9 type decl instances have their own id, which may collide with instance names in the file + // but is otherwise unique + id_ = identity_; + } + */ +} template void IFC_PARSE_API IfcUtil::IfcBaseClass::set_attribute_value(size_t index, const Blank& value); diff --git a/src/ifcparse/IfcSIPrefix.cpp b/src/ifcparse/IfcSIPrefix.cpp index 09a768102a..22cf78d011 100644 --- a/src/ifcparse/IfcSIPrefix.cpp +++ b/src/ifcparse/IfcSIPrefix.cpp @@ -120,8 +120,7 @@ double IfcParse::get_SI_equivalent(typename Schema::IfcNamedUnit* named_unit) { if (component->declaration().is(Schema::IfcSIUnit::Class())) { si_unit = component->template as(); typename Schema::IfcValue* value = factor->ValueComponent(); - // @todo provide sufficient context - scale = value->data().get_attribute_value(nullptr, nullptr, 0, 0); + scale = value->as()->get_attribute_value(0); } } else if (named_unit->declaration().is(Schema::IfcSIUnit::Class())) { si_unit = named_unit->template as(); diff --git a/src/ifcparse/rocksdb_map_adapter.h b/src/ifcparse/rocksdb_map_adapter.h index 49f8b58339..9a01777bd6 100644 --- a/src/ifcparse/rocksdb_map_adapter.h +++ b/src/ifcparse/rocksdb_map_adapter.h @@ -25,10 +25,18 @@ #include #include +template +struct is_std_tuple : std::false_type {}; + +template +struct is_std_tuple> : std::true_type {}; + // Serialization and deserialization primitives template struct DefaultCodec; +// @todo specialize for integral types in one go + // Specialization for size_t. template <> struct DefaultCodec { @@ -46,6 +54,38 @@ struct DefaultCodec { } }; +// Specialization for uint32_t. +template <> +struct DefaultCodec { + std::string encode(const uint32_t& v) const { + std::string s(sizeof(v), 0); + memcpy(s.data(), &v, sizeof(v)); + return s; + } + uint32_t decode(const std::string& s) const { + uint32_t v = 0; + // @todo take min of sizeof(v), len(s) + // @todo unify all serialization primitives + memcpy(&v, s.data(), sizeof(v)); + return v; + } +}; + +// Specialization for std::vector. +template <> +struct DefaultCodec> { + std::string encode(const std::vector& vs) const { + std::string s(sizeof(uint32_t) * vs.size(), 0); + memcpy(s.data(), vs.data(), s.size()); + return s; + } + std::vector decode(const std::string& s) const { + std::vector vs(s.size() / sizeof(uint32_t), 0); + memcpy(vs.data(), s.data(), s.size()); + return vs; + } +}; + // Specialization for std::string (identity) template <> struct DefaultCodec { @@ -67,7 +107,7 @@ std::string key_to_string(const KeyT& key) { } // Convert from a string to a key. For non-string types, we assume numeric keys. -template +template ::value, int>::type = 0> KeyT key_from_string(const std::string& s) { // @todo tuples if constexpr (std::is_same_v) { @@ -79,6 +119,51 @@ KeyT key_from_string(const std::string& s) { } } +template +std::string tuple_to_string_impl(const Tuple& t, std::index_sequence) { + std::ostringstream oss; + // Unpack the tuple; add a pipe before each element except the first. + ((oss << (Is == 0 ? "" : "|") << std::to_string(std::get(t))), ...); + return oss.str(); +} + +template +std::string key_to_string(const std::tuple& key) { + return tuple_to_string_impl(key, std::index_sequence_for{}); +} + +// Helper: Convert a string token to the desired numeric type. +template +T convert_string(const std::string& token) { + if constexpr (std::is_integral_v) { + return static_cast(std::stoll(token)); + } else if constexpr (std::is_floating_point_v) { + return static_cast(std::stod(token)); + } else { + static_assert(sizeof(T) == 0, "convert_string not implemented for this type"); + } +} + +// Helper: Build a tuple from a vector of string tokens. +template +TupleT tuple_from_string_impl(const std::vector& tokens, std::index_sequence) { + return std::make_tuple(convert_string>(tokens[Is])...); +} + +template ::value, int>::type = 0> +TupleT key_from_string(const std::string& s) { + std::vector tokens; + std::istringstream iss(s); + std::string token; + while (std::getline(iss, token, '|')) { + tokens.push_back(token); + } + if (tokens.size() != std::tuple_size::value) { + throw std::runtime_error("Invalid tuple format"); + } + return tuple_from_string_impl(tokens, std::make_index_sequence::value>{}); +} + // rocksdb_map_adapter: a std::map-like interface on a RocksDB keyspace with a given prefix. // The mapped_type is templated and encoded/decoded via Codec. template > diff --git a/src/ifcparse/rocksdb_set_view.h b/src/ifcparse/rocksdb_set_view.h new file mode 100644 index 0000000000..c90d59678e --- /dev/null +++ b/src/ifcparse/rocksdb_set_view.h @@ -0,0 +1,195 @@ +/******************************************************************************** +* * +* This file is part of IfcOpenShell. * +* * +* IfcOpenShell is free software: you can redistribute it and/or modify * +* it under the terms of the Lesser GNU General Public License as published by * +* the Free Software Foundation, either version 3.0 of the License, or * +* (at your option) any later version. * +* * +* IfcOpenShell is distributed in the hope that it will be useful, * +* but WITHOUT ANY WARRANTY; without even the implied warranty of * +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +* Lesser GNU General Public License for more details. * +* * +* You should have received a copy of the Lesser GNU General Public License * +* along with this program. If not, see . * +* * +********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include + + +template +class rocksdb_set_view { +public: + using key_type = KeyT; + using value_type = key_type; + +private: + rocksdb::DB* db_; + std::string prefix_; + +public: + // Constructor: provide a pointer to an open RocksDB instance and the key-space prefix, + // e.g. "i|" + rocksdb_set_view(rocksdb::DB* db, const std::string& prefix) + : db_(db), prefix_(prefix) {} + + // --- Iterator --- + class iterator { + public: + using value_type = key_type; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + using pointer = const value_type*; + using reference = const value_type&; + + private: + rocksdb::DB* db_; + std::string prefix_; + std::unique_ptr it_; + mutable value_type cached_value_; + + // Helper: extract the key (i.e. the value) from the current RocksDB key. + value_type extract_current_value() const { + std::string full_key = it_->key().ToString(); + std::string remainder = full_key.substr(prefix_.size()); + size_t pos = remainder.find('|'); + std::string key_str = (pos != std::string::npos) ? remainder.substr(0, pos) : remainder; + return key_from_string(key_str); + } + + // Validates the current iterator state. + void check_valid() { + if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_)) + it_.reset(); + } + + public: + iterator() : db_(nullptr), prefix_(), it_(nullptr) {} + + iterator(rocksdb::DB* db, const std::string& prefix, + std::unique_ptr iter) + : db_(db), prefix_(prefix), it_(std::move(iter)) + { + check_valid(); + } + + iterator(const iterator& other) + : db_(other.db_), prefix_(other.prefix_) + { + if (other.it_) { + std::string curr = other.it_->key().ToString(); + it_.reset(db_->NewIterator(rocksdb::ReadOptions{})); + it_->Seek(curr); + if (!it_->Valid() || it_->key().ToString() != curr) + it_.reset(); + } + } + + iterator& operator=(const iterator& other) { + if (this != &other) { + db_ = other.db_; + prefix_ = other.prefix_; + if (other.it_) { + std::string curr = other.it_->key().ToString(); + it_.reset(db_->NewIterator(rocksdb::ReadOptions{})); + it_->Seek(curr); + if (!it_->Valid() || it_->key().ToString() != curr) + it_.reset(); + } else { + it_.reset(); + } + } + return *this; + } + + // Dereference: extract the key part from the RocksDB key. + value_type operator*() const { + return extract_current_value(); + } + + // Pointer access (via a cached value) + const value_type* operator->() const { + cached_value_ = **this; + return &cached_value_; + } + + // Pre-increment: advance the iterator and skip over any duplicate keys. + iterator& operator++() { + if (it_) { + // Record the current key value. + value_type curr = extract_current_value(); + do { + it_->Next(); + } while (it_ && it_->Valid() && it_->key().starts_with(prefix_) && + (extract_current_value() == curr)); + if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_)) + it_.reset(); + } + return *this; + } + + iterator operator++(int) { + iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const iterator& other) const { + if (!it_ && !other.it_) + return true; + if (it_ && other.it_) + return it_->key().ToString() == other.it_->key().ToString(); + return false; + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + }; + + // Returns an iterator to the first element in the key-space (or end() if none exist). + iterator begin() const { + auto iter = std::unique_ptr(db_->NewIterator(rocksdb::ReadOptions{})); + iter->Seek(prefix_); + if (iter->Valid() && iter->key().starts_with(prefix_)) + return iterator(db_, prefix_, std::move(iter)); + return end(); + } + + // Returns an iterator representing the end of the key-space. + iterator end() const { + return iterator(); + } + + // Read-only find: returns an iterator to the element with the given key if it exists. + iterator find(const key_type& key) const { + std::string key_str = key_to_string(key); + // Construct the search key: prefix + key_str + separator. + std::string start_key = prefix_ + key_str + "|"; + auto iter = std::unique_ptr(db_->NewIterator(rocksdb::ReadOptions{})); + iter->Seek(start_key); + if (iter->Valid() && iter->key().starts_with(prefix_)) { + std::string full_key = iter->key().ToString(); + std::string remainder = full_key.substr(prefix_.size()); + size_t pos = remainder.find('|'); + std::string found_key_str = (pos != std::string::npos) ? remainder.substr(0, pos) : remainder; + if (key_from_string(found_key_str) == key) + return iterator(db_, prefix_, std::move(iter)); + } + return end(); + } + + size_t erase(const key_type& key) { + // @todo + } +}; diff --git a/src/ifcparse/set_to_map_transformer.h b/src/ifcparse/set_to_map_transformer.h new file mode 100644 index 0000000000..21bf597f4f --- /dev/null +++ b/src/ifcparse/set_to_map_transformer.h @@ -0,0 +1,113 @@ +/******************************************************************************** +* * +* This file is part of IfcOpenShell. * +* * +* IfcOpenShell is free software: you can redistribute it and/or modify * +* it under the terms of the Lesser GNU General Public License as published by * +* the Free Software Foundation, either version 3.0 of the License, or * +* (at your option) any later version. * +* * +* IfcOpenShell is distributed in the hope that it will be useful, * +* but WITHOUT ANY WARRANTY; without even the implied warranty of * +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +* Lesser GNU General Public License for more details. * +* * +* You should have received a copy of the Lesser GNU General Public License * +* along with this program. If not, see . * +* * +********************************************************************************/ + +#include +#include +#include +#include + +// map_transformer: wraps a set-like construct so that its iterator returns +// a value_type where of the underlying set element as the key, with the value +// that same key transformed via a function. +template +class set_to_map_transformer { +public: + using key_type = typename BaseSet::value_type; + using transformed_mapped_type = std::invoke_result_t; + using value_type = std::pair; + using mapped_type = transformed_mapped_type; + +private: + BaseSet* base_map_; + Transform transform_; + +public: + set_to_map_transformer(BaseSet* map, Transform transform) + : base_map_(map), transform_(transform) {} + + class iterator { + public: + using base_iterator = typename BaseSet::iterator; + using iterator_category = std::forward_iterator_tag; + using difference_type = typename std::iterator_traits::difference_type; + using key_type = typename BaseSet::key_type; + using transformed_mapped_type = std::invoke_result_t; + using value_type = std::pair; + + private: + base_iterator base_it_; + Transform* transform_ptr_; + + mutable value_type cached_value_; + + public: + iterator() : base_it_(), transform_ptr_(nullptr) {} + iterator(base_iterator base_it, Transform* transform_ptr) + : base_it_(base_it), transform_ptr_(transform_ptr) {} + + // On dereference, return a pair where the key is the set value and the mapped value + // is the result of applying the transform to the underlying value. + value_type operator*() const { + auto base_val = *base_it_; + return { base_val, (*transform_ptr_)(base_val) }; + } + + // operator-> uses a mutable cache to return a pointer to the current value. + value_type* operator->() const { + cached_value_ = **this; + return &cached_value_; + } + + iterator& operator++() { + ++base_it_; + return *this; + } + + iterator operator++(int) { + iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const iterator& other) const { + return base_it_ == other.base_it_; + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + }; + + iterator begin() { + return iterator(base_map_->begin(), &transform_); + } + + iterator end() { + return iterator(base_map_->end(), &transform_); + } + + iterator find(const key_type& k) { + return iterator(base_map_->find(k), &transform_); + } + + size_t erase(const key_type& k) { + // @todo + return 0; + } +}; \ No newline at end of file diff --git a/win/build-deps.cmd b/win/build-deps.cmd index 5e2cf7e3d3..1f9581e652 100644 --- a/win/build-deps.cmd +++ b/win/build-deps.cmd @@ -195,6 +195,8 @@ IF "%IFCOS_INSTALL_PYTHON%"=="TRUE" ( echo PYTHONHOME=%PYTHONHOME%>>"%~dp0\%BUILD_DEPS_CACHE_PATH%" ) +goto :rocksdb + :proj IF EXIST "%INSTALL_DIR%\proj-9.2.1" (