diff --git a/src/ifcparse/IfcBaseClass.h b/src/ifcparse/IfcBaseClass.h index d0560b0df7..4a1a383598 100644 --- a/src/ifcparse/IfcBaseClass.h +++ b/src/ifcparse/IfcBaseClass.h @@ -117,6 +117,8 @@ public: void unset_attribute_value(size_t i); + AttributeValue get_attribute_value(size_t index) const; + uint32_t identity() const { return identity_; } uint32_t id() const { return id_; } @@ -131,7 +133,7 @@ class IFC_PARSE_API IfcBaseEntity : public IfcBaseClass { IfcBaseEntity(IfcEntityInstanceData&& data); IfcBaseEntity(size_t n) - : IfcBaseClass(IfcEntityInstanceData(storage_t(n))) + : IfcBaseClass(IfcEntityInstanceData(in_memory_attribute_storage(n))) {} virtual const IfcParse::declaration& declaration() const = 0; @@ -172,7 +174,7 @@ class IFC_PARSE_API IfcBaseType : public IfcBaseClass { {} IfcBaseType() - : IfcBaseClass(IfcEntityInstanceData(storage_t(1))) + : IfcBaseClass(IfcEntityInstanceData(in_memory_attribute_storage(1))) {} virtual const IfcParse::declaration& declaration() const = 0; diff --git a/src/ifcparse/IfcEntityInstanceData.cpp b/src/ifcparse/IfcEntityInstanceData.cpp index 6da3f5fd11..66fe985884 100644 --- a/src/ifcparse/IfcEntityInstanceData.cpp +++ b/src/ifcparse/IfcEntityInstanceData.cpp @@ -1,5 +1,6 @@ #include "IfcEntityInstanceData.h" #include "IfcBaseClass.h" +#include "IfcFile.h" // @todo is size() still needed? class SizeVisitor { @@ -28,100 +29,300 @@ public: int operator()(const aggregate_of_aggregate_of_instance::ptr& i) const { return i->size(); } }; +namespace { + + // Trait to detect contiguous containers (vector / string) + template + struct is_contiguous_container : std::false_type {}; + template + struct is_contiguous_container> : std::true_type {}; + template + struct is_contiguous_container> : std::true_type {}; + + template + bool serialize(std::string& val, const T& t) { + return false; + } + + template ::value, int>::type = 0> + bool serialize(std::string& val, const T& t) { + auto s = sizeof(typename T::value_type) * t.size(); + val.resize(s); + val[0] = TypeEncoder::encode_type(); + memcpy(val.data() + 1, t.data(), s); + return true; + } + + bool serialize(std::string& val, const IfcUtil::IfcBaseClass* t) { + auto s = sizeof(size_t); + val.resize(s + 2); + val[0] = TypeEncoder::encode_type(); + // 1 = entity - stored by id (entity name) + // 2 = type - stored by identity (internal counter in class) + val[1] = t->declaration().as_entity() ? 1 : 2; + size_t iden = t->declaration().as_entity() ? t->id() : t->identity(); + memcpy(val.data() + 2, &iden, s); + return true; + } + + + bool serialize(std::string& val, const EnumerationReference& v) { + auto s = sizeof(size_t); + val.resize(s * 2 + 1); + val[0] = TypeEncoder::encode_type(); + size_t vv = v.enumeration()->index_in_schema(); + memcpy(val.data() + 1, &vv, sizeof(size_t)); + vv = v.index(); + memcpy(val.data() + 1, &vv, sizeof(size_t)); + return true; + } + + bool serialize(std::string& val, aggregate_of_instance::ptr& t) { + std::vector ids; + // @nb this has to be identity, because needs to work for typedecls as well + std::transform(t->begin(), t->end(), std::back_inserter(ids), [](auto& x) { return x->identity(); }); + return false; + } + + bool serialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) { + return false; + } + + /* + template + bool deserialize(std::string& val, const T& t) {} + */ + + template ::value, int>::type = 0> + bool deserialize(std::string& val, T& t) { + // @todo vector of vector + if (val[0] != TypeEncoder::encode_type()) { + return false; + } + auto s = (val.size() - 1) / sizeof(typename T::value_type); + t.resize(s); + memcpy(t.data(), val.data() + 1, s * sizeof(typename T::value_type)); + return true; + } + + template || std::is_floating_point_v, int>::type = 0> + bool deserialize(std::string& val, T& t) { + if (val[0] != TypeEncoder::encode_type()) { + return false; + } + auto s = (val.size() - 1) / sizeof(T); + memcpy(&t, val.data() + 1, sizeof(T)); + return true; + } + + bool deserialize(std::string& val, boost::logic::tribool& t) { + if (val[0] != TypeEncoder::encode_type()) { + return false; + } + if (val[1] == 0) { + t = false; + } else if (val[1] == 1) { + t = true; + } else if (val[1] == 2) { + t = boost::logic::indeterminate; + } else { + return false; + } + } + + bool deserialize(std::string& val, boost::dynamic_bitset<>& t) { + if (val[0] != TypeEncoder::encode_type>()) { + return false; + } + t = boost::dynamic_bitset<>(val.substr(1)); + return true; + } + + bool deserialize(std::string& val, aggregate_of_instance::ptr& t) { + return false; + } + + bool deserialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) { + return false; + } +} + +namespace { + template + inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_) + { + if (storage_model_ == 0) { + return array_.storage_ptr->get(index_); + } else { + T val; + if constexpr ( + // the following types cannot be directly deserialized from rocksdb, but need to be constructed + !std::is_same_v && + !std::is_same_v>, IfcUtil::IfcBaseClass>) + { + std::string str; + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + deserialize(str, val); + } + return val; + } + } + + template + inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_) + { + if (storage_model_ == 0) { + return array_.storage_ptr->has(index_); + } else { + std::string str; + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + return str[0] == TypeEncoder::encode_type(); + } + } + + inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_) + { + if (storage_model_ == 0) { + return array_.storage_ptr->index(index_); + } else { + std::string str; + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + return (size_t) str[0] - 'A'; + } + } + +} + AttributeValue::operator int() const { - return array_->get(index_); + return dispatch_get_(array_, storage_model_, instance_name_, index_); } AttributeValue::operator bool() const { - return array_->get(index_); + return dispatch_get_(array_, storage_model_, instance_name_, index_); } AttributeValue::operator double() const { - return array_->get(index_); + return dispatch_get_(array_, storage_model_, instance_name_, index_); } AttributeValue::operator boost::logic::tribool() const { - if (array_->has(index_)) { - return array_->get(index_); + if (dispatch_has_(array_, storage_model_, instance_name_, index_)) { + return dispatch_get_(array_, storage_model_, instance_name_, index_); } - return array_->get(index_); + return dispatch_get_(array_, storage_model_, instance_name_, index_); } AttributeValue::operator std::string() const { - if (array_->has(index_)) { + if (dispatch_has_(array_, storage_model_, instance_name_, index_)) { // @todo this is silly, but the way things currently work, // @todo also we don't really need to store a reference to the enumeration type, when this same type is already stored on the definition of the entity and no other value can be provided. - return array_->get(index_).value(); + if (storage_model_ == 0) { + return dispatch_get_(array_, storage_model_, instance_name_, index_).value(); + } else { + std::string str; + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + size_t v; + memcpy(&v, str.data() + 1, sizeof(size_t)); + auto decl = schema_->declarations()[v]->as_enumeration_type(); + memcpy(&v, str.data() + 5, sizeof(size_t)); + return decl->lookup_enum_value(v); + } + } + return dispatch_get_(array_, storage_model_, instance_name_, index_); +} + +AttributeValue::operator EnumerationReference() const +{ + if (storage_model_ == 0) { + return dispatch_get_(array_, storage_model_, instance_name_, index_); + } else { + std::string str; + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + size_t v; + memcpy(&v, str.data() + 1, sizeof(size_t)); + auto decl = schema_->declarations()[v]->as_enumeration_type(); + memcpy(&v, str.data() + 5, sizeof(size_t)); + return EnumerationReference(decl, v); } - return array_->get(index_); } AttributeValue::operator boost::dynamic_bitset<>() const { - return array_->get>(index_); + return dispatch_get_>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator IfcUtil::IfcBaseClass* () const { - return array_->get(index_); + if (storage_model_ == 0) { + return dispatch_get_(array_, storage_model_, instance_name_, index_); + } else { + std::string str; + array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str); + size_t v; + memcpy(&v, str.data() + 1, sizeof(size_t)); + auto decl = schema_->declarations()[v]->as_enumeration_type(); + memcpy(&v, str.data() + 5, sizeof(size_t)); + return array_.db_ptr->assert_existance(v); + } } AttributeValue::operator std::vector() const { - return array_->get>(index_); + return dispatch_get_>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator std::vector() const { - return array_->get>(index_); + return dispatch_get_>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator std::vector() const { - return array_->get>(index_); + return dispatch_get_>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator std::vector>() const { - return array_->get>>(index_); + return dispatch_get_>>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator boost::shared_ptr() const { - return array_->get>(index_); + return dispatch_get_>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator std::vector>() const { - return array_->get>>(index_); + return dispatch_get_>>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator std::vector>() const { - return array_->get>>(index_); + return dispatch_get_>>(array_, storage_model_, instance_name_, index_); } AttributeValue::operator boost::shared_ptr() const { - return array_->get>(index_); + return dispatch_get_>(array_, storage_model_, instance_name_, index_); } bool AttributeValue::isNull() const { - return array_->has(index_); + return dispatch_has_(array_, storage_model_, instance_name_, index_); } unsigned int AttributeValue::size() const { - return array_->apply_visitor(SizeVisitor{}, index_); + // @todo + return array_.storage_ptr->apply_visitor(SizeVisitor{}, index_); } IfcUtil::ArgumentType AttributeValue::type() const { - return static_cast(array_->index(index_)); + return static_cast(dispatch_index_(array_, storage_model_, instance_name_, index_)); } diff --git a/src/ifcparse/IfcEntityInstanceData.h b/src/ifcparse/IfcEntityInstanceData.h index f96f23ba83..241e1bf7f5 100644 --- a/src/ifcparse/IfcEntityInstanceData.h +++ b/src/ifcparse/IfcEntityInstanceData.h @@ -25,6 +25,13 @@ #include "aggregate_of_instance.h" #include "IfcSchema.h" +#pragma push_macro("Handle") +#undef Handle + +#include + +#pragma pop_macro("Handle") + #include #include #include @@ -36,7 +43,7 @@ private: size_t index_; public: - EnumerationReference(const IfcParse::enumeration_type* enumeration, size_t index) + EnumerationReference(const IfcParse::enumeration_type* enumeration = nullptr, size_t index = 0) : enumeration_(enumeration) , index_(index) {} @@ -58,7 +65,12 @@ class Derived {}; class empty_aggregate_t {}; class empty_aggregate_of_aggregate_t {}; -typedef VariantArray < +template +struct parameter_pack { + static constexpr size_t size = sizeof...(Args); +}; + +typedef parameter_pack < // A null argument, it will always serialize to $ Blank, // @todo Derived is not really necessary anymore, just serialize correctly based on schema @@ -113,26 +125,75 @@ typedef VariantArray < std::vector>, // An aggregate of an aggregate of entities. E.g. ((#1, #2), (#3)) aggregate_of_aggregate_of_instance::ptr -> storage_t; +> type_variant_parameter_pack; + +template +struct pack_to_variant_array; + +template +struct pack_to_variant_array> { + using type = VariantArray; +}; + +using in_memory_attribute_storage = pack_to_variant_array::type; + +template +struct TypeEncoder_t; + +template +struct TypeEncoder_t> { + template + static char encode_type() { + return 'A' + ::impl::TypeIndex_v; + } +}; + +using TypeEncoder = TypeEncoder_t; struct MutableAttributeValue { int name_; uint8_t index_; }; +namespace IfcParse { + namespace impl { + class rocks_db_file_storage; + } +} + // short lived struct AttributeValue { - const storage_t* array_; uint8_t index_; - + uint8_t storage_model_ = 0; + size_t instance_name_; + // @todo couple with db_ptr; + IfcParse::schema_definition* schema_; + union pointer_type { + const in_memory_attribute_storage* storage_ptr; + IfcParse::impl::rocks_db_file_storage* db_ptr; + pointer_type(IfcParse::impl::rocks_db_file_storage* db) : db_ptr(db) {} + pointer_type(const in_memory_attribute_storage* ims) : storage_ptr(ims) {} + }; + pointer_type array_; + AttributeValue() - : array_(nullptr) - , index_(0) + : index_(0) + , array_((const in_memory_attribute_storage*)nullptr) + , storage_model_(0) {} - AttributeValue(const storage_t* arr, uint8_t index) - : array_(arr) - , index_(index) + AttributeValue(const in_memory_attribute_storage* arr, uint8_t index) + : index_(index) + , array_(arr) + , storage_model_(0) + {} + + AttributeValue(IfcParse::schema_definition* schema, IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t index) + : index_(index) + , array_(db) + , storage_model_(1) + , instance_name_(instance_name) + , schema_(schema) {} operator int() const; @@ -153,17 +214,60 @@ struct AttributeValue { operator std::vector>() const; operator boost::shared_ptr() const; + operator EnumerationReference() const; + bool isNull() const; unsigned int size() const; IfcUtil::ArgumentType type() const; }; +struct rocks_db_attribute_storage { +private: + IfcParse::impl::rocks_db_file_storage* fs; + + template + auto apply_visitor_impl(Visitor&& visitor, std::size_t idx, std::integral_constant) const { + return apply_visitor_impl(std::forward(visitor), idx, std::integral_constant{}); + } + + template + void apply_visitor_impl(Visitor&&, std::size_t, std::integral_constant) const { + throw std::runtime_error("Invalid variant index"); + } + +public: + size_t size() const { + // @todo + return 8; + } + + template + void set(std::size_t index, T&& value) { + // @todo + } + + template + bool has(std::size_t index) const { + // @todo + return false; + } + + template + auto apply_visitor(Visitor&& visitor, std::size_t index) const { + return apply_visitor_impl(std::forward(visitor), index, std::integral_constant{}); + } +}; + class IFC_PARSE_API IfcEntityInstanceData { public: - storage_t storage_; + std::variant storage_; - IfcEntityInstanceData(storage_t&& storage) + IfcEntityInstanceData(in_memory_attribute_storage&& storage) + : storage_(std::move(storage)) + {} + + IfcEntityInstanceData(rocks_db_attribute_storage&& storage) : storage_(std::move(storage)) {} @@ -182,15 +286,29 @@ class IFC_PARSE_API IfcEntityInstanceData { AttributeValue get_attribute_value(size_t index) const; - /* - template - void set_attribute_value(size_t index, const T& t); + template + void set_attribute_value(std::size_t index, T&& value) { + std::visit([&index, &value](auto& x) { + return x.set(index, value); + }, storage_); + } - void set_attribute_value(size_t index, AttributeValue&, IfcUtil::ArgumentType attr_type = IfcUtil::Argument_UNKNOWN); - */ + template + bool has_attribute_value(std::size_t index) const { + return std::visit([&index](const auto& x) { + return x.has(index); + }, storage_); + } + + template + auto apply_visitor(Visitor&& visitor, std::size_t index) const { + return std::visit([&index, &visitor](const auto& x) { + return x.apply_visitor(std::forward(visitor), index); + }, storage_); + } size_t size() const { - return storage_.size(); + return std::visit([](const auto& x) { return x.size(); }, storage_); } void toString(std::ostream&, bool upper = false, const IfcParse::entity* ent = nullptr) const; diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index 14892b96ff..a1ccfd1d4f 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -259,10 +259,10 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re } if (tokens_.empty()) { - return IfcEntityInstanceData(storage_t(0)); + return IfcEntityInstanceData(in_memory_attribute_storage(0)); } - storage_t storage(decl != nullptr + in_memory_attribute_storage storage(decl != nullptr ? (std::min)(parameter_types.size(), tokens_.size()) : tokens_.size() ); @@ -328,3 +328,96 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re return IfcEntityInstanceData(std::move(storage)); } + +IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_iterator::operator*() const { + auto it = storage_->byid_.find(*read_id_()); + if (it != storage_->byid_.end()) { + // @todo define an implicit std::to_string() in all map adapters with leading 0s + auto jt = storage_->instance_cache_.find(it->second); + if (jt != storage_->instance_cache_.end()) { + return jt->second; + } else { + return storage_->assert_existance(it->second); + } + } +} + +const IfcParse::declaration* IfcParse::impl::rocks_db_file_storage::rocksdb_types_iterator::operator*() const { + return storage_->file->schema()->declarations()[*read_id_()]; +} + +IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(size_t instanceId) { + std::string v; + rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instanceId) + "|t", &v); + if (s.ok()) { + size_t s; + memcpy(&s, v.data(), sizeof(size_t)); + auto decl = file->schema()->declarations()[s]; + IfcEntityInstanceData data(rocks_db_attribute_storage{}); + auto inst = file->schema()->instantiate(decl, std::move(data)); + inst->id_ = instanceId; + instance_cache_.insert({ inst->identity(), inst }); + byid_.insert({ inst->id(), inst->identity() }); + return inst; + } + throw std::runtime_error(""); +} + + +#include "rocksdb/merge_operator.h" + +namespace { + + class ConcatenateIdMergeOperator : public rocksdb::AssociativeMergeOperator { + public: + virtual bool Merge(const rocksdb::Slice& key, + const rocksdb::Slice* existing_value, + const rocksdb::Slice& value, + std::string* new_value, + rocksdb::Logger* logger) const override { + if (existing_value) { + new_value->assign(existing_value->data(), existing_value->size()); + new_value->append(value.data(), value.size()); + } else { + new_value->assign(value.data(), value.size()); + } + return true; + } + + virtual const char* Name() const override { + return "ConcatenateIdMergeOperator"; + } + }; + +} + + +// @todo naming +IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile) + : file(ffile) + // @todo db is not initialized here yet + , byguid_internal_(db, "g|") + , byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); }) + , byid_(db, "d|") + , bytype_(db, "t|") +{ + rocksdb::Options options; + options.create_if_missing = true; + options.merge_operator.reset(new ConcatenateIdMergeOperator()); + rocksdb::Status status = rocksdb::DB::Open(options, filepath, &db); +} + +IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int id) +{ + // @todo rename assert_existance() -> instance_by_id(); + return assert_existance(id); +} + +IfcUtil::IfcBaseClass* IfcParse::impl::in_memory_file_storage::instance_by_id(int id) +{ + auto it = byid_.find(id); + if (it == byid_.end()) { + throw IfcException("Instance #" + boost::lexical_cast(id) + " not found"); + } + return it->second; +} \ No newline at end of file diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index a61c9f94b7..65b6501502 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -24,6 +24,9 @@ #include "IfcParse.h" #include "IfcSchema.h" #include "IfcSpfHeader.h" +#include "rocksdb_map_adapter.h" +#include "map_variant.h" +#include "map_transformer.h" #include #include @@ -95,94 +98,469 @@ struct parse_context { IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size); }; -/// This class provides several static convenience functions and variables -/// and provide access to the entities in an IFC file +#include +#include +#include +#include +#include +#include + +template +class variant_iterator { +public: + // The variant type holding one of the underlying iterators. + using variant_type = std::variant; + + // Assuming that all iterator types have the same value_type, difference_type, etc. + using value_type = std::common_type_t::value_type...>; + using difference_type = std::common_type_t::difference_type...>; + using pointer = value_type*; + using reference = value_type&; + // For simplicity, we use input_iterator_tag; if all underlying iterators support more, + // you could compute the common iterator_category. + using iterator_category = std::input_iterator_tag; + + // Default constructor. + variant_iterator() = default; + + // Construct from any one of the underlying iterator types. + template + variant_iterator(Iterator it) : it_(it) { } + + // Dereference operator. + decltype(auto) operator*() const { + return std::visit([](const auto& iter) -> decltype(auto) { + return *iter; + }, it_); + } + + // Arrow operator. + decltype(auto) operator->() const { + return std::visit([](const auto& iter) -> decltype(auto) { + return iter.operator->(); + }, it_); + } + + // Pre-increment operator. + variant_iterator& operator++() { + std::visit([](auto& iter) { ++iter; }, it_); + return *this; + } + + // Post-increment operator. + variant_iterator operator++(int) { + variant_iterator temp(*this); + ++(*this); + return temp; + } + + // Pre-decrement operator. + variant_iterator& operator--() { + std::visit([](auto& iter) { --iter; }, it_); + return *this; + } + + // Post-decrement operator. + variant_iterator operator--(int) { + variant_iterator temp(*this); + --(*this); + return temp; + } + + // Equality comparison. + friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) { + return lhs.it_ == rhs.it_; + } + + // Inequality comparison. + friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) { + return !(lhs == rhs); + } + +private: + variant_type it_; +}; + +namespace impl { + struct in_memory_file_storage { + IfcParse::IfcSpfLexer* tokens; + IfcParse::IfcSpfStream* stream; + + IfcParse::IfcFile* file; + + unresolved_references references_to_resolve; + + typedef std::map entities_by_type_t; + typedef boost::unordered_map identity_by_id_t; + typedef boost::unordered_map entity_by_iden_t; + typedef std::map entity_by_guid_t; + typedef std::tuple inverse_attr_record; + enum INVERSE_ATTR { + INSTANCE_ID, + INSTANCE_TYPE, + ATTRIBUTE_INDEX + }; + typedef std::map> entities_by_ref_t; + typedef std::map> entities_by_ref_excl_t; + typedef std::map ref_map_t; + typedef map_transformer, std::function> entity_by_id_t; + typedef entity_by_id_t::iterator iterator; + + identity_by_id_t idenbyid_; + + in_memory_file_storage() + : byid_( + &idenbyid_, + [this](size_t v) { return byidentity_[v]; }, + [](IfcUtil::IfcBaseClass* inst) { return inst->identity(); } + ) + {} + + class type_iterator : private entities_by_type_t::const_iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = entities_by_type_t::key_type; + using difference_type = typename entities_by_type_t::const_iterator::difference_type; + using pointer = value_type const*; + using reference = value_type const&; + + type_iterator() : entities_by_type_t::const_iterator() {}; + + type_iterator(const entities_by_type_t::const_iterator& iter) + : entities_by_type_t::const_iterator(iter) {}; + + entities_by_type_t::key_type const* operator->() const { + return &entities_by_type_t::const_iterator::operator->()->first; + } + + entities_by_type_t::key_type const& operator*() const { + return entities_by_type_t::const_iterator::operator*().first; + } + + type_iterator& operator++() { + entities_by_type_t::const_iterator::operator++(); + return *this; + } + + type_iterator operator++(int) { + type_iterator tmp(*this); + operator++(); + return tmp; + } + + bool operator!=(const type_iterator& other) const { + const entities_by_type_t::const_iterator& self_ = *this; + const entities_by_type_t::const_iterator& other_ = other; + return self_ != other_; + } + }; + + + static bool guid_map_; + static bool guid_map() { return guid_map_; } + static void guid_map(bool b) { guid_map_ = b; } + + entity_by_id_t byid_; + // this is for simple types + entity_by_iden_t byidentity_; + // entities_by_type_t bytype_; + entities_by_type_t bytype_excl_; + // entities_by_ref_t byref_; + entities_by_ref_t byref_excl_; + entity_by_guid_t byguid_; + + void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); + void try_read_semicolon() const; + + void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); + void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); + + // @todo is this still used + IfcEntityInstanceData read(unsigned int index); + void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id); + + file_open_status good_ = file_open_status::SUCCESS; + + IfcUtil::IfcBaseClass* instance_by_id(int id); + + void add_type_ref(IfcUtil::IfcBaseClass* new_entity) { + auto ty = new_entity->declaration().as_entity(); + if (bytype_excl_.find(ty) == bytype_excl_.end()) { + bytype_excl_[ty].reset(new aggregate_of_instance()); + } + bytype_excl_[ty]->push(new_entity); + } + void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) { + // @todo + } + + void add_inverse_ref(IfcUtil::IfcBaseClass* new_entity) { + auto ty = new_entity->declaration().as_entity(); + if (bytype_excl_.find(ty) == bytype_excl_.end()) { + bytype_excl_[ty].reset(new aggregate_of_instance()); + } + bytype_excl_[ty]->push(new_entity); + } + void remove_inverse_ref(IfcUtil::IfcBaseClass* new_entity) { + // @todo + } + + void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); + }; + + struct rocks_db_file_storage { + // to make sure that instance pointer are constant during file lifetime + // cache instances because we want stable pointers + // @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side? + typedef std::map entity_by_iden_cache_t; + entity_by_iden_cache_t instance_cache_; + + // lookup id->identity + typedef rocksdb_map_adapter identity_by_id_t; + identity_by_id_t byid_; + + // typedef map_transformer, std::function, std::function> entity_by_identity_t; + // storage is now Instance name -> Identity -> Pointer (cached) + // entity_by_identity_t byidentity_; + + // index in schema to binary serialized ids + typedef rocksdb_map_adapter instance_id_str_by_type_t; + instance_id_str_by_type_t bytype_; + + // guid -> id + typedef rocksdb_map_adapter instance_id_by_guid_str_t; + instance_id_by_guid_str_t byguid_internal_; + + // guid -> id -> instance + typedef map_transformer, std::function, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_by_guid_t; + entity_by_guid_t byguid_; + + rocksdb::DB* db; + IfcParse::IfcFile* file; + + // @todo naming + rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile); + + bool read_schema(const IfcParse::schema_definition*& schema) { + // @todo + schema = nullptr; + return true; + } + + + IfcUtil::IfcBaseClass* assert_existance(size_t instanceId); + + // @todo this could be another map_adapter? + class rocksdb_instance_iterator { + private: + rocksdb::Iterator* state_; + rocks_db_file_storage* storage_; + + static constexpr char prefix_[] = "a|"; + + boost::optional read_id_() const { + auto sv = state_->key().ToStringView(); + auto ii = sv.find("|", 2); + if (ii != decltype(sv)::npos) { + char* pEnd; + long result = strtol(sv.data() + 2, &pEnd, 10); + if (*pEnd == '|') { + return (size_t)result; + } + } + return boost::none; + } + public: + rocksdb_instance_iterator() + : state_(nullptr) + , storage_(nullptr) + {} + rocksdb_instance_iterator(rocks_db_file_storage* fs) + : storage_(fs) + { + state_ = fs->db->NewIterator(rocksdb::ReadOptions()); + state_->Seek(prefix_); + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + } + } + rocksdb_instance_iterator& operator++() { + if (!state_) { + return *this; + } + auto last_id = read_id_(); + while (state_->Valid()) { + state_->Next(); + // Stop if we've left the prefix range. + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + break; + } + if (read_id_() != last_id) { + break; + } + } + return *this; + } + rocksdb_instance_iterator operator++(int) { + rocksdb_instance_iterator temp = *this; + ++(*this); + return temp; + } + bool operator==(const rocksdb_instance_iterator& other) const { + if (state_ == nullptr && other.state_ == nullptr) { + return true; + } else { + return read_id_() == other.read_id_(); + } + } + + bool operator!=(const rocksdb_instance_iterator& other) const { + return !(*this == other); + } + + IfcUtil::IfcBaseClass* operator*() const; + }; + + // @todo merge iterators (template?) + class rocksdb_types_iterator { + private: + rocksdb::Iterator* state_; + const rocks_db_file_storage* storage_; + + static constexpr char prefix_[] = "t|"; + + boost::optional read_id_() const { + auto sv = state_->key().ToStringView(); + auto ii = sv.find("|", 2); + if (ii != decltype(sv)::npos) { + char* pEnd; + long result = strtol(sv.data() + 2, &pEnd, 10); + if (*pEnd == '|') { + return (size_t)result; + } + } + return boost::none; + } + public: + using iterator_category = std::forward_iterator_tag; + using value_type = const IfcParse::declaration*; + // @todo ? + using difference_type = ptrdiff_t; + using pointer = value_type const*; + using reference = value_type const&; + + rocksdb_types_iterator() + : state_(nullptr) + , storage_(nullptr) + {} + + rocksdb_types_iterator(const rocks_db_file_storage* fs) + : storage_(fs) + { + state_ = fs->db->NewIterator(rocksdb::ReadOptions()); + state_->Seek(prefix_); + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + } + } + + rocksdb_types_iterator& operator++() { + if (!state_) { + return *this; + } + auto last_id = read_id_(); + while (state_->Valid()) { + state_->Next(); + // Stop if we've left the prefix range. + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + break; + } + if (read_id_() != last_id) { + break; + } + } + return *this; + } + + rocksdb_types_iterator operator++(int) { + rocksdb_types_iterator temp = *this; + ++(*this); + return temp; + } + + bool operator==(const rocksdb_types_iterator& other) const { + if (state_ == nullptr && other.state_ == nullptr) { + return true; + } else { + return read_id_() == other.read_id_(); + } + } + + bool operator!=(const rocksdb_types_iterator& other) const { + return !(*this == other); + } + + const IfcParse::declaration* operator*() const; + }; + + using const_iterator = rocksdb_types_iterator; + + void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); + void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); + + // @todo a bit hard as a map because of value_type being an aggregate + void add_type_ref(IfcUtil::IfcBaseClass* new_entity); + void remove_type_ref(IfcUtil::IfcBaseClass* new_entity); + + IfcUtil::IfcBaseClass* instance_by_id(int id); + + void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); + }; +} + +enum filetype { + ifcspf, + ifcxml, + rocksdb, + autodetect +}; + +/// This class provides access to the entity instances in an IFC file +/// The file takes ownership of instances added to this file and deletes them when the file is deleted. class IFC_PARSE_API IfcFile { public: - unresolved_references references_to_resolve; - - typedef std::map entities_by_type_t; - typedef boost::unordered_map entity_by_id_t; - typedef boost::unordered_map entity_by_iden_t; - typedef std::map entity_by_guid_t; - typedef std::tuple inverse_attr_record; - enum INVERSE_ATTR { - INSTANCE_ID, - INSTANCE_TYPE, - ATTRIBUTE_INDEX - }; - typedef std::map> entities_by_ref_t; - typedef std::map> entities_by_ref_excl_t; - typedef std::map ref_map_t; - typedef entity_by_id_t::const_iterator const_iterator; - - class type_iterator : private entities_by_type_t::const_iterator { - public: - type_iterator() : entities_by_type_t::const_iterator(){}; - - type_iterator(const entities_by_type_t::const_iterator& iter) - : entities_by_type_t::const_iterator(iter){}; - - entities_by_type_t::key_type const* operator->() const { - return &entities_by_type_t::const_iterator::operator->()->first; - } - - entities_by_type_t::key_type const& operator*() const { - return entities_by_type_t::const_iterator::operator*().first; - } - - type_iterator& operator++() { - entities_by_type_t::const_iterator::operator++(); - return *this; - } - - type_iterator operator++(int) { - type_iterator tmp(*this); - operator++(); - return tmp; - } - - bool operator!=(const type_iterator& other) const { - const entities_by_type_t::const_iterator& self_ = *this; - const entities_by_type_t::const_iterator& other_ = other; - return self_ != other_; - } - }; - - static bool guid_map_; - static bool guid_map() { return guid_map_; } - static void guid_map(bool b) { guid_map_ = b; } private: typedef std::map entity_entity_map_t; + // @todo determine the constness of things (probably needs to be all const, we don't want to overwrite) + // @todo we have variant_iterator and MapVariant, we probably need to retain only one? +public: + using const_iterator = variant_iterator; + using type_iterator = variant_iterator; + using storage_t = std::variant; + // @todo temporarily public for header + storage_t storage_; +private: file_open_status good_ = file_open_status::SUCCESS; const IfcParse::schema_definition* schema_; const IfcParse::declaration* ifcroot_type_; - // std::vector internal_attribute_vector_, internal_attribute_vector_simple_type_; - - entity_by_id_t byid_; - // this is for simple types - entity_by_iden_t byidentity_; - // entities_by_type_t bytype_; - entities_by_type_t bytype_excl_; - // entities_by_ref_t byref_; - entities_by_ref_t byref_excl_; - entity_by_guid_t byguid_; entity_entity_map_t entity_file_map_; - unsigned int MaxId; + unsigned int max_id_; IfcSpfHeader _header; void setDefaultHeaderValues(); - void initialize_(IfcParse::IfcSpfStream* stream); - - void build_inverses_(IfcUtil::IfcBaseClass*); - typedef boost::multi_index_container< int, boost::multi_index::indexed_by< @@ -195,37 +573,39 @@ class IFC_PARSE_API IfcFile { void process_deletion_(); public: - IfcParse::IfcSpfLexer* tokens; - IfcParse::IfcSpfStream* stream; - #ifdef USE_MMAP IfcFile(const std::string& path, bool mmap = false); #else - IfcFile(const std::string& path); + IfcFile(const std::string& path, filetype ty=ifcspf); #endif IfcFile(std::istream& stream, int length); IfcFile(void* data, int length); IfcFile(IfcParse::IfcSpfStream* stream); IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4")); - /// Deleting the file will also delete all new instances that were added to the file (via memory allocation) - virtual ~IfcFile(); + ~IfcFile() { + for (const auto& p : byidentity_) { + delete p.second; + } + } file_open_status good() const { return good_; } - /// Returns the first entity in the file, this probably is the entity - /// with the lowest id (EXPRESS ENTITY_INSTANCE_NAME) - const_iterator begin() const; - /// Returns the last entity in the file, this probably is the entity - /// with the highest id (EXPRESS ENTITY_INSTANCE_NAME) - const_iterator end() const; + /// Returns the first entity in the range of instances contained in the model, + /// in arbitrary order + auto begin() const { + return byid_.begin(); + } + + /// Returns the first entity in the range of instances contained in the model, + /// in arbitrary order + auto end() const { + return byid_.end(); + } type_iterator types_begin() const; type_iterator types_end() const; - // type_iterator types_incl_super_begin() const; - // type_iterator types_incl_super_end() const; - /// Returns all entities in the file that match the template argument. /// NOTE: This also returns subtypes of the requested type, for example: /// IfcWall will also return IfcWallStandardCase entities @@ -294,9 +674,9 @@ class IFC_PARSE_API IfcFile { size_t getTotalInverses(int instance_id); - unsigned int FreshId() { return ++MaxId; } + unsigned int FreshId() { return ++max_id_; } - unsigned int getMaxId() const { return MaxId; } + unsigned int getMaxId() const { return max_id_; } const IfcParse::declaration* ifcroot_type() const { return ifcroot_type_; } @@ -305,12 +685,6 @@ class IFC_PARSE_API IfcFile { IfcUtil::IfcBaseClass* addEntity(IfcUtil::IfcBaseClass* entity, int id = -1); void addEntities(aggregate_of_instance::ptr entities); - void batch() { batch_mode_ = true; } - void unbatch() { - process_deletion_(); - batch_mode_ = false; - } - /// Removes entity instance from file and unsets references. /// /// Attention when running removeEntity inside a loop over a list of entities to be removed. @@ -327,20 +701,31 @@ class IFC_PARSE_API IfcFile { static std::string createTimestamp() ; - void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); - void try_read_semicolon() const; - - void register_inverse(unsigned, const IfcParse::entity* from_entity, Token, int attribute_index); - void register_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); - void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); - const IfcParse::schema_definition* schema() const { return schema_; } std::pair getUnit(const std::string& unit_type); void build_inverses(); - entity_by_guid_t& internal_guid_map() { return byguid_; }; + // @todo variant apply_visitor + void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); + void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); + + typedef VariantMap entity_by_guid_t; + entity_by_guid_t byguid_; + typedef VariantMap identity_by_id_t; + identity_by_id_t byid_; + typedef VariantMap entity_by_iden_t; + entity_by_iden_t byidentity_; + + // @todo + entity_by_guid_t internal_guid_map() { return byguid_; }; + + void add_type_ref(IfcUtil::IfcBaseClass* new_entity); + void remove_type_ref(IfcUtil::IfcBaseClass* new_entity); + void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); + + void build_inverses_(IfcUtil::IfcBaseClass*); }; #ifdef WITH_IFCXML diff --git a/src/ifcparse/IfcHierarchyHelper.h b/src/ifcparse/IfcHierarchyHelper.h index a8fbdf444b..11a64caaf0 100644 --- a/src/ifcparse/IfcHierarchyHelper.h +++ b/src/ifcparse/IfcHierarchyHelper.h @@ -434,17 +434,17 @@ class IFC_PARSE_API IfcHierarchyHelper : public IfcParse::IfcFile { aggregate_of_instance::ptr related_objects(new aggregate_of_instance); related_objects->push(related_object); - IfcEntityInstanceData data = IfcEntityInstanceData(storage_t(T::Class().attribute_count())); - data.storage_.set(0, (std::string)IfcParse::IfcGlobalId()); - data.storage_.set(1, owner_hist); + IfcEntityInstanceData data = IfcEntityInstanceData(in_memory_attribute_storage(T::Class().attribute_count())); + data.set_attribute_value(0, (std::string)IfcParse::IfcGlobalId()); + data.set_attribute_value(1, owner_hist); int relating_index = 4; int related_index = 5; if (T::Class().name() == "IfcRelContainedInSpatialStructure" || std::is_base_of::value) { // some classes have attributes reversed. std::swap(relating_index, related_index); } - data.storage_.set(relating_index, relating_object); - data.storage_.set(related_index, related_objects); + data.set_attribute_value(relating_index, relating_object); + data.set_attribute_value(related_index, related_objects); T* t = (T*)Schema::get_schema().instantiate(&T::Class(), std::move(data)); addEntity(t); diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index d708f6152e..ad1648814a 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -687,7 +687,7 @@ std::string TokenFunc::toString(const Token& token) { // Reads the arguments from a list of token // Aditionally, registers the ids (i.e. #[\d]+) in the inverse map // -void IfcParse::IfcFile::load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { +void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { Token next = tokens->Next(); /* @@ -712,19 +712,19 @@ void IfcParse::IfcFile::load(unsigned entity_instance_name, const IfcParse::enti } else { return_value++; if (TokenFunc::isIdentifier(next) && entity) { - register_inverse(entity_instance_name, entity, next, attribute_index == -1 ? attribute_index_within_data : attribute_index); + register_inverse(entity_instance_name, entity, next.value_int, attribute_index == -1 ? attribute_index_within_data : attribute_index); } if (TokenFunc::isKeyword(next)) { try { - const auto* decl = schema_->declaration_by_name(TokenFunc::asStringRef(next)); + const auto* decl = file->schema()->declaration_by_name(TokenFunc::asStringRef(next)); parse_context ps; tokens->Next(); load(0, nullptr, ps, -1); - auto* simple_type_instance = schema_->instantiate(decl, ps.construct(-1, references_to_resolve, decl, boost::none)); + auto* simple_type_instance = file->schema()->instantiate(decl, ps.construct(-1, references_to_resolve, decl, boost::none)); //@todo decide addEntity(((IfcUtil::IfcBaseClass*)*entity)); context.push(simple_type_instance); - simple_type_instance->file_ = this; + simple_type_instance->file_ = file; } catch (IfcException& e) { Logger::Message(Logger::LOG_ERROR, e.what()); // #4070 We didn't actually capture an aggregate entry, undo length increment. @@ -741,19 +741,19 @@ void IfcParse::IfcFile::load(unsigned entity_instance_name, const IfcParse::enti // // Reads an Entity from the list of Tokens at the specified offset in the file // -IfcEntityInstanceData IfcParse::read(unsigned int i, IfcFile* f) { - Token datatype = f->tokens->Next(); +IfcEntityInstanceData IfcParse::impl::in_memory_file_storage::read(unsigned int i) { + Token datatype = tokens->Next(); if (!TokenFunc::isKeyword(datatype)) { throw IfcException("Unexpected token while parsing entity"); } - const IfcParse::declaration* ty = f->schema()->declaration_by_name(TokenFunc::asStringRef(datatype)); + const IfcParse::declaration* ty = file->schema()->declaration_by_name(TokenFunc::asStringRef(datatype)); parse_context pc; - f->tokens->Next(); - f->load(i, ty->as_entity(), pc, -1); - return IfcEntityInstanceData(pc.construct(i, f->references_to_resolve, ty, boost::none)); + tokens->Next(); + load(i, ty->as_entity(), pc, -1); + return IfcEntityInstanceData(pc.construct(i, references_to_resolve, ty, boost::none)); } -void IfcParse::IfcFile::try_read_semicolon() const { +void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const { unsigned int old_offset = tokens->stream->Tell(); Token semilocon = tokens->Next(); if (!TokenFunc::isOperator(semilocon, ';')) { @@ -761,18 +761,12 @@ void IfcParse::IfcFile::try_read_semicolon() const { } } -void IfcParse::IfcFile::register_inverse(unsigned id_from, const IfcParse::entity* from_entity, Token t, int attribute_index) { +void IfcParse::impl::in_memory_file_storage::register_inverse(unsigned id_from, const IfcParse::entity* from_entity, int inst_id, int attribute_index) { // Assume a check on token type has already been performed - const auto* e = from_entity; - byref_excl_[{t.value_int, e->index_in_schema(), attribute_index}].push_back(id_from); + byref_excl_[{inst_id, from_entity->index_in_schema(), attribute_index}].push_back(id_from); } -void IfcParse::IfcFile::register_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) { - const auto* e = from_entity; - byref_excl_[{inst->id(), e->index_in_schema(), attribute_index}].push_back(id_from); -} - -void IfcParse::IfcFile::unregister_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) { +void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) { std::vector& ids = byref_excl_[{inst->id(), from_entity->index_in_schema(), attribute_index}]; std::vector::iterator iter = std::find(ids.begin(), ids.end(), id_from); if (iter == ids.end()) { @@ -783,6 +777,40 @@ void IfcParse::IfcFile::unregister_inverse(unsigned id_from, const IfcParse::ent } } +namespace { + template + std::string to_string_fixed_width(const T& t, size_t w) { + std::ostringstream oss; + oss << std::setfill('0') << std::setw(w) << t; + return oss.str(); + } +} + +void IfcParse::impl::rocks_db_file_storage::register_inverse(unsigned id_from, const IfcParse::entity* from_entity, int inst_id, int attribute_index) { + static std::string s; + size_t v = id_from; + s.resize(sizeof(size_t)); + memcpy(s.data(), &v, sizeof(size_t)); + db->Merge( + rocksdb::WriteOptions{}, + "v|" + to_string_fixed_width(inst_id, 10) + "|" + to_string_fixed_width(from_entity->index_in_schema(), 4) + "|" + to_string_fixed_width(attribute_index, 2), + s); +} + +void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) { + static std::string s; + auto inst_id = inst->id(); + auto key = "v|" + to_string_fixed_width(inst_id, 10) + "|" + to_string_fixed_width(from_entity->index_in_schema(), 4) + "|" + to_string_fixed_width(attribute_index, 2); + if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) { + std::vector vals(s.size() / sizeof(size_t)); + memcpy(vals.data(), s.data(), s.size()); + vals.erase(std::find(vals.begin(), vals.end(), (size_t)id_from)); + s.resize(vals.size() * sizeof(size_t)); + memcpy(s.data(), vals.data(), s.size()); + db->Put(rocksdb::WriteOptions{}, key, s); + } +} + namespace { class StringBuilderVisitor : public boost::static_visitor { private: @@ -993,14 +1021,14 @@ void IfcEntityInstanceData::toString(std::ostream& ss, bool upper, const entity* if (i != 0) { ss << ","; } - if (storage_.has(i)) { + if (has_attribute_value(i)) { if (decl != nullptr && decl->derived()[i]) { ss << "*"; } else { ss << "$"; } } else { - storage_.apply_visitor(vis, i); + apply_visitor(vis, i); } } ss << ")"; @@ -1062,7 +1090,7 @@ class register_inverse_visitor { data_(data) {} void operator()(IfcUtil::IfcBaseClass* inst, int index) { - file_.register_inverse(data_->id(), data_->declaration().as_entity(), inst, index); + file_.register_inverse(data_->id(), data_->declaration().as_entity(), inst->id(), index); } }; @@ -1153,12 +1181,12 @@ void IfcUtil::IfcBaseClass::set_attribute_value(size_t i, const T& t) { if constexpr (std::is_pointer_v) { if (t) { - data_.storage_.set(i, t); + data_.set_attribute_value(i, t); } else { - data_.storage_.set(i, Blank{}); + data_.set_attribute_value(i, Blank{}); } } else { - data_.storage_.set(i, t); + data_.set_attribute_value(i, t); } auto new_attribute = data_.get_attribute_value(i); @@ -1175,7 +1203,7 @@ void IfcUtil::IfcBaseClass::set_attribute_value(size_t i, const T& t) { if (it != file_->internal_guid_map().end()) { Logger::Warning("Duplicate guid " + guid); } - file_->internal_guid_map()[guid] = file_->instance_by_id(this->id()); + file_->internal_guid_map().insert({ guid, file_->instance_by_id(this->id()) }); } catch (IfcParse::IfcException& e) { Logger::Error(e); } @@ -1194,64 +1222,77 @@ void IfcUtil::IfcBaseClass::set_attribute_value(const std::string& s, const T& t // #ifdef USE_MMAP IfcFile::IfcFile(const std::string& fn, bool mmap) { - initialize_(new IfcSpfStream(fn, mmap)); + IfcSpfStream s(fn, mmap); + storage_ = impl::in_memory_file_storage{}; + std::get(storage_).read_from_stream(&s); } #else -IfcFile::IfcFile(const std::string& path) { - IfcSpfStream s(path); - initialize_(&s); +IfcFile::IfcFile(const std::string& path, filetype ty) { + // @todo allow for rocksdb from path + if (ty == ifcspf) { + IfcSpfStream s(path); + storage_ = impl::in_memory_file_storage{}; + std::get(storage_).read_from_stream(&s, schema_, max_id_); + } else { + storage_ = impl::rocks_db_file_storage(path, this); + std::get(storage_).read_schema(schema_); + } + ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); } #endif IfcFile::IfcFile(std::istream& stream, int length) { IfcSpfStream s(stream, length); - initialize_(&s); + storage_ = impl::in_memory_file_storage{}; + std::get(storage_).read_from_stream(&s, schema_, max_id_); + ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); } IfcFile::IfcFile(void* data, int length) { IfcSpfStream s(data, length); - initialize_(&s); + storage_ = impl::in_memory_file_storage{}; + std::get(storage_).read_from_stream(&s, schema_, max_id_); + ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); } IfcFile::IfcFile(IfcParse::IfcSpfStream* s) { - initialize_(s); + storage_ = impl::in_memory_file_storage{}; + std::get(storage_).read_from_stream(s, schema_, max_id_); + ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); } IfcFile::IfcFile(const IfcParse::schema_definition* schema) - : schema_(schema), - ifcroot_type_(schema_->declaration_by_name("IfcRoot")), - MaxId(0), - tokens(0), - stream(0) { + : schema_(schema) + , ifcroot_type_(schema_->declaration_by_name("IfcRoot")) + , max_id_(0) +{ + storage_ = impl::in_memory_file_storage{}; setDefaultHeaderValues(); } -void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { +void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfStream* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) { // Initialize a "C" locale for locale-independent // number parsing. See comment above on line 41. init_locale(); - MaxId = 0; tokens = 0; - stream = 0; - schema_ = 0; - - // setDefaultHeaderValues(); - stream = s; if (!stream->valid) { + // @todo set good on parent file good_ = file_open_status::READ_ERROR; return; } - tokens = new IfcSpfLexer(stream, this); + tokens = new IfcSpfLexer(stream, file); std::vector schemas; - _header.file(this); - if (_header.tryRead()) { + // @todo this line makes no sense + file->header().file(file); + + if (file->header().tryRead()) { try { - schemas = _header.file_schema().schema_identifiers(); + schemas = file->header().file_schema().schema_identifiers(); } catch (...) { // Purposely empty catch block } @@ -1261,19 +1302,19 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { if (schemas.size() == 1) { try { - schema_ = IfcParse::schema_by_name(schemas.front()); + schema = IfcParse::schema_by_name(schemas.front()); } catch (const IfcParse::IfcException& e) { good_ = file_open_status::UNSUPPORTED_SCHEMA; Logger::Error(e); } } - if (schema_ == 0) { + if (schema == 0) { Logger::Message(Logger::LOG_ERROR, "No support for file schema encountered (" + boost::algorithm::join(schemas, ", ") + ")"); return; } - ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); + auto ifcroot_type_ = schema->declaration_by_name("IfcRoot"); boost::circular_buffer token_stream(3, Token()); @@ -1296,7 +1337,7 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { current_id = (unsigned)TokenFunc::asIdentifier(token_stream[0]); const IfcParse::declaration* entity_type; try { - entity_type = schema_->declaration_by_name(TokenFunc::asStringRef(token_stream[2])); + entity_type = schema->declaration_by_name(TokenFunc::asStringRef(token_stream[2])); } catch (const IfcException& ex) { Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream[2].startPos)); goto advance; @@ -1316,8 +1357,8 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { Logger::Error(e); break; } - instance = schema_->instantiate(entity_type, ps.construct(current_id, references_to_resolve, entity_type, boost::none)); - instance->file_ = this; + instance = schema->instantiate(entity_type, ps.construct(current_id, references_to_resolve, entity_type, boost::none)); + instance->file_ = file; instance->id_ = current_id; /// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code? @@ -1359,11 +1400,16 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { ss << "Overwriting instance with name #" << current_id; Logger::Message(Logger::LOG_WARNING, ss.str()); } - byid_[current_id] = instance; - MaxId = (std::max)(MaxId, current_id); + idenbyid_[current_id] = instance->identity(); + byidentity_[instance->identity()] = instance; + + // @nb cannot assign to byid_; + // byid_[current_id] = instance; + + max_id = (std::max)(max_id, current_id); } else if (token_stream[0].type == IfcParse::Token_IDENTIFIER && (instance != nullptr)) { - register_inverse(current_id, instance->declaration().as_entity(), token_stream[0], attribute_index); + register_inverse(current_id, instance->declaration().as_entity(), token_stream[0].value_int, attribute_index); } else if (token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == '(') { paren_stack_depth++; } else if (token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == ')') { @@ -1407,21 +1453,21 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { const auto& refattr = p.first.index_; if (auto* v = boost::get(&p.second)) { if (auto* name = boost::get(v)) { - entity_by_id_t::const_iterator it = byid_.find(*name); + auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found"); } else { - byid_[p.first.name_]->data().storage_.set(p.first.index_, it->second); + byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(p.first.index_, it->second); } } else if (auto* inst = boost::get(v)) { - byid_[p.first.name_]->data().storage_.set(p.first.index_, *inst); + byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(p.first.index_, *inst); } } else if (auto* v = boost::get>(&p.second)) { aggregate_of_instance::ptr instances(new aggregate_of_instance); instances->reserve(v->size()); for (const auto& vi : *v) { if (auto* name = boost::get(&vi)) { - entity_by_id_t::const_iterator it = byid_.find(*name); + auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found"); } else { @@ -1431,14 +1477,14 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { instances->push(*inst); } } - byid_[p.first.name_]->data().storage_.set(p.first.index_, instances); + byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(p.first.index_, instances); } else if (auto* v = boost::get>>(&p.second)) { aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance); for (const auto& vi : *v) { std::vector inner; for (const auto& vii : vi) { if (auto* name = boost::get(&vii)) { - entity_by_id_t::const_iterator it = byid_.find(*name); + auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found"); } else { @@ -1450,7 +1496,7 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { } instances->push(inner); } - byid_[p.first.name_]->data().storage_.set(p.first.index_, instances); + byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(p.first.index_, instances); } } @@ -1460,13 +1506,16 @@ void IfcFile::initialize_(IfcParse::IfcSpfStream* s) { } void IfcFile::recalculate_id_counter() { + /* + // @todo entity_by_id_t::key_type k = 0; for (auto& p : byid_) { if (p.first > k) { k = p.first; } } - MaxId = (unsigned int)k; + max_id_ = (unsigned int)k; + */ } class traversal_recorder { @@ -1574,8 +1623,15 @@ void IfcFile::addEntities(aggregate_of_instance::ptr entities) { } IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) { - if (id != -1 && byid_.find((unsigned)id) != byid_.end()) { - throw IfcParse::IfcException("An instance with id " + boost::lexical_cast(id) + " is already part of this file"); + if (id != -1) { + bool id_already_exists = false; + try { + instance_by_id(id); + id_already_exists = true; + } catch (...) {} + if (id_already_exists) { + throw IfcParse::IfcException("An instance with id " + boost::lexical_cast(id) + " is already part of this file"); + } } if (entity->declaration().schema() != schema()) { @@ -1612,7 +1668,8 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) if (entity->file_ == this) { if (entity->declaration().as_entity() == nullptr) { // While not a mapping that can be queried, we do need to free the instance later on - byidentity_[new_entity->identity()] = new_entity; + // @todo. why (over?)write this when adding from the same file? + byidentity_.insert({ new_entity->identity(), new_entity }); } // If it is part of this file @@ -1654,7 +1711,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) throw IfcParse::IfcException("Unable to map instance to file"); } // We directly use storage set not to trigger inverse recalculation which happens at the end - new_entity->data().storage_.set(i, eit->second); + new_entity->data().set_attribute_value(i, eit->second); } else if (attr_type == IfcUtil::Argument_AGGREGATE_OF_ENTITY_INSTANCE) { aggregate_of_instance::ptr instances = attr; aggregate_of_instance::ptr new_instances(new aggregate_of_instance); @@ -1666,7 +1723,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) new_instances->push(eit->second); } - new_entity->data().storage_.set(i, new_instances); + new_entity->data().set_attribute_value(i, new_instances); } else if (attr_type == IfcUtil::Argument_AGGREGATE_OF_AGGREGATE_OF_ENTITY_INSTANCE) { aggregate_of_aggregate_of_instance::ptr instances = attr; aggregate_of_aggregate_of_instance::ptr new_instances(new aggregate_of_aggregate_of_instance); @@ -1682,7 +1739,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) new_instances->push(list); } - new_entity->data().storage_.set(i, new_instances); + new_entity->data().set_attribute_value(i, new_instances); } else if ((decl != nullptr) && decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) { if (boost::math::isnan(conversion_factor)) { std::pair this_file_unit = {nullptr, 1.0}; @@ -1701,13 +1758,13 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) if (attr_type == IfcUtil::Argument_DOUBLE) { double v = attr; v *= conversion_factor; - new_entity->data().storage_.set(i, v); + new_entity->data().set_attribute_value(i, v); } else if (attr_type == IfcUtil::Argument_AGGREGATE_OF_DOUBLE) { std::vector v = attr; for (std::vector::iterator it = v.begin(); it != v.end(); ++it) { (*it) *= conversion_factor; } - new_entity->data().storage_.set(i, v); + new_entity->data().set_attribute_value(i, v); } else if (attr_type == IfcUtil::Argument_AGGREGATE_OF_AGGREGATE_OF_DOUBLE) { std::vector> v = attr; for (std::vector>::iterator it = v.begin(); it != v.end(); ++it) { @@ -1716,7 +1773,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) (*jt) *= conversion_factor; } } - new_entity->data().storage_.set(i, v); + new_entity->data().set_attribute_value(i, v); } } } @@ -1729,8 +1786,8 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) new_entity->as()->set_id(FreshId()); } else { new_entity->as()->set_id((unsigned int)id); - if ((unsigned)id > MaxId) { - MaxId = (unsigned)id; + if ((unsigned)id > max_id_) { + max_id_ = (unsigned)id; } } } @@ -1747,7 +1804,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) ss << "Overwriting entity with guid " << guid; Logger::Message(Logger::LOG_WARNING, ss.str()); } - byguid_[guid] = new_entity; + byguid_.insert({ guid, new_entity }); } catch (const std::exception& ex) { Logger::Message(Logger::LOG_ERROR, ex.what()); } @@ -1757,10 +1814,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) const IfcParse::declaration* ty = &new_entity->declaration(); if (ty->as_entity() != nullptr) { - if (bytype_excl_.find(ty) == bytype_excl_.end()) { - bytype_excl_[ty].reset(new aggregate_of_instance()); - } - bytype_excl_[ty]->push(new_entity); + add_type_ref(new_entity); } if (ty->as_entity() != nullptr) { @@ -1771,8 +1825,8 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) boost::optional id_value; if (id != -1) { id_value = (unsigned)id; - if ((unsigned)id > MaxId) { - MaxId = (unsigned)id; + if ((unsigned)id > max_id_) { + max_id_ = (unsigned)id; } } new_id = new_entity->as()->set_id(id_value); @@ -1787,7 +1841,8 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) Logger::Message(Logger::LOG_WARNING, ss.str()); } // The mapping by entity instance name is updated. - byid_[new_id] = new_entity; + byid_.insert({ new_id, new_entity->identity() }); + byidentity_.insert({ new_entity->identity(), new_entity }); } else if (new_entity->file_ == nullptr) { // For non-entity instances, no mappings are updated, but the file // pointer has to be set, so that actual copies are created in subsequent @@ -1795,7 +1850,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) new_entity->file_ = this; // While not a mapping that can be queried, we do need to free the instance - byidentity_[new_entity->identity()] = new_entity; + byidentity_.insert({ new_entity->identity(), new_entity }); } if ((ty->as_entity() != nullptr)) { @@ -1829,168 +1884,137 @@ void IfcFile::removeEntity(IfcUtil::IfcBaseClass* entity) { throw IfcParse::IfcException("Instance not part of this file"); } - batch_deletion_ids_.push_back(id); + aggregate_of_instance::ptr references = instances_by_reference(id); - if (!batch_mode_) { - process_deletion_(); + // Alter entity instances with INVERSE relations to the entity being + // deleted. This is necessary to maintain a valid IFC file, because + // dangling references to it's entities name should be removed. At this + // moment, inversely related instances affected by the removal of the + // entity being deleted are not deleted themselves. + if (references) { + for (aggregate_of_instance::it iit = references->begin(); iit != references->end(); ++iit) { + IfcUtil::IfcBaseEntity* related_instance = (IfcUtil::IfcBaseEntity*)*iit; + + if (std::find(batch_deletion_ids_.begin(), batch_deletion_ids_.end(), related_instance->id()) != batch_deletion_ids_.end()) { + continue; + } + + for (size_t i = 0; i < related_instance->data().size(); ++i) { + auto attr = related_instance->data().get_attribute_value(i); + if (attr.isNull()) { + continue; + } + + IfcUtil::ArgumentType attr_type = attr.type(); + switch (attr_type) { + case IfcUtil::Argument_ENTITY_INSTANCE: { + IfcUtil::IfcBaseClass* instance_attribute = attr; + if (instance_attribute == entity) { + related_instance->set_attribute_value(i, Blank{}); + } + } break; + case IfcUtil::Argument_AGGREGATE_OF_ENTITY_INSTANCE: { + aggregate_of_instance::ptr instance_list = attr; + if (instance_list->contains(entity)) { + instance_list->remove(entity); + if ((instance_list->size() == 0U) && related_instance->declaration().as_entity()->attribute_by_index(i)->optional()) { + // @todo we can also check the lower bound of the attribute type before setting to null. + related_instance->set_attribute_value(i, Blank{}); + } else { + related_instance->set_attribute_value(i, instance_list); + } + } + } break; + case IfcUtil::Argument_AGGREGATE_OF_AGGREGATE_OF_ENTITY_INSTANCE: { + aggregate_of_aggregate_of_instance::ptr instance_list_list = attr; + if (instance_list_list->contains(entity)) { + aggregate_of_aggregate_of_instance::ptr new_list(new aggregate_of_aggregate_of_instance); + for (aggregate_of_aggregate_of_instance::outer_it it = instance_list_list->begin(); it != instance_list_list->end(); ++it) { + std::vector instances = *it; + std::vector::iterator jt; + while ((jt = std::find(instances.begin(), instances.end(), entity)) != instances.end()) { + instances.erase(jt); + } + new_list->push(instances); + } + related_instance->set_attribute_value(i, new_list); + } + } break; + default: + break; + } + } + } } + + if (entity->declaration().is(*ifcroot_type_) && !entity->data().get_attribute_value(0).isNull()) { + const std::string global_id = entity->data().get_attribute_value(0); + auto it = byguid_.find(global_id); + if (it != byguid_.end()) { + byguid_.erase(it); + } else { + Logger::Warning("GlobalId on rooted instance not encountered in map"); + } + } + + byid_.erase(byid_.find(id)); + + const IfcParse::declaration* ty = &entity->declaration(); + + { + remove_type_ref(entity); + /*auto it = bytype_excl_.find(ty); + if (it != bytype_excl_.end()) { + it->second->remove(entity); + if (it->second->size() == 0) { + bytype_excl_.erase(ty); + } + }*/ + } + + // entity_file_map is in place to prevent duplicate definitions with usage of add(). + // Upon deletion the pairs need to be erased. + for (auto it = entity_file_map_.begin(); it != entity_file_map_.end();) { + if (it->second == entity) { + it = entity_file_map_.erase(it); + } else { + ++it; + } + } + + delete entity; } -void IfcFile::process_deletion_() { +void IfcParse::impl::in_memory_file_storage::process_deletion_inverse(IfcUtil::IfcBaseClass* entity) { + auto id = entity->id(); - for (const auto& id : batch_deletion_ids_.get<0>()) { - auto* entity = instance_by_id(id); + // Delete inverses into entity + byref_excl_.erase( + byref_excl_.lower_bound({ id, -1, -1 }), + byref_excl_.upper_bound({ id, std::numeric_limits::max(), std::numeric_limits::max() })); - aggregate_of_instance::ptr references = instances_by_reference(id); + // This is based on traversal which needs instances to still be contained in the map. + // another option would be to keep byid intact for the remainder of this loop + aggregate_of_instance::ptr entity_attributes = traverse(entity, 1); + for (aggregate_of_instance::it it = entity_attributes->begin(); it != entity_attributes->end(); ++it) { + IfcUtil::IfcBaseClass* entity_attribute = *it; + if (entity_attribute == entity) { + continue; + } + const unsigned int name = entity_attribute->id(); + // Do not update inverses for simple types (which have id()==0 in IfcOpenShell). + if (name != 0) { + // Find instances entity -> other + // and update inverses from entity into other + auto lower = byref_excl_.lower_bound({ name, -1, -1 }); + auto upper = byref_excl_.upper_bound({ name, std::numeric_limits::max(), std::numeric_limits::max() }); - // Alter entity instances with INVERSE relations to the entity being - // deleted. This is necessary to maintain a valid IFC file, because - // dangling references to it's entities name should be removed. At this - // moment, inversely related instances affected by the removal of the - // entity being deleted are not deleted themselves. - if (references) { - for (aggregate_of_instance::it iit = references->begin(); iit != references->end(); ++iit) { - IfcUtil::IfcBaseEntity* related_instance = (IfcUtil::IfcBaseEntity*)*iit; - - if (std::find(batch_deletion_ids_.begin(), batch_deletion_ids_.end(), related_instance->id()) != batch_deletion_ids_.end()) { - continue; - } - - for (size_t i = 0; i < related_instance->data().size(); ++i) { - auto attr = related_instance->data().get_attribute_value(i); - if (attr.isNull()) { - continue; - } - - IfcUtil::ArgumentType attr_type = attr.type(); - switch (attr_type) { - case IfcUtil::Argument_ENTITY_INSTANCE: { - IfcUtil::IfcBaseClass* instance_attribute = attr; - if (instance_attribute == entity) { - related_instance->set_attribute_value(i, Blank{}); - } - } break; - case IfcUtil::Argument_AGGREGATE_OF_ENTITY_INSTANCE: { - aggregate_of_instance::ptr instance_list = attr; - if (instance_list->contains(entity)) { - instance_list->remove(entity); - if ((instance_list->size() == 0U) && related_instance->declaration().as_entity()->attribute_by_index(i)->optional()) { - // @todo we can also check the lower bound of the attribute type before setting to null. - related_instance->set_attribute_value(i, Blank{}); - } else { - related_instance->set_attribute_value(i, instance_list); - } - } - } break; - case IfcUtil::Argument_AGGREGATE_OF_AGGREGATE_OF_ENTITY_INSTANCE: { - aggregate_of_aggregate_of_instance::ptr instance_list_list = attr; - if (instance_list_list->contains(entity)) { - aggregate_of_aggregate_of_instance::ptr new_list(new aggregate_of_aggregate_of_instance); - for (aggregate_of_aggregate_of_instance::outer_it it = instance_list_list->begin(); it != instance_list_list->end(); ++it) { - std::vector instances = *it; - std::vector::iterator jt; - while ((jt = std::find(instances.begin(), instances.end(), entity)) != instances.end()) { - instances.erase(jt); - } - new_list->push(instances); - } - related_instance->set_attribute_value(i, new_list); - } - } break; - default: - break; - } - } + for (auto byref_it = lower; byref_it != upper; ++byref_it) { + auto& ids = byref_it->second; + ids.erase(std::remove(ids.begin(), ids.end(), id), ids.end()); } } - - if (!batch_mode_) { - byref_excl_.erase( - byref_excl_.lower_bound({id, -1, -1}), - byref_excl_.upper_bound({id, std::numeric_limits::max(), std::numeric_limits::max()})); - - // byref_excl_.erase(id); - - // This is based on traversal which needs instances to still be contained in the map. - // another option would be to keep byid intact for the remainder of this loop - aggregate_of_instance::ptr entity_attributes = traverse(entity, 1); - for (aggregate_of_instance::it it = entity_attributes->begin(); it != entity_attributes->end(); ++it) { - IfcUtil::IfcBaseClass* entity_attribute = *it; - if (entity_attribute == entity) { - continue; - } - const unsigned int name = entity_attribute->id(); - // Do not update inverses for simple types (which have id()==0 in IfcOpenShell). - if (name != 0) { - { - auto lower = byref_excl_.lower_bound({name, -1, -1}); - auto upper = byref_excl_.upper_bound({name, std::numeric_limits::max(), std::numeric_limits::max()}); - - for (auto byref_it = lower; byref_it != upper; ++byref_it) { - auto& ids = byref_it->second; - ids.erase(std::remove(ids.begin(), ids.end(), id), ids.end()); - } - } - } - } - } - - if (entity->declaration().is(*ifcroot_type_) && !entity->data().get_attribute_value(0).isNull()) { - const std::string global_id = entity->data().get_attribute_value(0); - auto it = byguid_.find(global_id); - if (it != byguid_.end()) { - byguid_.erase(it); - } else { - Logger::Warning("GlobalId on rooted instance not encountered in map"); - } - } - - byid_.erase(byid_.find(id)); - - const IfcParse::declaration* ty = &entity->declaration(); - - { - auto it = bytype_excl_.find(ty); - if (it != bytype_excl_.end()) { - it->second->remove(entity); - if (it->second->size() == 0) { - bytype_excl_.erase(ty); - } - } - } - - // entity_file_map is in place to prevent duplicate definitions with usage of add(). - // Upon deletion the pairs need to be erased. - for (auto it = entity_file_map_.begin(); it != entity_file_map_.end();) { - if (it->second == entity) { - it = entity_file_map_.erase(it); - } else { - ++it; - } - } - - delete entity; - } - - if (batch_mode_) { - for (auto it = byref_excl_.begin(); it != byref_excl_.end();) { - bool do_delete = batch_deletion_ids_.get<1>().find(std::get(it->first)) != batch_deletion_ids_.get<1>().end(); - if (!do_delete) { - it->second.erase(std::remove_if(it->second.begin(), it->second.end(), [this](int x) { - return batch_deletion_ids_.get<1>().find(x) != batch_deletion_ids_.get<1>().end(); - }), - it->second.end()); - do_delete = it->second.empty(); - } - if (do_delete) { - it = byref_excl_.erase(it); - } else { - ++it; - } - } - } - - batch_deletion_ids_.clear(); + } } namespace { @@ -2015,9 +2039,10 @@ aggregate_of_instance::ptr IfcFile::instances_by_type(const IfcParse::declaratio aggregate_of_instance::ptr insts(new aggregate_of_instance); if (t->as_entity() != nullptr) { visit_subtypes(t->as_entity(), [this, &insts](const IfcParse::entity* ent) { - auto it = bytype_excl_.find(ent); - if (it != bytype_excl_.end()) { - insts->push(it->second); + auto subtype_insts = instances_by_type_excl_subtypes(ent); + // @todo stop returning empty shared_ptrs + if (subtype_insts) { + insts->push(subtype_insts); } }); } @@ -2025,8 +2050,27 @@ aggregate_of_instance::ptr IfcFile::instances_by_type(const IfcParse::declaratio } aggregate_of_instance::ptr IfcFile::instances_by_type_excl_subtypes(const IfcParse::declaration* t) { - entities_by_type_t::const_iterator it = bytype_excl_.find(t); - return (it == bytype_excl_.end()) ? aggregate_of_instance::ptr(new aggregate_of_instance) : it->second; + return std::visit([t](auto& x) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + auto it = x.bytype_excl_.find(t); + return (it == x.bytype_excl_.end()) ? aggregate_of_instance::ptr(new aggregate_of_instance) : it->second; + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + aggregate_of_instance::ptr ret(new aggregate_of_instance); + auto it = x.bytype_.find(t->index_in_schema()); + const auto& s = it->second; + // @todo generalize this, bytype_ should be a map_adapter + std::vector vals(s.size() / sizeof(size_t)); + memcpy(vals.data(), s.data(), s.size()); + for (auto& v : vals) { + ret->push(x.assert_existance(v)); + } + return ret; + } else { + throw std::runtime_error("Storage not initialized"); + aggregate_of_instance::ptr ret(new aggregate_of_instance); + return ret; + } + }, storage_); } aggregate_of_instance::ptr IfcFile::instances_by_type(const std::string& t) { @@ -2038,80 +2082,114 @@ aggregate_of_instance::ptr IfcFile::instances_by_type_excl_subtypes(const std::s } aggregate_of_instance::ptr IfcFile::instances_by_reference(int t) { - auto lower = byref_excl_.lower_bound({ t, -1, -1 }); - auto upper = byref_excl_.upper_bound({ t, std::numeric_limits::max(), std::numeric_limits::max() }); aggregate_of_instance::ptr ret(new aggregate_of_instance); - for (auto it = lower; it != upper; ++it) { - for (auto& i : it->second) { - ret->push(instance_by_id(i)); + std::visit([this, t, &ret](auto& x) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + auto lower = x.byref_excl_.lower_bound({ t, -1, -1 }); + auto upper = x.byref_excl_.upper_bound({ t, std::numeric_limits::max(), std::numeric_limits::max() }); + for (auto it = lower; it != upper; ++it) { + for (auto& i : it->second) { + ret->push(instance_by_id(i)); + } + } + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + // @todo + } else { + throw std::runtime_error("Storage not initialized"); } - } + }, storage_); return ret; } IfcUtil::IfcBaseClass* IfcFile::instance_by_id(int id) { - entity_by_id_t::const_iterator it = byid_.find(id); - if (it == byid_.end()) { - throw IfcException("Instance #" + boost::lexical_cast(id) + " not found"); - } - return it->second; + std::visit([id](auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + throw std::runtime_error("Storage not initialized"); + return (IfcUtil::IfcBaseClass*) nullptr; + } else { + return x.instance_by_id(id); + } + }, storage_); +} + +void IfcParse::IfcFile::add_type_ref(IfcUtil::IfcBaseClass* new_entity) +{ + std::visit([new_entity](auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + throw std::runtime_error("Storage not initialized"); + } else { + return x.add_type_ref(new_entity); + } + }, storage_); +} + + +void IfcParse::IfcFile::remove_type_ref(IfcUtil::IfcBaseClass* new_entity) +{ + std::visit([new_entity](auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + throw std::runtime_error("Storage not initialized"); + } else { + return x.remove_type_ref(new_entity); + } + }, storage_); +} + +void IfcParse::IfcFile::process_deletion_inverse(IfcUtil::IfcBaseClass* inst) +{ + std::visit([inst](auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + throw std::runtime_error("Storage not initialized"); + } else { + return x.process_deletion_inverse(inst); + } + }, storage_); } IfcUtil::IfcBaseClass* IfcFile::instance_by_guid(const std::string& guid) { - entity_by_guid_t::const_iterator it = byguid_.find(guid); + auto it = byguid_.find(guid); if (it == byguid_.end()) { throw IfcException("Instance with GlobalId '" + guid + "' not found"); } return it->second; } -// FIXME: Test destructor to delete entity and arg allocations -IfcFile::~IfcFile() { - std::set entities_to_delete; - for (const auto& pair : byid_) { - entities_to_delete.insert(pair.second); - } - for (const auto& pair : byidentity_) { - entities_to_delete.insert(pair.second); - } - for (auto* entity : entities_to_delete) { - delete entity; - } -} - -IfcFile::entity_by_id_t::const_iterator IfcFile::begin() const { - return byid_.begin(); -} - -IfcFile::entity_by_id_t::const_iterator IfcFile::end() const { - return byid_.end(); -} - IfcFile::type_iterator IfcFile::types_begin() const { - return bytype_excl_.begin(); + return std::visit([](const auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + throw std::runtime_error("Storage not initialized"); + return (IfcFile::type_iterator) impl::rocks_db_file_storage::rocksdb_types_iterator{}; + } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + return (IfcFile::type_iterator) x.bytype_excl_.begin(); + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + return (IfcFile::type_iterator) impl::rocks_db_file_storage::rocksdb_types_iterator(&x); + } + }, storage_); } IfcFile::type_iterator IfcFile::types_end() const { - return bytype_excl_.end(); + return std::visit([](const auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + throw std::runtime_error("Storage not initialized"); + return (IfcFile::type_iterator)impl::rocks_db_file_storage::rocksdb_types_iterator{}; + } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + return (IfcFile::type_iterator)x.bytype_excl_.end(); + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + return (IfcFile::type_iterator)impl::rocks_db_file_storage::rocksdb_types_iterator{}; + } + }, storage_); } -namespace { -struct id_instance_pair_sorter { - bool operator()(const IfcParse::IfcFile::entity_by_id_t::value_type& a, const IfcParse::IfcFile::entity_by_id_t::value_type& b) const { - return a.first < b.first; - } -}; -} // namespace - std::ostream& operator<<(std::ostream& out, const IfcParse::IfcFile& file) { file.header().write(out); - typedef std::vector> vector_t; - vector_t sorted(file.begin(), file.end()); - std::sort(sorted.begin(), sorted.end(), id_instance_pair_sorter()); + typedef std::vector vector_t; + vector_t sorted; + std::transform(file.begin(), file.end(), std::back_inserter(sorted), [file](const auto& x) { return file.byidentity_.find(x.second)->second; }); + std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a->id() < b->id(); }); - for (vector_t::const_iterator it = sorted.begin(); it != sorted.end(); ++it) { - const IfcUtil::IfcBaseClass* e = it->second; + for (auto& e : sorted) { + // @todo this check should no longer be necessary? if (e->declaration().as_entity() != nullptr) { e->toString(out, true); out << ";" << std::endl; @@ -2143,21 +2221,23 @@ std::string IfcFile::createTimestamp() { std::vector IfcFile::get_inverse_indices(int instance_id) { std::vector return_value; - auto lower = byref_excl_.lower_bound({instance_id, -1, -1}); - auto upper = byref_excl_.upper_bound({instance_id, std::numeric_limits::max(), std::numeric_limits::max()}); - // Mapping of instance id to attribute offset. std::map> mapping; - for (auto it = lower; it != upper; ++it) { - for (auto& i : it->second) { - // We only take the tuple for the type that id=i actually is, in order not - // to count double. Because byref contains mappings for every supertype of id=i. - if (instance_by_id(i)->declaration().index_in_schema() == std::get<1>(it->first)) { - mapping[i].push_back(std::get<2>(it->first)); + std::visit([&mapping, instance_id](const auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + auto lower = x.byref_excl_.lower_bound({ instance_id, -1, -1 }); + auto upper = x.byref_excl_.upper_bound({ instance_id, std::numeric_limits::max(), std::numeric_limits::max() }); + for (auto it = lower; it != upper; ++it) { + for (auto& i : it->second) { + mapping[i].push_back(std::get<2>(it->first)); + } } + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + // @todo } - } + }, storage_); auto refs = instances_by_reference(instance_id); @@ -2189,23 +2269,31 @@ aggregate_of_instance::ptr IfcFile::getInverse(int instance_id, const IfcParse:: aggregate_of_instance::ptr return_value(new aggregate_of_instance); visit_subtypes(type->as_entity(), [this, attribute_index, instance_id, &return_value](const IfcParse::declaration* ent) { - if (attribute_index == -1) { - auto lower = byref_excl_.lower_bound({ instance_id, ent->index_in_schema(), -1 }); - auto upper = byref_excl_.upper_bound({ instance_id, ent->index_in_schema(), std::numeric_limits::max() }); - for (auto it = lower; it != upper; ++it) { - for (auto& i : it->second) { - return_value->push(instance_by_id(i)); + std::visit([&return_value, this, attribute_index, instance_id, ent](const auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + if (attribute_index == -1) { + auto lower = x.byref_excl_.lower_bound({ instance_id, ent->index_in_schema(), -1 }); + auto upper = x.byref_excl_.upper_bound({ instance_id, ent->index_in_schema(), std::numeric_limits::max() }); + + for (auto it = lower; it != upper; ++it) { + for (auto& i : it->second) { + return_value->push(instance_by_id(i)); + } + } + } else { + auto it = x.byref_excl_.find({ instance_id, ent->index_in_schema(), attribute_index }); + if (it != x.byref_excl_.end()) { + for (auto& i : it->second) { + return_value->push(instance_by_id(i)); + } + } } + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + // @todo } - } else { - auto it = byref_excl_.find({ instance_id, ent->index_in_schema(), attribute_index }); - if (it != byref_excl_.end()) { - for (auto& i : it->second) { - return_value->push(instance_by_id(i)); - } - } - } + }, storage_); }); return return_value; @@ -2213,11 +2301,20 @@ aggregate_of_instance::ptr IfcFile::getInverse(int instance_id, const IfcParse:: size_t IfcFile::getTotalInverses(int instance_id) { size_t n = 0; - auto lower = byref_excl_.lower_bound({ instance_id, -1, -1 }); - auto upper = byref_excl_.upper_bound({ instance_id, std::numeric_limits::max(), std::numeric_limits::max() }); - for (auto it = lower; it != upper; ++it) { - n += it->second.size(); - } + + std::visit([&n, instance_id](const auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + auto lower = x.byref_excl_.lower_bound({ instance_id, -1, -1 }); + auto upper = x.byref_excl_.upper_bound({ instance_id, std::numeric_limits::max(), std::numeric_limits::max() }); + for (auto it = lower; it != upper; ++it) { + n += it->second.size(); + } + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + // @todo + } + }, storage_); + return n; } @@ -2318,7 +2415,15 @@ void IfcParse::IfcFile::build_inverses_(IfcUtil::IfcBaseClass* inst) { if (attr->declaration().as_entity() != nullptr) { unsigned entity_attribute_id = attr->id(); const auto* decl = inst->declaration().as_entity(); - byref_excl_[{entity_attribute_id, decl->index_in_schema(), idx}].push_back(inst->id()); + + std::visit([entity_attribute_id, decl, idx, inst](auto& x) { + if constexpr (std::is_same_v, std::monostate>) { + } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + x.byref_excl_[{entity_attribute_id, decl->index_in_schema(), idx}].push_back(inst->id()); + } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { + // @todo + } + }, storage_); } }; @@ -2327,16 +2432,20 @@ void IfcParse::IfcFile::build_inverses_(IfcUtil::IfcBaseClass* inst) { void IfcParse::IfcFile::build_inverses() { for (const auto& pair : *this) { - build_inverses_(pair.second); + build_inverses_(byidentity_.find(pair.second)->second); } } std::atomic_uint32_t IfcUtil::IfcBaseClass::counter_(0); -bool IfcParse::IfcFile::guid_map_ = true; +// bool IfcParse::IfcFile::guid_map_ = true; void IfcUtil::IfcBaseClass::unset_attribute_value(size_t index) { - data_.storage_.set(index, Blank{}); + data_.set_attribute_value(index, Blank{}); +} + +AttributeValue IfcUtil::IfcBaseClass::get_attribute_value(size_t index) const { + return data_.get_attribute_value(index); } void IfcUtil::IfcBaseClass::toString(std::ostream& out, bool upper) const @@ -2354,10 +2463,10 @@ void IfcUtil::IfcBaseClass::toString(std::ostream& out, bool upper) const } IfcEntityInstanceData::IfcEntityInstanceData(const IfcEntityInstanceData& data) - : storage_(data.storage_.size() ) + : storage_(data.size()) { - for (size_t i = 0; i < data.storage_.size(); ++i) { - data.storage_.apply_visitor([this, i](const auto& v) { + for (size_t i = 0; i < data.size(); ++i) { + data.apply_visitor([this, i](const auto& v) { using U = std::decay_t; if constexpr (std::is_same_v) { // @todo why did we ever choose shared_ptrs for these @@ -2369,7 +2478,7 @@ IfcEntityInstanceData::IfcEntityInstanceData(const IfcEntityInstanceData& data) v2->push(i); } } - storage_.set(i, v2); + set_attribute_value(i, v2); } else if constexpr (std::is_same_v) { aggregate_of_aggregate_of_instance::ptr v2(new aggregate_of_aggregate_of_instance); if (v) { @@ -2377,9 +2486,9 @@ IfcEntityInstanceData::IfcEntityInstanceData(const IfcEntityInstanceData& data) v2->push(i); } } - storage_.set(i, v2); + set_attribute_value(i, v2); } else { - storage_.set(i, v); + set_attribute_value(i, v); } }, i); } @@ -2387,7 +2496,16 @@ IfcEntityInstanceData::IfcEntityInstanceData(const IfcEntityInstanceData& data) AttributeValue IfcEntityInstanceData::get_attribute_value(size_t index) const { - return { &storage_, (uint8_t) index }; + return std::visit([index](const auto& x) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + return AttributeValue(&x, (uint8_t)index); + } else if constexpr (std::is_same_v, IfcParse::impl::rocks_db_file_storage>) { + // @todo + return AttributeValue{}; + } else { + return AttributeValue{}; + } + }, storage_); } diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index 10b26630ed..7701f52867 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -164,8 +164,6 @@ class IFC_PARSE_API IfcSpfLexer { void TokenString(unsigned int offset, std::string& result); }; -IFC_PARSE_API IfcEntityInstanceData read(unsigned int index, IfcFile* file); - IFC_PARSE_API aggregate_of_instance::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level = -1); IFC_PARSE_API aggregate_of_instance::ptr traverse_breadth_first(IfcUtil::IfcBaseClass* instance, int max_level = -1); diff --git a/src/ifcparse/IfcSpfHeader.cpp b/src/ifcparse/IfcSpfHeader.cpp index 6fb7c0bce0..9fa685fbad 100644 --- a/src/ifcparse/IfcSpfHeader.cpp +++ b/src/ifcparse/IfcSpfHeader.cpp @@ -37,44 +37,69 @@ static const char* const DATA = "DATA"; using namespace IfcParse; namespace { - IfcEntityInstanceData read_from_file(IfcFile* f, size_t s) { - parse_context pc; - f->tokens->Next(); - f->load(-1, nullptr, pc, -1); - return pc.construct(-1, f->references_to_resolve, nullptr, s); + IfcEntityInstanceData read_from_spf_file(IfcFile* f, size_t s) { + return std::visit([f, s](auto& m) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + parse_context pc; + m.tokens->Next(); + m.load(-1, nullptr, pc, -1); + return pc.construct(-1, m.references_to_resolve, nullptr, s); + } else { + // std::unreachable(); + return IfcEntityInstanceData(in_memory_attribute_storage(10)); + } + }, f->storage_); } } HeaderEntity::HeaderEntity(const char* const datatype, size_t size, IfcFile* file) : datatype_(datatype) , file_(file) - , data_(file ? read_from_file(file, size) : IfcEntityInstanceData(storage_t(size))) + , data_(file ? read_from_spf_file(file, size) : IfcEntityInstanceData(in_memory_attribute_storage(size))) {} HeaderEntity::~HeaderEntity() { } void IfcSpfHeader::readSemicolon() { - if (!TokenFunc::isOperator(file_->tokens->Next(), ';')) { - throw IfcException(std::string("Expected ;")); - } + std::visit([](auto& m) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + if (!TokenFunc::isOperator(m.tokens->Next(), ';')) { + throw IfcException(std::string("Expected ;")); + } + } else { + // std::unreachable(); + } + }, file_->storage_); } void IfcSpfHeader::readParen() { - if (!TokenFunc::isOperator(file_->tokens->Next(), '(')) { - throw IfcException(std::string("Expected (")); - } + std::visit([](auto& m) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + if (!TokenFunc::isOperator(m.tokens->Next(), '(')) { + throw IfcException(std::string("Expected (")); + } + } else { + // std::unreachable(); + } + }, file_->storage_); } void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) { - if (TokenFunc::asStringRef(file_->tokens->Next()) != term) { - throw IfcException(std::string("Expected " + term)); - } - if (trail == TRAILING_SEMICOLON) { - readSemicolon(); - } else if (trail == TRAILING_PAREN) { - readParen(); - } + std::visit([this, term, trail](auto& m) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + if (TokenFunc::asStringRef(m.tokens->Next()) != term) { + throw IfcException(std::string("Expected " + term)); + } + if (trail == TRAILING_SEMICOLON) { + readSemicolon(); + } else if (trail == TRAILING_PAREN) { + readParen(); + } + } else { + // std::unreachable(); + } + }, file_->storage_); } void IfcSpfHeader::read() { diff --git a/src/ifcparse/IfcSpfHeader.h b/src/ifcparse/IfcSpfHeader.h index f71dacfa68..f357c11230 100644 --- a/src/ifcparse/IfcSpfHeader.h +++ b/src/ifcparse/IfcSpfHeader.h @@ -39,14 +39,6 @@ class IFC_PARSE_API HeaderEntity { HeaderEntity(const char* const datatype, size_t size, IfcParse::IfcFile* file); virtual ~HeaderEntity(); - void setValue(unsigned int index, const std::string& string) { - data_.storage_.set(index, string); - } - - void setValue(unsigned int index, const std::vector& strings) { - data_.storage_.set(index, strings); - } - public: virtual size_t getArgumentCount() const { return data_.size(); @@ -71,8 +63,8 @@ class IFC_PARSE_API FileDescription : public HeaderEntity { std::vector description() const { return data_.get_attribute_value(0); } std::string implementation_level() const { return data_.get_attribute_value(1); } - void description(const std::vector& value) { setValue(0, value); } - void implementation_level(const std::string& value) { setValue(1, value); } + void description(const std::vector& value) { data_.set_attribute_value(0, value); } + void implementation_level(const std::string& value) { data_.set_attribute_value(1, value); } }; class IFC_PARSE_API FileName : public HeaderEntity { @@ -87,13 +79,13 @@ class IFC_PARSE_API FileName : public HeaderEntity { std::string originating_system() const { return data_.get_attribute_value(5); } std::string authorization() const { return data_.get_attribute_value(6); } - void name(const std::string& value) { setValue(0, value); } - void time_stamp(const std::string& value) { setValue(1, value); } - void author(const std::vector& value) { setValue(2, value); } - void organization(const std::vector& value) { setValue(3, value); } - void preprocessor_version(const std::string& value) { setValue(4, value); } - void originating_system(const std::string& value) { setValue(5, value); } - void authorization(const std::string& value) { setValue(6, value); } + void name(const std::string& value) { data_.set_attribute_value(0, value); } + void time_stamp(const std::string& value) { data_.set_attribute_value(1, value); } + void author(const std::vector& value) { data_.set_attribute_value(2, value); } + void organization(const std::vector& value) { data_.set_attribute_value(3, value); } + void preprocessor_version(const std::string& value) { data_.set_attribute_value(4, value); } + void originating_system(const std::string& value) { data_.set_attribute_value(5, value); } + void authorization(const std::string& value) { data_.set_attribute_value(6, value); } }; class IFC_PARSE_API FileSchema : public HeaderEntity { @@ -102,7 +94,7 @@ class IFC_PARSE_API FileSchema : public HeaderEntity { std::vector schema_identifiers() const { return data_.get_attribute_value(0); } - void schema_identifiers(const std::vector& value) { setValue(0, value); } + void schema_identifiers(const std::vector& value) { data_.set_attribute_value(0, value); } }; class IFC_PARSE_API IfcSpfHeader { diff --git a/src/ifcparse/IfcUtil.cpp b/src/ifcparse/IfcUtil.cpp index 41289752d0..2f65218da2 100644 --- a/src/ifcparse/IfcUtil.cpp +++ b/src/ifcparse/IfcUtil.cpp @@ -209,7 +209,7 @@ IfcUtil::IfcBaseEntity::IfcBaseEntity(IfcEntityInstanceData&& data) void IfcUtil::IfcBaseEntity::populate_derived() { for (auto it = declaration().as_entity()->derived().begin(); it != declaration().as_entity()->derived().end(); ++it) { if (*it) { - this->data().storage_.set( + this->data().set_attribute_value( std::distance(declaration().as_entity()->derived().begin(), it), Derived{} ); diff --git a/src/ifcparse/map_transformer.h b/src/ifcparse/map_transformer.h new file mode 100644 index 0000000000..1aabc3b7cd --- /dev/null +++ b/src/ifcparse/map_transformer.h @@ -0,0 +1,120 @@ +/******************************************************************************** +* * +* This file is part of IfcOpenShell. * +* * +* IfcOpenShell is free software: you can redistribute it and/or modify * +* it under the terms of the Lesser GNU General Public License as published by * +* the Free Software Foundation, either version 3.0 of the License, or * +* (at your option) any later version. * +* * +* IfcOpenShell is distributed in the hope that it will be useful, * +* but WITHOUT ANY WARRANTY; without even the implied warranty of * +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +* Lesser GNU General Public License for more details. * +* * +* You should have received a copy of the Lesser GNU General Public License * +* along with this program. If not, see . * +* * +********************************************************************************/ + +#include +#include +#include +#include + +// map_transformer: wraps a map-like construct so that its iterator returns +// a value_type where the mapped element is transformed via a function. +template +class map_transformer { +public: + using key_type = typename BaseMap::key_type; + using base_mapped_type = typename BaseMap::mapped_type; + using transformed_mapped_type = std::invoke_result_t; + using value_type = std::pair; + +private: + BaseMap* base_map_; + Transform transform_; + + TransformBack transform_back_; + +public: + map_transformer(BaseMap* map, Transform transform, TransformBack transform_back) + : base_map_(map), transform_(transform), transform_back_(transform_back){} + + class iterator { + public: + using base_iterator = typename BaseMap::iterator; + using iterator_category = std::forward_iterator_tag; + using difference_type = typename std::iterator_traits::difference_type; + using key_type = typename BaseMap::key_type; + using transformed_mapped_type = std::invoke_result_t; + using value_type = std::pair; + + private: + base_iterator base_it_; + Transform* transform_ptr_; + + mutable value_type cached_value_; + + public: + iterator() : base_it_(), transform_ptr_(nullptr) {} + iterator(base_iterator base_it, Transform* transform_ptr) + : base_it_(base_it), transform_ptr_(transform_ptr) {} + + // On dereference, return a pair where the key is unchanged and the mapped value + // is the result of applying the transform to the underlying mapped value. + // @todo should these also all be const references so that key/value can be non-copyable (i.e unique_ptr) + value_type operator*() const { + auto base_val = *base_it_; + return { base_val.first, (*transform_ptr_)(base_val.second) }; + } + + // operator-> uses a mutable cache to return a pointer to the current value. + value_type* operator->() const { + cached_value_ = **this; + return &cached_value_; + } + + iterator& operator++() { + ++base_it_; + return *this; + } + + iterator operator++(int) { + iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const iterator& other) const { + return base_it_ == other.base_it_; + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + }; + + iterator begin() { + return iterator(base_map_->begin(), &transform_); + } + + iterator end() { + return iterator(base_map_->end(), &transform_); + } + + iterator find(const key_type& k) { + return iterator(base_map_->find(k), &transform_); + } + + // @todo still not sure if this is a good idea, do we want to insert into the transformed map? + std::pair insert(const value_type& val) { + auto p = base_map_->insert({ val.first, transform_back_(val.second) }); + return { iterator(p.first, &transform_), p.second }; + } + + size_t erase(const key_type& key) { + return base_map_->erase(key); + } +}; \ No newline at end of file diff --git a/src/ifcparse/map_variant.h b/src/ifcparse/map_variant.h new file mode 100644 index 0000000000..b65b2a4391 --- /dev/null +++ b/src/ifcparse/map_variant.h @@ -0,0 +1,174 @@ +/******************************************************************************** +* * +* This file is part of IfcOpenShell. * +* * +* IfcOpenShell is free software: you can redistribute it and/or modify * +* it under the terms of the Lesser GNU General Public License as published by * +* the Free Software Foundation, either version 3.0 of the License, or * +* (at your option) any later version. * +* * +* IfcOpenShell is distributed in the hope that it will be useful, * +* but WITHOUT ANY WARRANTY; without even the implied warranty of * +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +* Lesser GNU General Public License for more details. * +* * +* You should have received a copy of the Lesser GNU General Public License * +* along with this program. If not, see . * +* * +********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +// VariantMap: A map interface that delegates to one of several map types. +// The underlying maps are referenced by pointers (not moved into the variant). +// All map types must share the same key_type, mapped_type, and value_type. +template +class VariantMap { +public: + // The variant holds a pointer to the map + using variant_type = std::variant; + variant_type map_; + + // Deduce common types from the first map type. + // @todo these are not common types, but just the 1st + using key_type = typename std::tuple_element<0, std::tuple>::type::key_type; + using mapped_type = typename std::tuple_element<0, std::tuple>::type::mapped_type; + using value_type = typename std::tuple_element<0, std::tuple>::type::value_type; + + using underlying_iterator_variant = std::variant; + + class iterator { + public: + using value_type = VariantMap::value_type; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; + + underlying_iterator_variant it_var; + + // mutable cache to support operator-> (so that it->second works) + mutable std::unique_ptr cached_value_ptr_; + + iterator() = default; + + explicit iterator(underlying_iterator_variant v) + : it_var(std::move(v)) {} + + iterator(const iterator& other) + : it_var(other.it_var), cached_value_ptr_(nullptr) {} + + iterator& operator=(const iterator& other) { + if (this != &other) { + it_var = other.it_var; + cached_value_ptr_.reset(); // clear the cache + } + return *this; + } + + value_type operator*() const { + return std::visit([](auto& it) -> value_type { return *it; }, it_var); + } + + value_type* operator->() const { + // @todo we need to make a copy here (stored in unique_ptr) because the + // value_type appears to be pair instead of or something + // related... + cached_value_ptr_ = std::make_unique(**this); + return cached_value_ptr_.get(); + } + + iterator& operator++() { + std::visit([](auto& it) { ++it; }, it_var); + return *this; + } + + iterator operator++(int) { + iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const iterator& other) const { + return it_var == other.it_var; + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + }; + + VariantMap() {} + + template + VariantMap(MapT* m) : map_(m) {} + + iterator begin() const{ + return std::visit([](auto m) -> iterator { + if constexpr (std::is_same_v, std::monostate>) { + return iterator{}; + } else { + return iterator(m->begin()); + } + }, map_); + } + + iterator end() const { + return std::visit([](auto m) -> iterator { + if constexpr (std::is_same_v, std::monostate>) { + return iterator{}; + } else { + return iterator(m->end()); + } + }, map_); + } + + iterator find(const key_type& key) const { + return std::visit([&key](auto m) -> iterator { + if constexpr (std::is_same_v, std::monostate>) { + return iterator{}; + } else { + return iterator(m->find(key)); + } + }, map_); + } + + size_t erase(const key_type& key) { + return std::visit([&key](auto m) -> size_t { + if constexpr (std::is_same_v, std::monostate>) { + return size_t(0); + } else { + return m->erase(key); + } + }, map_); + } + + size_t erase(const iterator& it) { + return std::visit([&it](auto m) -> size_t { + if constexpr (std::is_same_v, std::monostate>) { + return size_t(0); + } else { + // @todo erasing by iterator would be more efficient + return m->erase(it->first); + } + }, map_); + } + + std::pair insert(const value_type& val) { + return std::visit([this, &val](auto m) -> std::pair { + // @todo is monostate still necessary here? + if constexpr (!std::is_same_v, std::monostate>) { + auto result = m->insert(val); + return { iterator(result.first), result.second }; + } else { + return { end(), false }; + } + }, map_); + } +}; diff --git a/src/ifcparse/parse_ifcxml.cpp b/src/ifcparse/parse_ifcxml.cpp index 967347352a..9aec98007d 100644 --- a/src/ifcparse/parse_ifcxml.cpp +++ b/src/ifcparse/parse_ifcxml.cpp @@ -279,7 +279,7 @@ static void end_element(void* user, const xmlChar* tag) { } */ // @todo - // back.inst()->data().storage_.set(back.idx(), elems); + // back.inst()->data().set_attribute_value(back.idx(), elems); } if (state->dialect == ifcxml_dialect_ifc2x3 && state->stack.back().ntype() == stack_node::node_instance) { @@ -325,7 +325,7 @@ static void process_characters(void* user, const xmlChar* character, int len) { if (!val.empty()) { // type declaration always at idx 0 visit_any([&state](auto& v) { - state->stack.back().inst()->data().storage_.set(0, v); + state->stack.back().inst()->data().set_attribute_value(0, v); }, val); } } else if (state_type == stack_node::node_header_entry) { @@ -479,7 +479,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) } } - auto untyped = IfcEntityInstanceData(storage_t(decl->as_entity() != nullptr ? decl->as_entity()->attribute_count() : 1)); + auto untyped = IfcEntityInstanceData(in_memory_attribute_storage(decl->as_entity() != nullptr ? decl->as_entity()->attribute_count() : 1)); const IfcParse::entity* entity = decl->as_entity(); if (entity != nullptr) { @@ -494,7 +494,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) auto val = parse_attribute_value(attr->type_of_attribute(), pair.second); if (!val.empty()) { visit_any([&untyped, idx](auto& v) { - untyped.storage_.set(idx, v); + untyped.set_attribute_value(idx, v); }, val); } } else { @@ -531,7 +531,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) IfcUtil::IfcBaseClass* inst; auto inst_ = create_instance(decl); instance_to_attribute(inst_, state->stack.back().idx(), inst); - // state->stack.back().inst()->data().storage_.set(state->stack.back().idx(), attr); + // state->stack.back().inst()->data().set_attribute_value(state->stack.back().idx(), attr); state->stack.push_back(stack_node::instance(id, inst)); } else if (state_type == stack_node::node_aggregate) { @@ -592,7 +592,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) if (inst != nullptr) { int idx = (*found)->entity_reference()->attribute_index( (*found)->attribute_reference()); - inst->data().storage_.set(idx, state->stack.back().inst()); + inst->data().set_attribute_value(idx, state->stack.back().inst()); state->stack.push_back(stack_node::instance(id, inst)); } else { Logger::Error("Unknown attribute " + tagname); @@ -663,12 +663,12 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) int idx = state->stack.back().inv_attr()->entity_reference()->attribute_index( state->stack.back().inv_attr()->attribute_reference()); if (inst != nullptr) { - inst->data().storage_.set(idx, state->stack.back().inst()); + inst->data().set_attribute_value(idx, state->stack.back().inst()); } else { Logger::Error("Internal error, inverse attribute not processed"); } } else if (state_type == stack_node::node_instance_attribute) { - state->stack.back().inst()->data().storage_.set(state->stack.back().idx(), inst); + state->stack.back().inst()->data().set_attribute_value(state->stack.back().idx(), inst); } if (entity == nullptr) { diff --git a/src/ifcparse/rocksdb_map_adapter.h b/src/ifcparse/rocksdb_map_adapter.h new file mode 100644 index 0000000000..44ffbbb592 --- /dev/null +++ b/src/ifcparse/rocksdb_map_adapter.h @@ -0,0 +1,251 @@ +/******************************************************************************** +* * +* This file is part of IfcOpenShell. * +* * +* IfcOpenShell is free software: you can redistribute it and/or modify * +* it under the terms of the Lesser GNU General Public License as published by * +* the Free Software Foundation, either version 3.0 of the License, or * +* (at your option) any later version. * +* * +* IfcOpenShell is distributed in the hope that it will be useful, * +* but WITHOUT ANY WARRANTY; without even the implied warranty of * +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * +* Lesser GNU General Public License for more details. * +* * +* You should have received a copy of the Lesser GNU General Public License * +* along with this program. If not, see . * +* * +********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +// Serialization and deserialization primitives +template +struct DefaultCodec; + +// Specialization for size_t. +template <> +struct DefaultCodec { + std::string encode(const size_t& v) const { + std::string s(sizeof(v), 0); + size_t temp = v; + for (size_t i = 0; i < sizeof(v); i++) { + s[sizeof(v) - i - 1] = static_cast(temp & 0xFF); + temp >>= 8; + } + return s; + } + size_t decode(const std::string& s) const { + size_t v = 0; + for (size_t i = 0; i < s.size(); i++) { + v = (v << 8) | static_cast(s[i]); + } + return v; + } +}; + +// Specialization for std::string (identity) +template <> +struct DefaultCodec { + std::string encode(const std::string& v) const { + return v; + } + std::string decode(const std::string& s) const { + return s; + } +}; + +template +std::string key_to_string(const KeyT& key) { + if constexpr (std::is_same_v) { + return key; + } else { + return std::to_string(key); + } +} + +// Convert from a string to a key. For non-string types, we assume numeric keys. +template +KeyT key_from_string(const std::string& s) { + // @todo tuples + if constexpr (std::is_same_v) { + return s; + } else if constexpr (std::is_integral_v) { + return static_cast(std::stoll(s)); + } else { + static_assert(sizeof(KeyT) == 0, "key_from_string not implemented for this type"); + } +} + +// rocksdb_map_adapter: a std::map-like interface on a RocksDB keyspace with a given prefix. +// The mapped_type is templated and encoded/decoded via Codec. +template > +class rocksdb_map_adapter { +public: + using key_type = KeyT; + using mapped_type = MappedT; + using value_type = std::pair; + +private: + rocksdb::DB* db_; + std::string prefix_; + Codec codec_; + +public: + rocksdb_map_adapter(rocksdb::DB* db, const std::string& prefix) + : db_(db), prefix_(prefix), codec_(Codec{}) {} + + class iterator { + public: + using value_type = std::pair; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + using pointer = value_type*; + using reference = value_type&; + + private: + rocksdb::DB* db_; + std::string prefix_; + Codec codec_; + // When it_ is nullptr, this iterator is at end. + std::unique_ptr it_; + mutable value_type cached_value_; + + void check_valid() { + if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_)) { + it_.reset(); + } + } + + public: + iterator() : db_(nullptr), prefix_(), codec_(Codec{}), it_(nullptr) {} + + iterator(rocksdb::DB* db, const std::string& prefix, + std::unique_ptr iter, Codec codec = Codec{}) + : db_(db), prefix_(prefix), codec_(codec), it_(std::move(iter)) + { + check_valid(); + } + + iterator(const iterator& other) + : db_(other.db_), prefix_(other.prefix_), codec_(other.codec_) + { + if (other.it_) { + std::string curr = other.it_->key().ToString(); + it_.reset(db_->NewIterator(rocksdb::ReadOptions{})); + it_->Seek(curr); + if (!it_->Valid() || it_->key().ToString() != curr) + it_.reset(); + } + } + + iterator& operator=(const iterator& other) { + if (this != &other) { + db_ = other.db_; + prefix_ = other.prefix_; + codec_ = other.codec_; + if (other.it_) { + std::string curr = other.it_->key().ToString(); + it_.reset(db_->NewIterator(rocksdb::ReadOptions{})); + it_->Seek(curr); + if (!it_->Valid() || it_->key().ToString() != curr) + it_.reset(); + } else { + it_.reset(); + } + } + return *this; + } + + value_type operator*() const { + std::string full_key = it_->key().ToString(); + std::string key_without_prefix = full_key.substr(prefix_.size()); + std::string value_str = it_->value().ToString(); + return { key_from_string(key_without_prefix), codec_.decode(value_str) }; + } + + // operator-> uses a mutable cache to return a pointer to the current value. + value_type* operator->() const { + cached_value_ = **this; + return &cached_value_; + } + + iterator& operator++() { + if (it_) { + it_->Next(); + check_valid(); + } + return *this; + } + + iterator operator++(int) { + iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const iterator& other) const { + if (!it_ && !other.it_) return true; + if (it_ && other.it_) + return it_->key().ToString() == other.it_->key().ToString(); + return false; + } + + bool operator!=(const iterator& other) const { + return !(*this == other); + } + }; + + iterator begin() const { + auto iter = std::unique_ptr(db_->NewIterator(rocksdb::ReadOptions{})); + iter->Seek(prefix_); + if (iter->Valid() && iter->key().starts_with(prefix_)) { + return iterator(db_, prefix_, std::move(iter), codec_); + } + return end(); + } + + iterator end() const { + return iterator(); + } + + iterator find(const key_type& key) const { + std::string key_str = key_to_string(key); + std::string full_key = prefix_ + key_str; + auto iter = std::unique_ptr(db_->NewIterator(rocksdb::ReadOptions{})); + iter->Seek(full_key); + if (iter->Valid() && iter->key().ToString() == full_key) + return iterator(db_, prefix_, std::move(iter), codec_); + return end(); + } + + size_t erase(const key_type& key) { + std::string key_str = key_to_string(key); + std::string full_key = prefix_ + key_str; + rocksdb::Status s = db_->Delete(rocksdb::WriteOptions{}, full_key); + return s.ok() ? 1 : 0; + } + + std::pair insert(const value_type& val) { + std::string key_str = key_to_string(val.first); + std::string full_key = prefix_ + key_str; + std::string existing; + rocksdb::Status s = db_->Get(rocksdb::ReadOptions{}, full_key, &existing); + if (s.ok()) { + // Key already exists. + return { find(val.first), false }; + } + std::string encoded = codec_.encode(val.second); + s = db_->Put(rocksdb::WriteOptions{}, full_key, encoded); + if (!s.ok()) { + return { end(), false }; + } + return { find(val.first), true }; + } +}; diff --git a/src/ifcparse/variantarray.h b/src/ifcparse/variantarray.h index 6cd4053624..bd236e6d8f 100644 --- a/src/ifcparse/variantarray.h +++ b/src/ifcparse/variantarray.h @@ -42,18 +42,6 @@ namespace impl { template struct is_unique_ptr> : std::true_type {}; - /* - // Trait to find index of type in parameter pack - template - struct TypeIndex; - template - struct TypeIndex : std::integral_constant {}; - template - struct TypeIndex : std::integral_constant::value> {}; - template - constexpr std::size_t TypeIndex_v = TypeIndex::value; - */ - // Trait to find index of type in parameter pack considering inheritance template struct TypeIndex; @@ -239,7 +227,7 @@ public: return apply_visitor_impl(std::forward(visitor), index, std::integral_constant{}); } - auto size() const { + size_t size() const { return size_and_indices_ ? size_and_indices_[0] : 0; }