diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index 71fb8cab28..a23998bc8b 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -266,6 +266,7 @@ int main(int argc, char** argv) { #ifdef WITH_HDF5 ("cache-file", new po::typed_value(&cache_file), "geometry cache file") #endif + ("stream", "Use streaming conversion (currently supported with conversion to RocksDB)") ; po::options_description ifc_options("IFC options"); @@ -712,16 +713,27 @@ int main(int argc, char** argv) { else if (output_extension == RDB) { int exit_code = EXIT_FAILURE; try { - if (init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) { + if (vmap.count("stream")) { time_t start, end; time(&start); - RocksDbSerializer s(ifc_file, IfcUtil::path::to_utf8(output_filename)); + RocksDbSerializer s(IfcUtil::path::to_utf8(input_filename), IfcUtil::path::to_utf8(output_filename), true); Logger::Status("Populating RocksDB Key-Value store..."); s.finalize(); time(&end); Logger::Status("Done! Conversion took " + format_duration(start, end)); exit_code = EXIT_SUCCESS; - } + } else { + if (init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) { + time_t start, end; + time(&start); + RocksDbSerializer s(ifc_file, IfcUtil::path::to_utf8(output_filename)); + Logger::Status("Populating RocksDB Key-Value store..."); + s.finalize(); + time(&end); + Logger::Status("Done! Conversion took " + format_duration(start, end)); + exit_code = EXIT_SUCCESS; + } + } } catch (const std::exception& e) { Logger::Error(e); } diff --git a/src/ifcopenshell-python/ifcopenshell/__init__.py b/src/ifcopenshell-python/ifcopenshell/__init__.py index ecbd27f81a..810c370035 100644 --- a/src/ifcopenshell-python/ifcopenshell/__init__.py +++ b/src/ifcopenshell-python/ifcopenshell/__init__.py @@ -292,6 +292,13 @@ def guess_format(path: Path) -> Union[str, None]: return None +def stream2(path: Union[Path, str]): + streamer = ifcopenshell_wrapper.InstanceStreamer(str(path)) + while streamer: + if inst := streamer.read_instance_py(): + yield inst + + version_core = ifcopenshell_wrapper.version() __version__ = version = "0.0.0" get_log = ifcopenshell_wrapper.get_log diff --git a/src/ifcparse/IfcEntityInstanceData.cpp b/src/ifcparse/IfcEntityInstanceData.cpp index 82165e51b5..da40d96745 100644 --- a/src/ifcparse/IfcEntityInstanceData.cpp +++ b/src/ifcparse/IfcEntityInstanceData.cpp @@ -74,7 +74,7 @@ namespace { } else { std::string str; if (!array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str).ok()) { - return TypeEncoder::encode_type() - 'A'; + return TypeEncoder::encode_type() - 'A'; } return (size_t) str[0] - 'A'; } diff --git a/src/ifcparse/IfcEntityInstanceData.h b/src/ifcparse/IfcEntityInstanceData.h index 7117212e08..c784261aa6 100644 --- a/src/ifcparse/IfcEntityInstanceData.h +++ b/src/ifcparse/IfcEntityInstanceData.h @@ -414,19 +414,25 @@ class IFC_PARSE_API IfcEntityInstanceData { {} IfcEntityInstanceData(IfcEntityInstanceData&& other) noexcept - : storage_(other.storage_) + : storage_(std::exchange(other.storage_, nullptr)) {} - // No copy-constructor anymore because we need the instance for storage model context + // No copy-constructor/-assignment anymore because we need the instance for storage model context IfcEntityInstanceData(const IfcEntityInstanceData&) = delete; + IfcEntityInstanceData& operator=(const IfcEntityInstanceData&) = delete; - IfcEntityInstanceData& operator=(IfcEntityInstanceData&& other) { + IfcEntityInstanceData& operator=(IfcEntityInstanceData&& other) noexcept { if (this != &other) { - storage_ = other.storage_; + delete storage_; + storage_ = std::exchange(other.storage_, nullptr); } return *this; } + ~IfcEntityInstanceData() { + delete storage_; + } + AttributeValue get_attribute_value(void* storage, const IfcParse::declaration*, std::size_t identity, size_t index) const; template diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index 291217a302..e5755068c4 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -3,7 +3,7 @@ IfcParse::parse_context::~parse_context() { for (auto& t : tokens_) { - boost::apply_visitor([](auto& v) { + std::visit([](auto& v) { if constexpr (std::is_same_v, parse_context*>) { delete v; } @@ -31,14 +31,14 @@ namespace { // Specialization when there are multiple types in the variant template - struct is_type_in_variant, T> + struct is_type_in_variant, T> { - static constexpr bool value = std::is_same::value || is_type_in_variant, T>::value; + static constexpr bool value = std::is_same::value || is_type_in_variant, T>::value; }; // Specialization when there is only one type left in the variant template - struct is_type_in_variant, T> + struct is_type_in_variant, T> { static constexpr bool value = std::is_same::value; }; @@ -107,7 +107,7 @@ namespace { return; } - typedef boost::variant< + typedef std::variant< Blank, std::vector, @@ -125,21 +125,21 @@ namespace { auto append_to_aggregate_storage = [&aggregate_storage](const auto& v) { if constexpr (is_type_in_variant_v>>) { - if (aggregate_storage.which() == 0) { + if (aggregate_storage.index() == 0) { aggregate_storage = std::vector>{ v }; } else { - if (auto* vec_ptr = boost::get>>(&aggregate_storage)) { + if (auto* vec_ptr = std::get_if>>(&aggregate_storage)) { vec_ptr->push_back(v); } else { if constexpr (std::is_same_v, int>) { - auto* vec_ptr2 = boost::get>(&aggregate_storage); + auto* vec_ptr2 = std::get_if>(&aggregate_storage); if (vec_ptr2) { // double[] + int vec_ptr2->push_back((double) v); } } if constexpr (std::is_same_v, double>) { - auto* vec_ptr2 = boost::get>(&aggregate_storage); + auto* vec_ptr2 = std::get_if>(&aggregate_storage); if (vec_ptr2) { // int[] -> double[] + double std::vector ps(vec_ptr2->begin(), vec_ptr2->end()); @@ -149,7 +149,7 @@ namespace { } if constexpr (std::is_same_v, std::vector>) { - auto* vec_ptr2 = boost::get>>(&aggregate_storage); + auto* vec_ptr2 = std::get_if>>(&aggregate_storage); if (vec_ptr2) { // double[][] + int[] std::vector vd(v.begin(), v.end()); @@ -157,7 +157,7 @@ namespace { } } if constexpr (std::is_same_v, std::vector>) { - auto* vec_ptr2 = boost::get>>(&aggregate_storage); + auto* vec_ptr2 = std::get_if>>(&aggregate_storage); if (vec_ptr2) { // int[][] -> double[][] + double[] std::vector> vvd; @@ -171,7 +171,7 @@ namespace { } // @todo would be cool if we can trace this back to file offset - auto current = boost::apply_visitor([](auto v) { + auto current = std::visit([](auto v) { if constexpr (!std::is_same_v) { return std::string(typeid(typename decltype(v)::value_type).name()); } else { @@ -206,7 +206,7 @@ namespace { }; for (auto& t : p.tokens_) { - boost::apply_visitor([&aggregate_storage, &append_to_aggregate_storage, aggr, instance_id, attribute_id](const auto& v) { + std::visit([&aggregate_storage, &append_to_aggregate_storage, aggr, instance_id, attribute_id](const auto& v) { if constexpr (std::is_same_v, IfcParse::Token>) { // @todo get aggregate of enumeration dispatch_token(instance_id, attribute_id, v, aggr && aggr->type_of_element()->as_named_type() ? aggr->type_of_element()->as_named_type()->declared_type() : nullptr, append_to_aggregate_storage); @@ -221,7 +221,7 @@ namespace { }, t); } - boost::apply_visitor(fn, aggregate_storage); + std::visit(fn, aggregate_storage); } } @@ -274,7 +274,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re auto index = (uint8_t) std::distance(tokens_.begin(), it); - boost::apply_visitor([this, &storage, name, &references_to_resolve, index, param_type](const auto& v) { + std::visit([this, &storage, name, &references_to_resolve, index, param_type](const auto& v) { if constexpr (std::is_same_v, IfcParse::Token>) { dispatch_token(name, index, v, param_type && param_type->as_named_type() ? param_type->as_named_type()->declared_type() : nullptr, [this, &storage, name, &references_to_resolve, index](auto v) { if constexpr (std::is_same_v, IfcParse::reference_or_simple_type>) { @@ -552,8 +552,104 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) { if (line.find("MANIFEST-") == 0) { return FT_ROCKSDB; } + + return FT_UNKNOWN; } else { // @todo just return SPF for now, but ideally this will be augmented with all other options return FT_IFCSPF; } } + +std::optional> IfcParse::InstanceStreamer::read_instance() { + std::optional> return_value; + + if (header_ && yielded_header_instances_ < 3) { + if (yielded_header_instances_ == 0) { + return_value.emplace( + 0, + &header_->file_description()->declaration(), + std::move(header_->file_description()->data()) + ); + } else if (yielded_header_instances_ == 1) { + return_value.emplace( + 0, + &header_->file_name()->declaration(), + std::move(header_->file_name()->data()) + ); + } else if (yielded_header_instances_ == 2) { + return_value.emplace( + 0, + &header_->file_schema()->declaration(), + std::move(header_->file_schema()->data()) + ); + } + yielded_header_instances_ += 1; + return return_value; + } + + unsigned current_id = 0; + while (good_ && !lexer_->stream->eof && !current_id) { + if (token_stream_[0].type == IfcParse::Token_IDENTIFIER && + token_stream_[1].type == IfcParse::Token_OPERATOR && + token_stream_[1].value_char == '=' && + token_stream_[2].type == IfcParse::Token_KEYWORD) { + current_id = (unsigned)TokenFunc::asIdentifier(token_stream_[0]); + const IfcParse::declaration* entity_type; + try { + entity_type = schema_->declaration_by_name(TokenFunc::asStringRef(token_stream_[2])); + } catch (const IfcException& ex) { + Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].startPos)); + goto advance; + } + + if (entity_type->as_entity() == nullptr) { + Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].startPos)); + goto advance; + } + + parse_context ps; + lexer_->Next(); + try { + storage_.load(current_id, entity_type->as_entity(), ps, -1); + } catch (const IfcInvalidTokenException& e) { + good_ = file_open_status::INVALID_SYNTAX; + Logger::Error(e); + break; + } + + /// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code? + // Update the status after every 1000 instances parsed + if (((++progress_) % 1000) == 0) { + std::stringstream ss; + ss << "\r#" << current_id; + Logger::Status(ss.str(), false); + } + + auto data = ps.construct(current_id, references_to_resolve_, entity_type, boost::none); + + return_value.emplace( + (size_t)current_id, + entity_type, + std::move(data) + ); + } + advance: + Token next_token; + try { + next_token = lexer_->Next(); + } catch (const IfcException& e) { + Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated"); + } catch (...) { + Logger::Message(Logger::LOG_ERROR, "Parsing terminated"); + } + + if (!lexer_->stream->eof && next_token.type == Token_NONE) { + good_ = file_open_status::INVALID_SYNTAX; + break; + } + + token_stream_.push_back(next_token); + } + + return return_value; +} diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index b9684ac363..bd1c320991 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -24,18 +24,14 @@ #include "IfcParse.h" #include "IfcSchema.h" #include "IfcSpfHeader.h" -#include "rocksdb_map_adapter.h" -#include "rocksdb_set_view.h" -#include "map_variant.h" -#include "map_transformer.h" -#include "set_to_map_transformer.h" +#include "storage.h" +#include "file_open_status.h" #include #include #include #include -#include -#include +#include #include #include @@ -76,520 +72,6 @@ namespace { namespace IfcParse { -class IFC_PARSE_API file_open_status { - public: - enum file_open_enum { - SUCCESS, - READ_ERROR, - NO_HEADER, - UNSUPPORTED_SCHEMA, - INVALID_SYNTAX - }; - - private: - file_open_enum error_; - - public: - file_open_status(file_open_enum error) - : error_(error) {} - - operator file_open_enum() const { - return error_; - } - - file_open_enum value() const { - return error_; - } - - operator bool() const { - return error_ == SUCCESS; - } -}; - -struct InstanceReference { - int v; - size_t file_offset; - operator int() const { - return v; - } -}; - -typedef boost::variant reference_or_simple_type; -typedef std::list, std::vector>>>> unresolved_references; - -struct parse_context { - std::list< - boost::variant< - IfcUtil::IfcBaseClass*, - Token, - parse_context* - >> tokens_; - - parse_context() {}; - ~parse_context(); - - parse_context(const parse_context&) = delete; - parse_context& operator=(const parse_context&) = delete; - - parse_context(parse_context&&) = default; - parse_context& operator=(parse_context&&) = default; - - parse_context& push(); - - void push(Token t); - - void push(IfcUtil::IfcBaseClass* inst); - - IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size); -}; - -#include -#include -#include -#include -#include -#include - -#ifndef SWIG - -template -class variant_iterator { -public: - // The variant type holding one of the underlying iterators. - using variant_type = std::variant; - - // Assuming that all iterator types have the same value_type, difference_type, etc. - using value_type = std::common_type_t::value_type...>; - using difference_type = std::common_type_t::difference_type...>; - using pointer = value_type*; - using reference = value_type&; - // For simplicity, we use input_iterator_tag; if all underlying iterators support more, - // you could compute the common iterator_category. - using iterator_category = std::input_iterator_tag; - - // Default constructor. - variant_iterator() = default; - - // Construct from any one of the underlying iterator types. - template - variant_iterator(Iterator it) : it_(it) { } - - // Dereference operator. - decltype(auto) operator*() const { - return std::visit([](const auto& iter) -> decltype(auto) { - return *iter; - }, it_); - } - - // Arrow operator. - decltype(auto) operator->() const { - return std::visit([](const auto& iter) -> decltype(auto) { - return iter.operator->(); - }, it_); - } - - // Pre-increment operator. - variant_iterator& operator++() { - std::visit([](auto& iter) { ++iter; }, it_); - return *this; - } - - // Post-increment operator. - variant_iterator operator++(int) { - variant_iterator temp(*this); - ++(*this); - return temp; - } - - // Pre-decrement operator. - variant_iterator& operator--() { - std::visit([](auto& iter) { --iter; }, it_); - return *this; - } - - // Post-decrement operator. - variant_iterator operator--(int) { - variant_iterator temp(*this); - --(*this); - return temp; - } - - // Equality comparison. - friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) { - return lhs.it_ == rhs.it_; - } - - // Inequality comparison. - friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) { - return !(lhs == rhs); - } - -private: - variant_type it_; -}; - -#endif - -namespace impl { - struct in_memory_file_storage { - IfcParse::IfcSpfLexer* tokens; - IfcParse::IfcSpfStream* stream; - IfcParse::IfcFile* file; - - unresolved_references references_to_resolve; - - typedef std::map entities_by_type_t; - typedef boost::unordered_map entity_instance_by_name_t; - typedef boost::unordered_map type_instance_by_name_t; - typedef std::map entity_instance_by_guid_t; - typedef std::tuple inverse_attr_record; - enum INVERSE_ATTR { - INSTANCE_ID, - INSTANCE_TYPE, - ATTRIBUTE_INDEX - }; - typedef std::map> entities_by_ref_t; - typedef entity_instance_by_name_t::iterator iterator; - - in_memory_file_storage() : tokens(nullptr), stream(nullptr), file(nullptr) {} - in_memory_file_storage(const in_memory_file_storage&) = delete; - in_memory_file_storage(const in_memory_file_storage&&) = delete; - - class type_iterator : public entities_by_type_t::const_iterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = entities_by_type_t::key_type; - using difference_type = typename entities_by_type_t::const_iterator::difference_type; - using pointer = value_type const*; - using reference = value_type const&; - - type_iterator() : entities_by_type_t::const_iterator() {}; - - type_iterator(const entities_by_type_t::const_iterator& iter) - : entities_by_type_t::const_iterator(iter) {}; - - entities_by_type_t::key_type const* operator->() const { - return &entities_by_type_t::const_iterator::operator->()->first; - } - - entities_by_type_t::key_type const& operator*() const { - return entities_by_type_t::const_iterator::operator*().first; - } - - type_iterator& operator++() { - entities_by_type_t::const_iterator::operator++(); - return *this; - } - - type_iterator operator++(int) { - type_iterator tmp(*this); - operator++(); - return tmp; - } - }; - - - static bool guid_map_; - static bool guid_map() { return guid_map_; } - static void guid_map(bool b) { guid_map_ = b; } - - entity_instance_by_name_t byid_; - type_instance_by_name_t tbyid_; - entities_by_type_t bytype_excl_; - entities_by_ref_t byref_excl_; - entity_instance_by_guid_t byguid_; - - void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); - void try_read_semicolon() const; - - void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); - void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); - - // @todo is this still used - IfcEntityInstanceData read(unsigned int index); - void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id); - - file_open_status good_ = file_open_status::SUCCESS; - - IfcUtil::IfcBaseClass* instance_by_id(int id); - - void add_type_ref(IfcUtil::IfcBaseClass* new_entity) { - auto ty = new_entity->declaration().as_entity(); - if (ty) { - if (bytype_excl_.find(ty) == bytype_excl_.end()) { - bytype_excl_[ty].reset(new aggregate_of_instance()); - } - bytype_excl_[ty]->push(new_entity); - } - } - void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) { - auto ty = new_entity->declaration().as_entity(); - if (ty) { - auto it = bytype_excl_.find(ty); - if (it != bytype_excl_.end()) { - it->second->remove(new_entity); - if (it->second->size() == 0) { - bytype_excl_.erase(ty); - } - } - } - } - - void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); - - template - T* create(); - - IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl); - }; - - class rocks_db_file_storage { - public: - rocksdb::DB* db; - rocksdb::WriteOptions wopts; - rocksdb::ReadOptions ropts; - IfcParse::IfcFile* file; - - enum instance_ref { - typedecl_ref, - entityinstance_ref - }; - - // to make sure that instance pointer are constant during file lifetime - // cache instances because we want stable pointers - // @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side? - typedef std::map entity_by_iden_cache_t; - entity_by_iden_cache_t instance_cache_, type_instance_cache_; - - // @todo all these size_ts should probably be uint32_t for consistency with in-mem storage - - // lookup id->identity - // typedef rocksdb_map_adapter identity_by_id_t; - // identity_by_id_t byid_; - typedef rocksdb_set_view instance_name_view_t; - instance_name_view_t instance_ids_; - typedef set_to_map_transformer> entity_instance_by_name_t; - entity_instance_by_name_t instance_by_name_; - - // typedef map_transformer, std::function, std::function> entity_by_id_t; - // storage is now Instance name -> Identity -> Pointer (cached) - // entity_by_id_t byidentity_; - - // index in schema to binary serialized ids - typedef rocksdb_map_adapter instance_id_str_by_type_t; - instance_id_str_by_type_t bytype_; - - // guid -> id - typedef rocksdb_map_adapter instance_id_by_guid_str_t; - instance_id_by_guid_str_t byguid_internal_; - - // guid -> id -> instance - typedef map_transformer, std::function, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_instance_by_guid_t; - entity_instance_by_guid_t byguid_; - - typedef std::tuple inverse_attr_record; - enum INVERSE_ATTR { - INSTANCE_ID, - INSTANCE_TYPE, - ATTRIBUTE_INDEX - }; - typedef rocksdb_map_adapter> entities_by_ref_t; - entities_by_ref_t byref_excl_; - - // @todo naming - rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file); - ~rocks_db_file_storage(); - - bool read_schema(const IfcParse::schema_definition*& schema); - - IfcUtil::IfcBaseClass* assert_existance(size_t instanceId, instance_ref r); - - // @todo this could be another map_adapter? - /* - class rocksdb_instance_iterator { - private: - rocksdb::Iterator* state_; - rocks_db_file_storage* storage_; - - static constexpr char prefix_[] = "i|"; - - boost::optional read_id_() const { - auto sv = state_->key().ToStringView(); - auto ii = sv.find("|", 2); - if (ii != decltype(sv)::npos) { - char* pEnd; - long result = strtol(sv.data() + 2, &pEnd, 10); - if (*pEnd == '|') { - return (size_t)result; - } - } - return boost::none; - } - public: - rocksdb_instance_iterator() - : state_(nullptr) - , storage_(nullptr) - {} - rocksdb_instance_iterator(rocks_db_file_storage* fs) - : storage_(fs) - { - state_ = fs->db->NewIterator(rocksdb::ReadOptions()); - state_->Seek(prefix_); - if (!state_->Valid() || !state_->key().starts_with(prefix_)) { - delete state_; - state_ = nullptr; - } - } - rocksdb_instance_iterator& operator++() { - if (!state_) { - return *this; - } - auto last_id = read_id_(); - while (state_->Valid()) { - state_->Next(); - // Stop if we've left the prefix range. - if (!state_->Valid() || !state_->key().starts_with(prefix_)) { - delete state_; - state_ = nullptr; - break; - } - if (read_id_() != last_id) { - break; - } - } - return *this; - } - rocksdb_instance_iterator operator++(int) { - rocksdb_instance_iterator temp = *this; - ++(*this); - return temp; - } - bool operator==(const rocksdb_instance_iterator& other) const { - if (state_ == nullptr && other.state_ == nullptr) { - return true; - } else { - return read_id_() == other.read_id_(); - } - } - - bool operator!=(const rocksdb_instance_iterator& other) const { - return !(*this == other); - } - - IfcUtil::IfcBaseClass* operator*() const; - }; - */ - - // @todo merge iterators (template?) - class rocksdb_types_iterator { - private: - rocksdb::Iterator* state_; - const rocks_db_file_storage* storage_; - - static constexpr char prefix_[] = "t|"; - - boost::optional read_id_() const { - auto sv = state_->key().ToStringView(); - auto ii = sv.find("|", 2); - if (ii != decltype(sv)::npos) { - char* pEnd; - long result = strtol(sv.data() + 2, &pEnd, 10); - if (*pEnd == '|') { - return (size_t)result; - } - } - return boost::none; - } - public: - using iterator_category = std::forward_iterator_tag; - using value_type = const IfcParse::declaration*; - // @todo ? - using difference_type = ptrdiff_t; - using pointer = value_type const*; - using reference = value_type const&; - - rocksdb_types_iterator() - : state_(nullptr) - , storage_(nullptr) - {} - - rocksdb_types_iterator(const rocks_db_file_storage* fs) - : storage_(fs) - { - state_ = fs->db->NewIterator(rocksdb::ReadOptions()); - state_->Seek(prefix_); - if (!state_->Valid() || !state_->key().starts_with(prefix_)) { - delete state_; - state_ = nullptr; - } - } - - rocksdb_types_iterator& operator++() { - if (!state_) { - return *this; - } - auto last_id = read_id_(); - while (state_->Valid()) { - state_->Next(); - // Stop if we've left the prefix range. - if (!state_->Valid() || !state_->key().starts_with(prefix_)) { - delete state_; - state_ = nullptr; - break; - } - if (read_id_() != last_id) { - break; - } - } - return *this; - } - - rocksdb_types_iterator operator++(int) { - rocksdb_types_iterator temp = *this; - ++(*this); - return temp; - } - - bool operator==(const rocksdb_types_iterator& other) const { - if (state_ == nullptr && other.state_ == nullptr) { - return true; - } else { - return read_id_() == other.read_id_(); - } - } - - bool operator!=(const rocksdb_types_iterator& other) const { - return !(*this == other); - } - - value_type const& operator*() const; - - value_type const* operator->() const { - return &operator*(); - } - }; - - // @todo rocksdb_instance_iterator? - using const_iterator = entity_instance_by_name_t::iterator; - - void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); - void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); - - // @todo a bit hard as a map because of value_type being an aggregate - void add_type_ref(IfcUtil::IfcBaseClass* new_entity); - void remove_type_ref(IfcUtil::IfcBaseClass* new_entity); - - IfcUtil::IfcBaseClass* instance_by_id(int id); - - void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); - - template - T* create(); - - IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl); - }; -} - enum filetype { FT_IFCSPF, FT_IFCXML, @@ -601,6 +83,95 @@ enum filetype { filetype guess_file_type(const std::string& fn); + +class InstanceStreamer { +private: + IfcSpfStream* stream_; + IfcSpfLexer* lexer_; + IfcSpfHeader* header_; + boost::circular_buffer token_stream_; + const IfcParse::schema_definition* schema_; + const IfcParse::declaration* ifcroot_type_; + IfcParse::impl::in_memory_file_storage storage_; + IfcParse::file_open_status good_ = IfcParse::file_open_status::SUCCESS; + int progress_; + IfcParse::unresolved_references references_to_resolve_; + int yielded_header_instances_ = 0; + +public: + operator bool() const { + return good_ && !lexer_->stream->eof; + } + + IfcParse::file_open_status status() const { + return good_; + } + + const IfcParse::unresolved_references& references() const { + return references_to_resolve_; + } + + IfcParse::unresolved_references& references() { + return references_to_resolve_; + } + + const IfcParse::impl::in_memory_file_storage::entities_by_ref_t& inverses() const { + return storage_.byref_excl_; + } + + IfcParse::impl::in_memory_file_storage::entities_by_ref_t& inverses() { + return storage_.byref_excl_; + } + + InstanceStreamer(const std::string& fn) + : stream_(new IfcSpfStream(fn)) + , lexer_(new IfcSpfLexer(stream_)) + , token_stream_(3, Token{}) + , schema_(nullptr) + , ifcroot_type_(nullptr) + , progress_(0) + { + good_ = file_open_status::NO_HEADER; + header_ = new IfcParse::IfcSpfHeader(lexer_); + if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) { + try { + schema_ = IfcParse::schema_by_name(header_->file_schema()->schema_identifiers().front()); + good_ = file_open_status::SUCCESS; + } catch (const IfcParse::IfcException&) { + } + } + storage_.file = nullptr; + storage_.schema = schema_; + storage_.tokens = lexer_; + storage_.references_to_resolve = &references_to_resolve_; + } + + InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer) + : stream_(nullptr) + , lexer_(lexer) + , header_(nullptr) + , token_stream_(3, Token{}) + , schema_(schema) + , ifcroot_type_(schema->declaration_by_name("IfcRoot")) + , progress_(0) + { + storage_.file = nullptr; + storage_.schema = schema_; + storage_.tokens = lexer_; + storage_.references_to_resolve = &references_to_resolve_; + } + + ~InstanceStreamer() { + delete stream_; + if (stream_) { + delete lexer_; + } + delete header_; + } + + std::optional> read_instance(); +}; + /// This class provides access to the entity instances in an IFC file /// The file takes ownership of instances added to this file and deletes them when the file is deleted. class IFC_PARSE_API IfcFile { @@ -661,7 +232,7 @@ private: ~IfcFile(); - file_open_status good() const { return good_; } + IfcParse::file_open_status good() const { return good_; } /// Returns the first entity in the range of instances contained in the model, /// in arbitrary order diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index a0ef386fc5..3d46e185fe 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -31,7 +31,6 @@ #include #include -#include #include #include #include @@ -263,8 +262,7 @@ void IfcSpfStream::Inc() { } } -IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_, IfcParse::IfcFile* file_) { - file = file_; +IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_) { stream = stream_; decoder_ = new IfcCharacterDecoder(stream_); } @@ -318,14 +316,14 @@ unsigned int IfcSpfLexer::skipComment() const { Token IfcSpfLexer::Next() { if (stream->eof) { - return NoneTokenPtr(); + return Token{}; } while ((skipWhitespace() != 0U) || (skipComment() != 0U)) { } if (stream->eof) { - return NoneTokenPtr(); + return Token{}; } unsigned int pos = stream->Tell(); @@ -369,7 +367,7 @@ Token IfcSpfLexer::Next() { if (len != 0) { t = GeneralTokenPtr(this, pos, stream->Tell()); } else { - t = NoneTokenPtr(); + t = Token{}; } // std::wcout << "token: " << pos << " " << TokenFunc::asStringRef(t).c_str() << std::endl; return t; @@ -525,7 +523,6 @@ Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end return token; } -Token IfcParse::NoneTokenPtr() { return Token(); } bool TokenFunc::isOperator(const Token& token) { return token.type == Token_OPERATOR; @@ -717,11 +714,11 @@ void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name, if (TokenFunc::isKeyword(next)) { try { - const auto* decl = file->schema()->declaration_by_name(TokenFunc::asStringRef(next)); + const auto* decl = (schema ? schema : file->schema())->declaration_by_name(TokenFunc::asStringRef(next)); parse_context ps; tokens->Next(); load(0, nullptr, ps, -1); - auto* simple_type_instance = file->schema()->instantiate(decl, ps.construct(-1, references_to_resolve, decl, boost::none)); + auto* simple_type_instance = (schema ? schema : file->schema())->instantiate(decl, ps.construct(-1, *references_to_resolve, decl, boost::none)); //@todo decide addEntity(((IfcUtil::IfcBaseClass*)*entity)); context.push(simple_type_instance); simple_type_instance->file_ = file; @@ -750,7 +747,7 @@ IfcEntityInstanceData IfcParse::impl::in_memory_file_storage::read(unsigned int parse_context pc; tokens->Next(); load(i, ty->as_entity(), pc, -1); - return IfcEntityInstanceData(pc.construct(i, references_to_resolve, ty, boost::none)); + return IfcEntityInstanceData(pc.construct(i, *references_to_resolve, ty, boost::none)); } void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const { @@ -814,7 +811,12 @@ void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from, if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) { std::vector vals(s.size() / sizeof(uint32_t)); memcpy(vals.data(), s.data(), s.size()); - vals.erase(std::find(vals.begin(), vals.end(), (uint32_t)id_from)); + auto it = std::find(vals.begin(), vals.end(), (uint32_t)id_from); + if (it != vals.end()) { + vals.erase(it); + } else { + Logger::Error("Unregistering non-existant inverse #" + std::to_string(id_from) + " on instance #" + std::to_string(inst_id) + " at attribute " + std::to_string(attribute_index)); + } s.resize(vals.size() * sizeof(uint32_t)); memcpy(s.data(), vals.data(), s.size()); db->Put(wopts, key, s); @@ -1348,7 +1350,9 @@ IfcFile::IfcFile(const std::string& path, filetype ty) { } else { throw std::runtime_error("Unsupported file format"); } - ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); + if (schema_) { + ifcroot_type_ = schema_->declaration_by_name("IfcRoot"); + } } #endif @@ -1405,16 +1409,15 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt // number parsing. See comment above on line 41. init_locale(); - tokens = 0; - stream = s; - if (!stream->valid) { + tokens = nullptr; + + if (!s->valid) { // @todo set good on parent file good_ = file_open_status::READ_ERROR; return; } - // @todo file ptr arg removed? - tokens = new IfcSpfLexer(stream, nullptr); + tokens = new IfcSpfLexer(s); std::vector schemas; @@ -1447,182 +1450,117 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt auto ifcroot_type_ = schema->declaration_by_name("IfcRoot"); - boost::circular_buffer token_stream(3, Token()); + InstanceStreamer streamer(schema, tokens); - IfcUtil::IfcBaseClass* instance = nullptr; - - unsigned current_id = 0; - int progress = 0; Logger::Status("Scanning file..."); - int paren_stack_depth = 0; - int attribute_index = -1; + while (streamer) { - while (!stream->eof) { - if (token_stream[0].type == IfcParse::Token_IDENTIFIER && - token_stream[1].type == IfcParse::Token_OPERATOR && - token_stream[1].value_char == '=' && - token_stream[2].type == IfcParse::Token_KEYWORD) { - attribute_index = 0; + auto inst = streamer.read_instance(); - current_id = (unsigned)TokenFunc::asIdentifier(token_stream[0]); - const IfcParse::declaration* entity_type; - try { - entity_type = schema->declaration_by_name(TokenFunc::asStringRef(token_stream[2])); - } catch (const IfcException& ex) { - Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream[2].startPos)); - goto advance; - } - - if (entity_type->as_entity() == nullptr) { - Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream[2].startPos)); - goto advance; - } - - parse_context ps; - tokens->Next(); - try { - load(current_id, entity_type->as_entity(), ps, -1); - } catch (const IfcInvalidTokenException& e) { - good_ = file_open_status::INVALID_SYNTAX; - Logger::Error(e); - break; - } - instance = schema->instantiate(entity_type, ps.construct(current_id, references_to_resolve, entity_type, boost::none)); - instance->file_ = file; - instance->id_ = current_id; - - /// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code? - // Update the status after every 1000 instances parsed - if (((++progress) % 1000) == 0) { - std::stringstream ss; - ss << "\r#" << current_id; - Logger::Status(ss.str(), false); - } - - if (instance->declaration().is(*ifcroot_type_)) { - try { - // @nb here we know we're using in-memory so 'nullptr, nullptr, 0' is safe - const std::string guid = instance->data().get_attribute_value(nullptr, nullptr, 0, 0); - if (byguid_.find(guid) != byguid_.end()) { - std::stringstream ss; - ss << "Instance encountered with non-unique GlobalId " << guid; - Logger::Message(Logger::LOG_WARNING, ss.str()); - } - byguid_[guid] = instance; - } catch (const IfcException& ex) { - Logger::Message(Logger::LOG_ERROR, ex.what()); - } - // this has consumed the instance tokens, set stack depth to 0 - paren_stack_depth = 0; - attribute_index = -1; - } - - const IfcParse::declaration* ty = &instance->declaration(); - - { - if (bytype_excl_.find(ty) == bytype_excl_.end()) { - bytype_excl_[ty].reset(new aggregate_of_instance()); - } - bytype_excl_[ty]->push(instance); - } - - if (byid_.find(current_id) != byid_.end()) { - std::stringstream ss; - ss << "Overwriting instance with name #" << current_id; - Logger::Message(Logger::LOG_WARNING, ss.str()); - } - - // byidentity_[instance->identity()] = instance; - byid_.insert({ current_id, instance }); - - // @nb cannot assign to byid_; - // byid_[current_id] = instance; - - max_id = (std::max)(max_id, current_id); - } else if (token_stream[0].type == IfcParse::Token_IDENTIFIER && (instance != nullptr)) { - register_inverse(current_id, instance->declaration().as_entity(), token_stream[0].value_int, attribute_index); - } else if (token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == '(') { - paren_stack_depth++; - } else if (token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == ')') { - paren_stack_depth--; - if (paren_stack_depth == 0) { - attribute_index = -1; - } - } else if (paren_stack_depth == 1 && token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == ',') { - attribute_index++; - } - - advance: - Token next_token; - try { - next_token = tokens->Next(); - } catch (const IfcException& e) { - Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated"); - } catch (...) { - Logger::Message(Logger::LOG_ERROR, "Parsing terminated"); - } - - if (!stream->eof && next_token.type == Token_NONE) { - good_ = file_open_status::INVALID_SYNTAX; + if (!inst) { + // No more instances to read break; + } + auto current_id = std::get<0>(*inst); + + auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst))); + instance->file_ = file; + instance->id_ = current_id; + + if (instance->declaration().is(*ifcroot_type_)) { + try { + // @nb here we know we're using in-memory so 'nullptr, nullptr, 0' is safe + const std::string guid = instance->data().get_attribute_value(nullptr, nullptr, 0, 0); + if (byguid_.find(guid) != byguid_.end()) { + std::stringstream ss; + ss << "Instance encountered with non-unique GlobalId " << guid; + Logger::Message(Logger::LOG_WARNING, ss.str()); + } + byguid_[guid] = instance; + } catch (const IfcException& ex) { + Logger::Message(Logger::LOG_ERROR, ex.what()); + } } - token_stream.push_back(next_token); + const IfcParse::declaration* ty = &instance->declaration(); + + { + if (bytype_excl_.find(ty) == bytype_excl_.end()) { + bytype_excl_[ty].reset(new aggregate_of_instance()); + } + bytype_excl_[ty]->push(instance); + } + + if (byid_.find(current_id) != byid_.end()) { + std::stringstream ss; + ss << "Overwriting instance with name #" << current_id; + Logger::Message(Logger::LOG_WARNING, ss.str()); + } + + // byidentity_[instance->identity()] = instance; + byid_.insert({ current_id, instance }); + + // @nb cannot assign to byid_; + // byid_[current_id] = instance; + + max_id = (std::max)(max_id, (unsigned int) current_id); } + good_ = streamer.status(); + byref_excl_ = streamer.inverses(); + Logger::Status("\rDone scanning file "); delete tokens; if (good_ != file_open_status::SUCCESS) { - references_to_resolve.clear(); return; } - for (const auto& p : references_to_resolve) { + for (const auto& p : streamer.references()) { const auto& ref = p.first.name_; const auto& refattr = p.first.index_; - if (auto* v = boost::get(&p.second)) { - if (auto* name = boost::get(v)) { + if (auto* v = std::get_if(&p.second)) { + if (auto* name = std::get_if(v)) { auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset)); } else { byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, it->second); } - } else if (auto* inst = boost::get(v)) { + } else if (auto* inst = std::get_if(v)) { byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst); } - } else if (auto* v = boost::get>(&p.second)) { + } else if (auto* v = std::get_if>(&p.second)) { aggregate_of_instance::ptr instances(new aggregate_of_instance); instances->reserve(v->size()); for (const auto& vi : *v) { - if (auto* name = boost::get(&vi)) { + if (auto* name = std::get_if(&vi)) { auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset)); } else { instances->push(it->second); } - } else if (auto* inst = boost::get(&vi)) { + } else if (auto* inst = std::get_if(&vi)) { instances->push(*inst); } } byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances); - } else if (auto* v = boost::get>>(&p.second)) { + } else if (auto* v = std::get_if>>(&p.second)) { aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance); for (const auto& vi : *v) { std::vector inner; for (const auto& vii : vi) { - if (auto* name = boost::get(&vii)) { + if (auto* name = std::get_if(&vii)) { auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset)); } else { inner.push_back(it->second); } - } else if (auto* inst = boost::get(&vii)) { + } else if (auto* inst = std::get_if(&vii)) { inner.push_back(*inst); } } @@ -1633,8 +1571,6 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt } Logger::Status("Done resolving references"); - - references_to_resolve.clear(); } void IfcFile::recalculate_id_counter() { diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index 7701f52867..229ea1b9c1 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -34,6 +34,7 @@ #include "IfcCharacterDecoder.h" #include "IfcSpfStream.h" #include "macros.h" +#include "storage.h" #include #include @@ -50,41 +51,6 @@ namespace IfcParse { -class IfcFile; -class IfcSpfLexer; - -enum TokenType { - Token_NONE, - Token_STRING, - Token_IDENTIFIER, - Token_OPERATOR, - Token_ENUMERATION, - Token_KEYWORD, - Token_INT, - Token_BOOL, - Token_FLOAT, - Token_BINARY -}; - -struct Token { - IfcSpfLexer* lexer; //TODO: remove it from here - unsigned startPos; - TokenType type; - union { - char value_char; //types: OPERATOR - int value_int; //types: INT, IDENTIFIER - double value_double; //types: FLOAT - }; - - Token() : lexer(0), - startPos(0), - type(Token_NONE) {} - Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type) - : lexer(_lexer), - startPos(_startPos), - type(_type) {} -}; - /// Provides functions to convert Tokens to binary data /// Tokens are merely offsets to where they can be read in the file class IFC_PARSE_API TokenFunc { @@ -142,7 +108,6 @@ class IFC_PARSE_API TokenFunc { // Token OperatorTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end); Token GeneralTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end); -Token NoneTokenPtr(); /// A stream of tokens to be read from a IfcSpfStream. class IFC_PARSE_API IfcSpfLexer { @@ -157,8 +122,8 @@ class IFC_PARSE_API IfcSpfLexer { return string; } IfcSpfStream* stream; - IfcFile* file; - IfcSpfLexer(IfcSpfStream* stream, IfcFile* file); + // IfcFile* file; + IfcSpfLexer(IfcSpfStream* stream); Token Next(); ~IfcSpfLexer(); void TokenString(unsigned int offset, std::string& result); diff --git a/src/ifcparse/IfcSpfHeader.cpp b/src/ifcparse/IfcSpfHeader.cpp index 71661cb64a..cf9c1f9b06 100644 --- a/src/ifcparse/IfcSpfHeader.cpp +++ b/src/ifcparse/IfcSpfHeader.cpp @@ -30,46 +30,40 @@ static const char* const DATA = "DATA"; using namespace IfcParse; namespace { - IfcEntityInstanceData read_from_spf_file(IfcFile* f, size_t s) { - return std::visit([f, s](auto& m) { - if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { - parse_context pc; - m.tokens->Next(); - m.load(-1, nullptr, pc, -1); - return pc.construct(-1, m.references_to_resolve, nullptr, s); - } else { - // std::unreachable(); - return IfcEntityInstanceData(in_memory_attribute_storage(10)); - } - }, f->storage_); + IfcEntityInstanceData read_from_spf_file(IfcParse::impl::in_memory_file_storage* storage, size_t s) { + if (storage != nullptr) { + parse_context pc; + storage->tokens->Next(); + storage->load(-1, nullptr, pc, -1); + return pc.construct(-1, *storage->references_to_resolve, nullptr, s); + } else { + // std::unreachable(); + return IfcEntityInstanceData(in_memory_attribute_storage(10)); + } + } +} // namespace + +void IfcSpfHeader::readSemicolon() { + if (storage_ != nullptr) { + if (!TokenFunc::isOperator(storage_->tokens->Next(), ';')) { + throw IfcException(std::string("Expected ;")); + } + } else { + // std::unreachable(); } } -void IfcSpfHeader::readSemicolon() { - std::visit([](auto& m) { - if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { - if (!TokenFunc::isOperator(m.tokens->Next(), ';')) { - throw IfcException(std::string("Expected ;")); - } - } else { - // std::unreachable(); - } - }, file_->storage_); -} - void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) { - std::visit([this, term, trail](auto& m) { - if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { - if (TokenFunc::asStringRef(m.tokens->Next()) != term) { - throw IfcException(std::string("Expected " + term)); - } - if (trail == TRAILING_SEMICOLON) { - readSemicolon(); - } - } else { - // std::unreachable(); + if (storage_ != nullptr) { + if (TokenFunc::asStringRef(storage_->tokens->Next()) != term) { + throw IfcException(std::string("Expected " + term)); } - }, file_->storage_); + if (trail == TRAILING_SEMICOLON) { + readSemicolon(); + } + } else { + // std::unreachable(); + } } IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file) @@ -86,15 +80,50 @@ IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file) file_name_->file_ = file_; file_schema_ = new Header_section_schema::file_schema({}); file_schema_->file_ = file_; + } else { + storage_ = std::visit([this](auto& m) -> decltype(storage_) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + return &m; + } + return nullptr; + }, file_->storage_); } } +IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcSpfLexer* lexer) +{ + storage_ = new impl::in_memory_file_storage; + storage_->tokens = lexer; + file_ = nullptr; + + // overwritten later in IfcFile::setDefaultHeaderValues() when we know the schema identifier + file_description_ = new Header_section_schema::file_description({}, ""); + file_description_->file_ = file_; + file_name_ = new Header_section_schema::file_name("", "", {}, {}, "", "", ""); + file_name_->file_ = file_; + file_schema_ = new Header_section_schema::file_schema({}); + file_schema_->file_ = file_; +} + IfcParse::IfcSpfHeader::~IfcSpfHeader() { delete file_schema_; delete file_name_; delete file_description_; } +void IfcParse::IfcSpfHeader::file(IfcParse::IfcFile* file) +{ + this->file_ = file; + if (file != nullptr) { + storage_ = std::visit([this](auto& m) -> decltype(storage_) { + if constexpr (std::is_same_v, impl::in_memory_file_storage>) { + return &m; + } + return nullptr; + }, file_->storage_); + } +} + void IfcSpfHeader::read() { readTerminal(ISO_10303_21, TRAILING_SEMICOLON); readTerminal(HEADER, TRAILING_SEMICOLON); @@ -111,19 +140,19 @@ void IfcSpfHeader::read() { readTerminal(Header_section_schema::file_description::Class().name_uc(), NONE); delete file_description_; - file_description_ = new Header_section_schema::file_description(read_from_spf_file(file_, Header_section_schema::file_description::Class().attribute_count())); + file_description_ = new Header_section_schema::file_description(read_from_spf_file(storage_, Header_section_schema::file_description::Class().attribute_count())); file_description_->file_ = file_; readSemicolon(); readTerminal(Header_section_schema::file_name::Class().name_uc(), NONE); delete file_name_; - file_name_ = new Header_section_schema::file_name(read_from_spf_file(file_, Header_section_schema::file_name::Class().attribute_count())); + file_name_ = new Header_section_schema::file_name(read_from_spf_file(storage_, Header_section_schema::file_name::Class().attribute_count())); file_name_->file_ = file_; readSemicolon(); readTerminal(Header_section_schema::file_schema::Class().name_uc(), NONE); delete file_schema_; - file_schema_ = new Header_section_schema::file_schema(read_from_spf_file(file_, Header_section_schema::file_schema::Class().attribute_count())); + file_schema_ = new Header_section_schema::file_schema(read_from_spf_file(storage_, Header_section_schema::file_schema::Class().attribute_count())); file_schema_->file_ = file_; readSemicolon(); } diff --git a/src/ifcparse/IfcSpfHeader.h b/src/ifcparse/IfcSpfHeader.h index c036bb84fb..6d47617316 100644 --- a/src/ifcparse/IfcSpfHeader.h +++ b/src/ifcparse/IfcSpfHeader.h @@ -23,6 +23,7 @@ #include "ifc_parse_api.h" #include "IfcEntityInstanceData.h" #include "Header_section_schema.h" +#include "storage.h" namespace IfcParse { @@ -31,6 +32,8 @@ class IfcFile; class IFC_PARSE_API IfcSpfHeader { private: IfcFile* file_; + IfcParse::impl::in_memory_file_storage* storage_ = nullptr; + mutable Header_section_schema::file_description* file_description_; mutable Header_section_schema::file_name* file_name_; mutable Header_section_schema::file_schema* file_schema_; @@ -43,11 +46,12 @@ class IFC_PARSE_API IfcSpfHeader { public: explicit IfcSpfHeader(IfcParse::IfcFile* file = nullptr); + explicit IfcSpfHeader(IfcParse::IfcSpfLexer* lexer); ~IfcSpfHeader(); IfcParse::IfcFile* file() { return file_; } - void file(IfcParse::IfcFile* file) { file_ = file; } + void file(IfcParse::IfcFile* file); void read(); bool tryRead(); diff --git a/src/ifcparse/file_open_status.h b/src/ifcparse/file_open_status.h new file mode 100644 index 0000000000..6125698567 --- /dev/null +++ b/src/ifcparse/file_open_status.h @@ -0,0 +1,42 @@ +#ifndef FILE_OPEN_STATUS_H +#define FILE_OPEN_STATUS_H + +#include "ifc_parse_api.h" + +namespace IfcParse { + + class IFC_PARSE_API file_open_status { + public: + enum file_open_enum { + SUCCESS, + READ_ERROR, + NO_HEADER, + UNSUPPORTED_SCHEMA, + INVALID_SYNTAX, + UNKNOWN + }; + + private: + file_open_enum error_; + + public: + file_open_status(file_open_enum error = UNKNOWN) + : error_(error) { + } + + operator file_open_enum() const { + return error_; + } + + file_open_enum value() const { + return error_; + } + + operator bool() const { + return error_ == SUCCESS; + } + }; + +} + +#endif // !FILE_OPEN_STATUS_H diff --git a/src/ifcparse/parse_ifcxml.cpp b/src/ifcparse/parse_ifcxml.cpp index 4887654b9f..56151626b4 100644 --- a/src/ifcparse/parse_ifcxml.cpp +++ b/src/ifcparse/parse_ifcxml.cpp @@ -446,14 +446,14 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) // Create an attribute value from an instance. Potentially NULL in case it is a // forward reference to an instance not yet encountered. - auto instance_to_attribute = [&state](const boost::variant& inst_or_ref, size_t attribute_index, IfcUtil::IfcBaseClass*& inst) { - if (inst_or_ref.which() == 0) { + auto instance_to_attribute = [&state](const std::variant& inst_or_ref, size_t attribute_index, IfcUtil::IfcBaseClass*& inst) { + if (inst_or_ref.index() == 0) { inst = nullptr; // This attribute is NULL initially and after parsing the complete // file populated in a subsequent step. - state->forward_references.push_back(std::make_tuple(inst->as(), attribute_index, boost::get(inst_or_ref))); + state->forward_references.push_back(std::make_tuple(inst->as(), attribute_index, std::get(inst_or_ref))); } else { - inst = boost::get(inst_or_ref); + inst = std::get(inst_or_ref); inst->set_attribute_value(attribute_index, inst); } }; @@ -461,7 +461,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) // Create or reference an instance from the file and set attributes based on XML attributes. auto create_instance = [&state, &attributes](const IfcParse::declaration* decl) { boost::optional id; - boost::variant rv; + std::variant rv; for (auto& pair : attributes) { if (pair.first == "id" || pair.first == "href" || pair.first == "ref") { @@ -619,7 +619,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs) instance_to_attribute(inst_or_reference, idx, inst); // @todo state->stack.back().inst(); - state->stack.push_back(stack_node::instance(id, boost::get(inst_or_reference))); + state->stack.push_back(stack_node::instance(id, std::get(inst_or_reference))); } else if (attribute_type->as_named_type()->declared_type()->as_select_type() != nullptr) { // Select types cause an additional indirection, so the current stack node is simply repeated state->stack.push_back(stack_node::select(state->stack.back().inst(), idx)); diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h new file mode 100644 index 0000000000..59917ccd61 --- /dev/null +++ b/src/ifcparse/storage.h @@ -0,0 +1,542 @@ +#ifndef STORAGE_H +#define STORAGE_H + +#include "rocksdb_map_adapter.h" +#include "rocksdb_set_view.h" +#include "map_variant.h" +#include "map_transformer.h" +#include "set_to_map_transformer.h" +#include "file_open_status.h" + +#include + +#include +#include +#include +#include +#include +#include + +#ifndef SWIG + +template +class variant_iterator { +public: + // The variant type holding one of the underlying iterators. + using variant_type = std::variant; + + // Assuming that all iterator types have the same value_type, difference_type, etc. + using value_type = std::common_type_t::value_type...>; + using difference_type = std::common_type_t::difference_type...>; + using pointer = value_type*; + using reference = value_type&; + // For simplicity, we use input_iterator_tag; if all underlying iterators support more, + // you could compute the common iterator_category. + using iterator_category = std::input_iterator_tag; + + // Default constructor. + variant_iterator() = default; + + // Construct from any one of the underlying iterator types. + template + variant_iterator(Iterator it) : it_(it) {} + + // Dereference operator. + decltype(auto) operator*() const { + return std::visit([](const auto& iter) -> decltype(auto) { + return *iter; + }, it_); + } + + // Arrow operator. + decltype(auto) operator->() const { + return std::visit([](const auto& iter) -> decltype(auto) { + return iter.operator->(); + }, it_); + } + + // Pre-increment operator. + variant_iterator& operator++() { + std::visit([](auto& iter) { ++iter; }, it_); + return *this; + } + + // Post-increment operator. + variant_iterator operator++(int) { + variant_iterator temp(*this); + ++(*this); + return temp; + } + + // Pre-decrement operator. + variant_iterator& operator--() { + std::visit([](auto& iter) { --iter; }, it_); + return *this; + } + + // Post-decrement operator. + variant_iterator operator--(int) { + variant_iterator temp(*this); + --(*this); + return temp; + } + + // Equality comparison. + friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) { + return lhs.it_ == rhs.it_; + } + + // Inequality comparison. + friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) { + return !(lhs == rhs); + } + +private: + variant_type it_; +}; + +#endif + +namespace IfcParse { + + struct InstanceReference { + int v; + size_t file_offset; + operator int() const { + return v; + } + }; + + typedef std::variant reference_or_simple_type; + typedef std::list, std::vector>>>> unresolved_references; + + class IfcFile; + class IfcSpfLexer; + class IfcSpfStream; + + enum TokenType { + Token_NONE, + Token_STRING, + Token_IDENTIFIER, + Token_OPERATOR, + Token_ENUMERATION, + Token_KEYWORD, + Token_INT, + Token_BOOL, + Token_FLOAT, + Token_BINARY + }; + + struct Token { + IfcSpfLexer* lexer; //TODO: remove it from here + unsigned startPos; + TokenType type; + union { + char value_char; //types: OPERATOR + int value_int; //types: INT, IDENTIFIER + double value_double; //types: FLOAT + }; + + Token() : lexer(0), + startPos(0), + type(Token_NONE) { + } + Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type) + : lexer(_lexer), + startPos(_startPos), + type(_type) { + } + }; + + struct parse_context { + std::list< + std::variant< + IfcUtil::IfcBaseClass*, + Token, + parse_context* + >> tokens_; + + parse_context() {}; + ~parse_context(); + + parse_context(const parse_context&) = delete; + parse_context& operator=(const parse_context&) = delete; + + parse_context(parse_context&&) = default; + parse_context& operator=(parse_context&&) = default; + + parse_context& push(); + + void push(Token t); + + void push(IfcUtil::IfcBaseClass* inst); + + IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size); + }; + + namespace impl { + struct in_memory_file_storage { + IfcParse::IfcSpfLexer* tokens; + // IfcParse::IfcSpfStream* stream; + + // Either one of these needs to be set + IfcParse::IfcFile* file; + const IfcParse::schema_definition* schema; + + unresolved_references* references_to_resolve = nullptr; + + typedef std::map entities_by_type_t; + typedef boost::unordered_map entity_instance_by_name_t; + typedef boost::unordered_map type_instance_by_name_t; + typedef std::map entity_instance_by_guid_t; + typedef std::tuple inverse_attr_record; + enum INVERSE_ATTR { + INSTANCE_ID, + INSTANCE_TYPE, + ATTRIBUTE_INDEX + }; + typedef std::map> entities_by_ref_t; + typedef entity_instance_by_name_t::iterator iterator; + + in_memory_file_storage() : tokens(nullptr), file(nullptr), schema(nullptr) {} + in_memory_file_storage(const in_memory_file_storage&) = delete; + in_memory_file_storage(const in_memory_file_storage&&) = delete; + + class type_iterator : public entities_by_type_t::const_iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = entities_by_type_t::key_type; + using difference_type = typename entities_by_type_t::const_iterator::difference_type; + using pointer = value_type const*; + using reference = value_type const&; + + type_iterator() : entities_by_type_t::const_iterator() {}; + + type_iterator(const entities_by_type_t::const_iterator& iter) + : entities_by_type_t::const_iterator(iter) {}; + + entities_by_type_t::key_type const* operator->() const { + return &entities_by_type_t::const_iterator::operator->()->first; + } + + entities_by_type_t::key_type const& operator*() const { + return entities_by_type_t::const_iterator::operator*().first; + } + + type_iterator& operator++() { + entities_by_type_t::const_iterator::operator++(); + return *this; + } + + type_iterator operator++(int) { + type_iterator tmp(*this); + operator++(); + return tmp; + } + }; + + + static bool guid_map_; + static bool guid_map() { return guid_map_; } + static void guid_map(bool b) { guid_map_ = b; } + + entity_instance_by_name_t byid_; + type_instance_by_name_t tbyid_; + entities_by_type_t bytype_excl_; + entities_by_ref_t byref_excl_; + entity_instance_by_guid_t byguid_; + + void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); + void try_read_semicolon() const; + + void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); + void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); + + // @todo is this still used + IfcEntityInstanceData read(unsigned int index); + void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id); + + file_open_status good_ = file_open_status::SUCCESS; + + IfcUtil::IfcBaseClass* instance_by_id(int id); + + void add_type_ref(IfcUtil::IfcBaseClass* new_entity) { + auto ty = new_entity->declaration().as_entity(); + if (ty) { + if (bytype_excl_.find(ty) == bytype_excl_.end()) { + bytype_excl_[ty].reset(new aggregate_of_instance()); + } + bytype_excl_[ty]->push(new_entity); + } + } + void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) { + auto ty = new_entity->declaration().as_entity(); + if (ty) { + auto it = bytype_excl_.find(ty); + if (it != bytype_excl_.end()) { + it->second->remove(new_entity); + if (it->second->size() == 0) { + bytype_excl_.erase(ty); + } + } + } + } + + void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); + + template + T* create(); + + IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl); + }; + + class rocks_db_file_storage { + public: + rocksdb::DB* db; + rocksdb::WriteOptions wopts; + rocksdb::ReadOptions ropts; + IfcParse::IfcFile* file; + + enum instance_ref { + typedecl_ref, + entityinstance_ref + }; + + // to make sure that instance pointer are constant during file lifetime + // cache instances because we want stable pointers + // @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side? + typedef std::map entity_by_iden_cache_t; + entity_by_iden_cache_t instance_cache_, type_instance_cache_; + + // @todo all these size_ts should probably be uint32_t for consistency with in-mem storage + + // lookup id->identity + // typedef rocksdb_map_adapter identity_by_id_t; + // identity_by_id_t byid_; + typedef rocksdb_set_view instance_name_view_t; + instance_name_view_t instance_ids_; + typedef set_to_map_transformer> entity_instance_by_name_t; + entity_instance_by_name_t instance_by_name_; + + // typedef map_transformer, std::function, std::function> entity_by_id_t; + // storage is now Instance name -> Identity -> Pointer (cached) + // entity_by_id_t byidentity_; + + // index in schema to binary serialized ids + typedef rocksdb_map_adapter instance_id_str_by_type_t; + instance_id_str_by_type_t bytype_; + + // guid -> id + typedef rocksdb_map_adapter instance_id_by_guid_str_t; + instance_id_by_guid_str_t byguid_internal_; + + // guid -> id -> instance + typedef map_transformer, std::function, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_instance_by_guid_t; + entity_instance_by_guid_t byguid_; + + typedef std::tuple inverse_attr_record; + enum INVERSE_ATTR { + INSTANCE_ID, + INSTANCE_TYPE, + ATTRIBUTE_INDEX + }; + typedef rocksdb_map_adapter> entities_by_ref_t; + entities_by_ref_t byref_excl_; + + // @todo naming + rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file); + ~rocks_db_file_storage(); + + bool read_schema(const IfcParse::schema_definition*& schema); + + IfcUtil::IfcBaseClass* assert_existance(size_t instanceId, instance_ref r); + + // @todo this could be another map_adapter? + /* + class rocksdb_instance_iterator { + private: + rocksdb::Iterator* state_; + rocks_db_file_storage* storage_; + + static constexpr char prefix_[] = "i|"; + + boost::optional read_id_() const { + auto sv = state_->key().ToStringView(); + auto ii = sv.find("|", 2); + if (ii != decltype(sv)::npos) { + char* pEnd; + long result = strtol(sv.data() + 2, &pEnd, 10); + if (*pEnd == '|') { + return (size_t)result; + } + } + return boost::none; + } + public: + rocksdb_instance_iterator() + : state_(nullptr) + , storage_(nullptr) + {} + rocksdb_instance_iterator(rocks_db_file_storage* fs) + : storage_(fs) + { + state_ = fs->db->NewIterator(rocksdb::ReadOptions()); + state_->Seek(prefix_); + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + } + } + rocksdb_instance_iterator& operator++() { + if (!state_) { + return *this; + } + auto last_id = read_id_(); + while (state_->Valid()) { + state_->Next(); + // Stop if we've left the prefix range. + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + break; + } + if (read_id_() != last_id) { + break; + } + } + return *this; + } + rocksdb_instance_iterator operator++(int) { + rocksdb_instance_iterator temp = *this; + ++(*this); + return temp; + } + bool operator==(const rocksdb_instance_iterator& other) const { + if (state_ == nullptr && other.state_ == nullptr) { + return true; + } else { + return read_id_() == other.read_id_(); + } + } + + bool operator!=(const rocksdb_instance_iterator& other) const { + return !(*this == other); + } + + IfcUtil::IfcBaseClass* operator*() const; + }; + */ + + // @todo merge iterators (template?) + class rocksdb_types_iterator { + private: + rocksdb::Iterator* state_; + const rocks_db_file_storage* storage_; + + static constexpr char prefix_[] = "t|"; + + boost::optional read_id_() const { + auto sv = state_->key().ToStringView(); + auto ii = sv.find("|", 2); + if (ii != decltype(sv)::npos) { + char* pEnd; + long result = strtol(sv.data() + 2, &pEnd, 10); + if (*pEnd == '|') { + return (size_t)result; + } + } + return boost::none; + } + public: + using iterator_category = std::forward_iterator_tag; + using value_type = const IfcParse::declaration*; + // @todo ? + using difference_type = ptrdiff_t; + using pointer = value_type const*; + using reference = value_type const&; + + rocksdb_types_iterator() + : state_(nullptr) + , storage_(nullptr) + { + } + + rocksdb_types_iterator(const rocks_db_file_storage* fs) + : storage_(fs) + { + state_ = fs->db->NewIterator(rocksdb::ReadOptions()); + state_->Seek(prefix_); + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + } + } + + rocksdb_types_iterator& operator++() { + if (!state_) { + return *this; + } + auto last_id = read_id_(); + while (state_->Valid()) { + state_->Next(); + // Stop if we've left the prefix range. + if (!state_->Valid() || !state_->key().starts_with(prefix_)) { + delete state_; + state_ = nullptr; + break; + } + if (read_id_() != last_id) { + break; + } + } + return *this; + } + + rocksdb_types_iterator operator++(int) { + rocksdb_types_iterator temp = *this; + ++(*this); + return temp; + } + + bool operator==(const rocksdb_types_iterator& other) const { + if (state_ == nullptr && other.state_ == nullptr) { + return true; + } else { + return read_id_() == other.read_id_(); + } + } + + bool operator!=(const rocksdb_types_iterator& other) const { + return !(*this == other); + } + + value_type const& operator*() const; + + value_type const* operator->() const { + return &operator*(); + } + }; + + // @todo rocksdb_instance_iterator? + using const_iterator = entity_instance_by_name_t::iterator; + + void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); + void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index); + + // @todo a bit hard as a map because of value_type being an aggregate + void add_type_ref(IfcUtil::IfcBaseClass* new_entity); + void remove_type_ref(IfcUtil::IfcBaseClass* new_entity); + + IfcUtil::IfcBaseClass* instance_by_id(int id); + + void process_deletion_inverse(IfcUtil::IfcBaseClass* inst); + + template + T* create(); + + IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl); + }; + } +} + +#endif // STORAGE_H diff --git a/src/ifcwrap/IfcParseWrapper.i b/src/ifcwrap/IfcParseWrapper.i index c7b86150fd..d94d860a96 100644 --- a/src/ifcwrap/IfcParseWrapper.i +++ b/src/ifcwrap/IfcParseWrapper.i @@ -34,6 +34,10 @@ private: %ignore IfcParse::IfcFile::internal_guid_map; %ignore IfcParse::IfcFile::storage_; +%ignore IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer); + +%ignore IfcParse::InstanceStreamer::read_instance; + %ignore in_memory_file_storage; %ignore rocks_db_file_storage; @@ -571,6 +575,7 @@ static IfcUtil::ArgumentType helper_fn_attribute_type(const IfcUtil::IfcBaseClas %include "../ifcparse/ifc_parse_api.h" %include "../ifcparse/IfcSpfHeader.h" %include "../ifcparse/IfcFile.h" +%include "../ifcparse/file_open_status.h" %include "../ifcparse/IfcBaseClass.h" %include "../ifcparse/IfcSchema.h" @@ -866,3 +871,203 @@ static IfcUtil::ArgumentType helper_fn_attribute_type(const IfcUtil::IfcBaseClas } %} +%extend IfcParse::InstanceStreamer { + PyObject* read_instance_py() { + auto simply_type_to_dictionary = [&](IfcUtil::IfcBaseClass* t) -> PyObject* { + const auto& nm = t->declaration().name(); + auto ifc_val = t->get_attribute_value(0); + auto attribute_val_py = ifc_val.apply_visitor([&](const auto& t) { + using U = std::decay_t; + + if constexpr (is_std_vector_v) { + return pythonize_vector(t); + } else if constexpr (std::is_same_v) { + return pythonize(std::string(t.value())); + } else if constexpr (std::is_same_v) { + if (feature_use_attribute_value_derived) { + return SWIG_NewPointerObj(new attribute_value_derived, SWIGTYPE_p_attribute_value_derived, SWIG_POINTER_OWN); + } else { + Py_INCREF(Py_None); + return static_cast(Py_None); + } + } else if constexpr (std::is_same_v) { + // cannot occur in streaming mode + Py_INCREF(Py_None); + return static_cast(Py_None); + } else if constexpr (std::is_same_v) { + // cannot occur in streaming mode + Py_INCREF(Py_None); + return static_cast(Py_None); + } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { + Py_INCREF(Py_None); + return static_cast(Py_None); + } else { + return pythonize(t); + } + }); + + PyObject* val = PyDict_New(); + { + const std::string& key_cpp = "type"; + auto name_py = pythonize(key_cpp); + auto value_py = pythonize(nm); + PyDict_SetItem(val, name_py, value_py); + Py_DECREF(name_py); + Py_DECREF(value_py); + } + { + const std::string& key_cpp = "value"; + auto name_py = pythonize(key_cpp); + PyDict_SetItem(val, name_py, attribute_val_py); + Py_DECREF(name_py); + Py_DECREF(attribute_val_py); + } + return val; + }; + + auto instance_reference_to_dict = [&](int i) -> PyObject* { + PyObject* val = PyDict_New(); + const std::string& key_cpp = "ref"; + auto name_py = pythonize(key_cpp); + auto value_py = pythonize(i); + PyDict_SetItem(val, name_py, value_py); + Py_DECREF(name_py); + Py_DECREF(value_py); + return val; + }; + + if (!*self) { + Py_INCREF(Py_None); + return Py_None; + } + auto inst = self->read_instance(); + if (!inst) { + Py_INCREF(Py_None); + return Py_None; + } + PyObject* d = PyDict_New(); + + { + const std::string& key_cpp = "id"; + auto name_py = pythonize(key_cpp); + auto value_py = pythonize((int) std::get<0>(*inst)); + PyDict_SetItem(d, name_py, value_py); + Py_DECREF(name_py); + Py_DECREF(value_py); + } + { + const std::string& key_cpp = "type"; + auto name_py = pythonize(key_cpp); + auto value_py = pythonize(std::get<1>(*inst)->name()); + PyDict_SetItem(d, name_py, value_py); + Py_DECREF(name_py); + Py_DECREF(value_py); + } + { + const auto* decl = std::get<1>(*inst); + const auto& data = std::get<2>(*inst); + + for (size_t i = 0; i < decl->as_entity()->attribute_count(); i++) { + auto val = data.get_attribute_value(nullptr, decl, 0, i); + + // sets dict member, returns void + val.apply_visitor([&](const auto& t) -> void { + using T = std::decay_t; + PyObject* attribute_val_py; + if constexpr (std::is_same_v) { + attribute_val_py = simply_type_to_dictionary(t); + } else { + using U = std::decay_t; + + if constexpr (is_std_vector_v) { + attribute_val_py = pythonize_vector(t); + } else if constexpr (std::is_same_v) { + attribute_val_py = pythonize(std::string(t.value())); + } else if constexpr (std::is_same_v) { + if (feature_use_attribute_value_derived) { + attribute_val_py = SWIG_NewPointerObj(new attribute_value_derived, SWIGTYPE_p_attribute_value_derived, SWIG_POINTER_OWN); + } else { + Py_INCREF(Py_None); + attribute_val_py = static_cast(Py_None); + } + } else if constexpr (std::is_same_v) { + // cannot occur in streaming mode + Py_INCREF(Py_None); + attribute_val_py = static_cast(Py_None); + } else if constexpr (std::is_same_v) { + // cannot occur in streaming mode + Py_INCREF(Py_None); + attribute_val_py = static_cast(Py_None); + } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { + Py_INCREF(Py_None); + attribute_val_py = static_cast(Py_None); + } else { + attribute_val_py = pythonize(t); + } + } + + { + auto name_py = pythonize(decl->as_entity()->attribute_by_index(i)->name()); + PyDict_SetItem(d, name_py, attribute_val_py); + Py_DECREF(name_py); + Py_DECREF(attribute_val_py); + } + }); + } + + for (auto& p : $self->references()) { + int index = p.first.index_; + auto name_py = pythonize(decl->as_entity()->attribute_by_index(index)->name()); + + std::visit([&](const auto& v) -> void { + PyObject* attribute_val_py; + using T = std::decay_t; + + if constexpr (std::is_same_v) { + if (auto* inst = std::get_if(&v)) { + // So this never happens? + } else if (auto* name = std::get_if(&v)) { + attribute_val_py = instance_reference_to_dict(*name); + } + } else if constexpr (std::is_same_v>) { + attribute_val_py = PyTuple_New(v.size()); + size_t idx = 0; + for (auto const& inner : v) { + if (auto* inst = std::get_if(&inner)) { + PyTuple_SetItem(attribute_val_py, idx++, simply_type_to_dictionary(*inst)); + } else if (auto* name = std::get_if(&inner)) { + PyTuple_SetItem(attribute_val_py, idx++, instance_reference_to_dict(*name)); + } + } + } else if constexpr (std::is_same_v>>) { + attribute_val_py = PyTuple_New(v.size()); + size_t outer_idx = 0; + for (auto const& inner : v) { + PyObject* inner_py = PyTuple_New(inner.size()); + size_t idx = 0; + for (auto const& innermost : inner) { + if (auto* inst = std::get_if(&innermost)) { + PyTuple_SetItem(inner_py, idx++, simply_type_to_dictionary(*inst)); + } else if (auto* name = std::get_if(&innermost)) { + PyTuple_SetItem(inner_py, idx++, instance_reference_to_dict(*name)); + } + } + PyTuple_SetItem(attribute_val_py, outer_idx++, inner_py); + } + } + + PyDict_SetItem(d, name_py, attribute_val_py); + Py_DECREF(name_py); + Py_DECREF(attribute_val_py); + + }, p.second); + } + } + + $self->references().clear(); + $self->inverses().clear(); + + return d; + } +} + \ No newline at end of file diff --git a/src/serializers/RocksDbSerializer.cpp b/src/serializers/RocksDbSerializer.cpp index 1720f8c0f3..74c83ac861 100644 --- a/src/serializers/RocksDbSerializer.cpp +++ b/src/serializers/RocksDbSerializer.cpp @@ -14,7 +14,8 @@ RocksDbSerializer::RocksDbSerializer(IfcParse::IfcFile* file, const std::string& options.create_if_missing = true; options.merge_operator.reset(new ConcatenateIdMergeOperator()); rocksdb::Status status = rocksdb::DB::Open(options, rocksdb_filename, &db_);*/ - output_file_ = new IfcParse::IfcFile(file_->schema(), IfcParse::FT_ROCKSDB, rocksdb_filename_); + + output_file_ = new IfcParse::IfcFile(file->schema(), IfcParse::FT_ROCKSDB, rocksdb_filename_); // We promise never to add the same instance twice output_file_->check_existance_before_adding = false; @@ -22,8 +23,212 @@ RocksDbSerializer::RocksDbSerializer(IfcParse::IfcFile* file, const std::string& output_file_->calculate_unit_factors = false; } -void RocksDbSerializer::finalize() +RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream) + : file_(input_filename) + , rocksdb_filename_(rocksdb_filename) { +} + +namespace { + // @nb copied from IfcEntityInstanceData.cpp but operating on unresolved instances + bool serialize(std::string& val, const IfcParse::reference_or_simple_type& t) + { + auto s = sizeof(size_t); + val.resize(s + 2); + val[0] = TypeEncoder::encode_type(); + // 1 = entity - stored by id (entity name) + // 2 = type - stored by identity (internal counter in class) + val[1] = t.index() == 0 ? 'i' : 't'; + size_t iden; + if (auto* name = std::get_if(&t)) { + iden = *name; + } else if (auto* inst = std::get_if(&t)) { + iden = (*inst)->identity(); + } + memcpy(val.data() + 2, &iden, s); + return true; + } + + bool serialize(std::string& val, const std::vector& t) + { + // no attempt at alignment + val.resize(t.size() * (sizeof(size_t) + 1) + 1); + val[0] = TypeEncoder::encode_type(); + char* ptr = val.data() + 1; + for (auto it = t.begin(); it != t.end(); ++it) { + *ptr = it->index() == 0 ? 'i' : 't'; + ptr++; + size_t iden; + if (auto* name = std::get_if(&*it)) { + iden = *name; + } else if (auto* inst = std::get_if(&*it)) { + iden = (*inst)->identity(); + } + memcpy(ptr, &iden, sizeof(size_t)); + ptr += sizeof(size_t); + } + return true; + } + + bool serialize(std::string& val, const std::vector>& t) + { + // @todo + return false; + } +} + +namespace { + template + std::string to_string_fixed_width(const T& t, size_t w) { + // @todo currently inactive + std::ostringstream oss; + oss << /*std::setfill('0') << std::setw(w) <<*/ t; + return oss.str(); + } +} + +void RocksDbSerializer::write_streaming_() { + const auto& input_filename = std::get(file_); + + IfcParse::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr); + + std::string tmp; + + IfcParse::InstanceStreamer streamer(input_filename); + + while (streamer) { + auto inst = streamer.read_instance(); + if (inst) { + // name can be zero in case of header instances + auto name = std::get<0>(*inst); + const auto* decl = std::get<1>(*inst); + const auto& data = std::get<2>(*inst); + + const bool is_header = decl->schema() == &Header_section_schema::get_schema(); + + std::vector simple_type_instances; + + for (size_t i = 0; i < decl->as_entity()->attribute_count(); i++) { + auto val = data.get_attribute_value(nullptr, decl, 0, i); + val.apply_visitor([&](const auto& t) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + // instance is per definition a simple type here, because instance + // references are not resolved yet, but provided in vector of + // references + simple_type_instances.push_back(t); + } + rocks_db_attribute_storage{}.set(&storage, decl, name, i, t); + }); + } + + for (auto& p : streamer.references()) { + // @nb cast to int in order not be interpreted as a char when appending to string + int index = p.first.index_; + + std::visit([&](const auto& v) { + serialize(tmp, v); + + using T = std::decay_t; + + if constexpr (std::is_same_v) { + if (auto* inst = std::get_if(&v)) { + // So this never happens? + simple_type_instances.push_back(*inst); + } + } else if constexpr (std::is_same_v>) { + for (auto const& inner : v) { + if (auto* inst = std::get_if(&inner)) { + simple_type_instances.push_back(*inst); + } + } + } else if constexpr (std::is_same_v>>) { + for (auto const& inner : v) { + for (auto const& innermost : inner) { + if (auto* inst = std::get_if(&innermost)) { + simple_type_instances.push_back(*inst); + } + } + } + } + }, p.second); + + storage.db->Put( + storage.wopts, + (is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) + + (is_header ? decl->name() : std::to_string(p.first.name_)) + "|" + + std::to_string(index), tmp); + + auto write_inverse = [&](const IfcParse::reference_or_simple_type& v) { + if (auto* ref = std::get_if(&v)) { + auto key = "v|" + to_string_fixed_width(*ref, 10) + "|" + to_string_fixed_width(decl->index_in_schema(), 4) + "|" + to_string_fixed_width(index, 2); + static std::string s; + uint32_t vv = name; + s.resize(sizeof(uint32_t)); + memcpy(s.data(), &vv, sizeof(uint32_t)); + storage.db->Merge( + storage.wopts, key, s); + } + }; + + std::visit([&](auto const& val) { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + write_inverse(val); + } else if constexpr (std::is_same_v>) { + std::for_each(val.begin(), val.end(), write_inverse); + } else if constexpr (std::is_same_v>>) { + for (auto const& inner : val) { + std::for_each(inner.begin(), inner.end(), write_inverse); + } + } + }, p.second); + } + + for (const auto* inst : simple_type_instances) { + std::string s(sizeof(size_t), ' '); + size_t v = inst->declaration().index_in_schema(); + memcpy(s.data(), &v, sizeof(size_t)); + + storage.db->Put( + storage.wopts, + (inst->declaration().as_entity() ? "i|" : "t|") + std::to_string(inst->identity()) + "|_", s); + + auto val = inst->get_attribute_value(0); + val.apply_visitor([&](const auto& t) { + rocks_db_attribute_storage{}.set(&storage, &inst->declaration(), inst->identity(), 0, t); + }); + + // @nb we also need to delete them + delete inst; + } + + // Entity type as numeric ref to index_in_schema + if (!is_header) { + std::string s(sizeof(size_t), ' '); + size_t v = decl->index_in_schema(); + memcpy(s.data(), &v, sizeof(size_t)); + storage.db->Put( + storage.wopts, + (decl->as_entity() ? "i|" : "t|") + std::to_string(name) + "|_", s); + + { + size_t v = name; + std::string s(sizeof(size_t), ' '); + memcpy(s.data(), &v, sizeof(size_t)); + storage.db->Merge(storage.wopts, "t|" + std::to_string(decl->index_in_schema()), s); + } + } + + streamer.references().clear(); + streamer.inverses().clear(); + } + } +} + + +void RocksDbSerializer::write_non_streaming_() { // Build a map of instances and their references/dependencies std::map> dependencies, dependencies_inv; std::visit([&dependencies](const auto& m) { @@ -34,9 +239,9 @@ void RocksDbSerializer::finalize() } } } - }, file_->storage_); + }, std::get(file_)->storage_); // Add bottom-rank nodes, inv mapping does not contain them - for (const auto& p : *file_) { + for (const auto& p : *std::get(file_)) { dependencies[p.first]; } @@ -80,7 +285,7 @@ void RocksDbSerializer::finalize() // Add them in topological order, so that add() never recurses into something not previously visited for (auto& i : deps_topo_order) { - output_file_->addEntity(file_->instance_by_id(i), i); + output_file_->addEntity(std::get(file_)->instance_by_id(i), i); } // Copy inverses @@ -102,4 +307,13 @@ void RocksDbSerializer::finalize() delete output_file_; } +void RocksDbSerializer::finalize() { + if (file_.index() == 0) { + write_non_streaming_(); + } else { + write_streaming_(); + } +} + + #endif \ No newline at end of file diff --git a/src/serializers/RocksDbSerializer.h b/src/serializers/RocksDbSerializer.h index e7456d28ef..89bf9ab5ca 100644 --- a/src/serializers/RocksDbSerializer.h +++ b/src/serializers/RocksDbSerializer.h @@ -10,11 +10,14 @@ class SERIALIZERS_API RocksDbSerializer : public Serializer { private: rocksdb::DB* db_; std::string rocksdb_filename_; - IfcParse::IfcFile* file_; + std::variant file_; IfcParse::IfcFile* output_file_; + void write_streaming_(); + void write_non_streaming_(); public: RocksDbSerializer(IfcParse::IfcFile* file, const std::string& rocksdb_filename); + RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream); virtual ~RocksDbSerializer() {}