From 4c13e2424c32d16e65cb8ab8f4812fa891dc5cba Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 11 Jun 2026 15:51:40 +0200 Subject: [PATCH] Configurable pointer type; std::from_chars(); aggregate inverses in vector; skip parse_context --- src/ifcgeom/kernel_registry.cpp | 4 + src/ifcparse/express.cpp | 8 + src/ifcparse/express.h | 35 +- src/ifcparse/file.cpp | 334 +------------ src/ifcparse/file.h | 5 +- src/ifcparse/parse.cpp | 810 +++++++++++++++++++++++--------- src/ifcparse/spf_header.cpp | 12 +- src/ifcparse/spf_header.h | 8 +- src/ifcparse/storage.h | 247 ++++++++-- src/plugin/plugin.cpp | 4 + src/pyodide/demo-app/index.html | 17 +- 11 files changed, 860 insertions(+), 624 deletions(-) diff --git a/src/ifcgeom/kernel_registry.cpp b/src/ifcgeom/kernel_registry.cpp index 962710da43..79f8e9acef 100644 --- a/src/ifcgeom/kernel_registry.cpp +++ b/src/ifcgeom/kernel_registry.cpp @@ -81,7 +81,11 @@ namespace { try { module = manager.load(path); } catch (const std::exception& e) { +#ifdef IFOPSH_PLUGIN_DEBUG std::cerr << "[ifcopenshell.plugin] skip kernel plugin " << path << ": " << e.what() << std::endl; +#else + static_cast(e); +#endif continue; } if (module.meta().kind_ != ifcopenshell::plugin::kind::kernel) { diff --git a/src/ifcparse/express.cpp b/src/ifcparse/express.cpp index a6503c4b56..71df55387b 100644 --- a/src/ifcparse/express.cpp +++ b/src/ifcparse/express.cpp @@ -9,19 +9,27 @@ uint32_t express::Base::identity() const { return data()->identity(); } uint32_t express::Base::id() const { return data()->id(); } const instance_data* express::Base::data() const { +#ifdef IFOPSH_SAFE_INSTANCE auto sp = data_.lock(); if (sp) { return sp.get(); } else { throw std::runtime_error("Trying to access deleted instance reference"); } +#else + return data_; +#endif } instance_data* express::Base::data() { +#ifdef IFOPSH_SAFE_INSTANCE auto sp = data_.lock(); if (sp) { return sp.get(); } else { throw std::runtime_error("Trying to access deleted instance reference"); } +#else + return data_; +#endif } diff --git a/src/ifcparse/express.h b/src/ifcparse/express.h index e4e2dd699d..abb501a1d8 100644 --- a/src/ifcparse/express.h +++ b/src/ifcparse/express.h @@ -31,6 +31,23 @@ class aggregate_of_instance; namespace ifcopenshell { + +#ifdef IFOPSH_SAFE_INSTANCE +using pointer_type = std::weak_ptr; +using shared_pointer_type = shared_pointer_type; +template +shared_pointer_type make_pointer_type(Args&&... args) { + return std::make_shared(std::forward(args)...); +} +#else +using pointer_type = instance_data*; +using shared_pointer_type = instance_data*; +template +shared_pointer_type make_pointer_type(Args&&... args) { + return new T(std::forward(args)...); +} +#endif + class file; namespace impl { struct in_memory_file_storage; @@ -49,12 +66,16 @@ class DeclaredType; class IFC_PARSE_API Base { protected: - std::weak_ptr data_; + ifcopenshell::pointer_type data_; const instance_data* data() const; instance_data* data(); public: operator bool() const { +#ifdef IFOPSH_SAFE_INSTANCE return !data_.expired(); +#else + return data_ != nullptr; +#endif } bool operator<(const Base& other) const { @@ -69,12 +90,16 @@ class IFC_PARSE_API Base { return !(*this == other); } - Base() {}; + Base() { +#ifndef IFOPSH_SAFE_INSTANCE + data_ = nullptr; +#endif + }; Base(std::nullopt_t) noexcept : Base() {} - Base(const std::weak_ptr& data) : data_(data) {} + Base(const ifcopenshell::pointer_type& data) : data_(data) {} // @todo try and make this private over time too - const std::weak_ptr& data_weak() const { return data_; } + const ifcopenshell::pointer_type& data_weak() const { return data_; } const ifcopenshell::declaration& declaration() const; @@ -150,7 +175,7 @@ class IFC_PARSE_API Select : public Base { public: Select() {} Select(std::nullopt_t) noexcept : Base() {} - Select(const std::weak_ptr& data) : Base(data) {} + Select(const ifcopenshell::pointer_type& data) : Base(data) {} Select(const Base& base) : Base(base.data_weak()) {} Base concrete() const { diff --git a/src/ifcparse/file.cpp b/src/ifcparse/file.cpp index c5a4012bda..229f93066e 100644 --- a/src/ifcparse/file.cpp +++ b/src/ifcparse/file.cpp @@ -10,336 +10,6 @@ #include #include -ifcopenshell::parse_context::~parse_context() { - for (auto& t : tokens_) { - std::visit([](auto& v) { - if constexpr (std::is_same_v, parse_context*>) { - delete v; - } - }, t); - } -} - -ifcopenshell::parse_context& ifcopenshell::parse_context::push() { - auto* pc = new parse_context; - tokens_.push_back(pc); - return *pc; -} - -void ifcopenshell::parse_context::push(token t) { - tokens_.push_back(t); -} - -void ifcopenshell::parse_context::push(const express::Base& inst) { - tokens_.push_back(inst); -} - -namespace { - template - struct is_type_in_variant; - - // Specialization when there are multiple types in the variant - template - struct is_type_in_variant, T> - { - static constexpr bool value = std::is_same::value || is_type_in_variant, T>::value; - }; - - // Specialization when there is only one type left in the variant - template - struct is_type_in_variant, T> - { - static constexpr bool value = std::is_same::value; - }; - - template - constexpr bool is_type_in_variant_v = is_type_in_variant::value; - - template - void dispatch_token(std::optional instance_id, int attribute_id, ifcopenshell::token t, ifcopenshell::declaration* decl, Fn fn) { - if (t.is_binary()) { - fn(t.as_binary()); - } else if (t.is_bool()) { - fn(t.as_bool()); - } else if (t.is_logical()) { - fn(t.as_logical()); - } else if (t.is_enumeration()) { - const auto& s = t.as_string(); - if (decl && decl->as_enumeration_type()) { - try { - fn(enumeration_reference(decl->as_enumeration_type(), decl->as_enumeration_type()->lookup_enum_offset(s))); - } catch (ifcopenshell::exception& e) { - logger::error("An enumeration literal '" + s + "' is not valid for type '" + decl->name() + "' at offset " + std::to_string(t.start_pos)); - } - } else { - logger::error("An enumeration literal '" + s + "' is not expected at attribute index '" + std::to_string(attribute_id) + "' at offset " + std::to_string(t.start_pos)); - } - } else if (t.is_int()) { - // @nb make sure is_int() comes before is_float() - fn(t.as_int()); - } else if (t.is_float()) { - fn(t.as_float()); - } else if (t.is_identifier()) { - fn(ifcopenshell::reference_or_simple_type{ifcopenshell::instance_reference{(int) t.as_identifier(), t.start_pos}}); - } else if (t.is_string()) { - fn(t.as_string()); - } else if (t.is_operator('*')) { - // This is only in place for the validator - fn(derived{}); - } - } - - template - void construct_(std::optional instance_id, int attribute_id, ifcopenshell::parse_context& p, const ifcopenshell::aggregation_type* aggr, Fn fn) { - if (p.tokens_.empty()) { - // @todo instead of ugly if-else we could also default initialize the respective - // variant types below. - if (aggr) { - auto aggr_type = ifcopenshell::make_aggregate(ifcopenshell::from_parameter_type(aggr->type_of_element())); - if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_INT) { - fn(std::vector{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_DOUBLE) { - fn(std::vector{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_STRING) { - fn(std::vector{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_BINARY) { - fn(std::vector>{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_ENTITY_INSTANCE) { - fn(std::vector{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_AGGREGATE_OF_INT) { - fn(std::vector>{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_AGGREGATE_OF_DOUBLE) { - fn(std::vector>{}); - } else if (aggr_type == ifcopenshell::Argument_AGGREGATE_OF_AGGREGATE_OF_ENTITY_INSTANCE) { - fn(std::vector>{}); - } - } - return; - } - - typedef std::variant< - blank, - - std::vector, - std::vector, - std::vector, - std::vector>, - std::vector, - - std::vector>, - std::vector>, - std::vector> - > possible_aggregation_types_t; - - possible_aggregation_types_t aggregate_storage; - - auto append_to_aggregate_storage = [&aggregate_storage](const auto& v) { - if constexpr (is_type_in_variant_v>>) { - if (aggregate_storage.index() == 0) { - aggregate_storage = std::vector>{ v }; - } else { - if (auto* vec_ptr = std::get_if>>(&aggregate_storage)) { - vec_ptr->push_back(v); - } else { - if constexpr (std::is_same_v, int>) { - auto* vec_ptr2 = std::get_if>(&aggregate_storage); - if (vec_ptr2) { - // double[] + int - vec_ptr2->push_back((double) v); - } - } - if constexpr (std::is_same_v, double>) { - auto* vec_ptr2 = std::get_if>(&aggregate_storage); - if (vec_ptr2) { - // int[] -> double[] + double - std::vector ps(vec_ptr2->begin(), vec_ptr2->end()); - ps.push_back(v); - aggregate_storage = ps; - } - } - - if constexpr (std::is_same_v, std::vector>) { - auto* vec_ptr2 = std::get_if>>(&aggregate_storage); - if (vec_ptr2) { - // double[][] + int[] - std::vector vd(v.begin(), v.end()); - vec_ptr2->push_back(vd); - } - } - if constexpr (std::is_same_v, std::vector>) { - auto* vec_ptr2 = std::get_if>>(&aggregate_storage); - if (vec_ptr2) { - // int[][] -> double[][] + double[] - std::vector> vvd; - for (auto& vv : *vec_ptr2) { - std::vector vd(vv.begin(), vv.end()); - vvd.push_back(vd); - } - vvd.push_back(v); - aggregate_storage = vvd; - } - } - - // @todo would be cool if we can trace this back to file offset - auto current = std::visit([](auto v) { - if constexpr (!std::is_same_v) { - return std::string(typeid(typename decltype(v)::value_type).name()); - } else { - // Cannot occur as aggregate_storage.which() == 0 - // is another branch several statements up. But is - // needed for consistency of return type. - return std::string{}; - } - }, aggregate_storage); - - logger::error("Inconsistent aggregate valuation while attempting to append " + std::string(typeid(decltype(v)).name()) + " to an aggregate of " + current); - - // @todo boolean -> logical upgrade - // wait a second... there are no aggregate of bool / logical in the schema.. - // - // if constexpr (std::is_same_v, bool>) { - // auto* vec_ptr = boost::get(&aggregate_storage); - // vec_ptr->push_back(v); - // } - // if constexpr (std::is_same_v, boost::tribool>) { - // auto* vec_ptr = boost::get(&aggregate_storage); - // std::vector ps(vec_ptr->begin(), vec_ptr->end()); - // ps.push_back(v); - // aggregate_storage = ps; - // } - } - } - } else { - // @todo would be cool if we can trace this back to file offset - logger::error(std::string("Aggregates of ") + typeid(decltype(v)).name() + " are not supported in the IfcOpenShell parser"); - } - }; - - for (auto& t : p.tokens_) { - std::visit([&aggregate_storage, &append_to_aggregate_storage, aggr, instance_id, attribute_id](const auto& v) { - if constexpr (std::is_same_v, ifcopenshell::token>) { - // @todo get aggregate of enumeration - dispatch_token(instance_id, attribute_id, v, aggr && aggr->type_of_element()->as_named_type() ? aggr->type_of_element()->as_named_type()->declared_type() : nullptr, append_to_aggregate_storage); - } else if constexpr (std::is_same_v, ifcopenshell::parse_context*>) { - // nested list - if constexpr (Depth < 3) { - construct_(instance_id, attribute_id, *v, nullptr, append_to_aggregate_storage); - } - } else { - append_to_aggregate_storage(ifcopenshell::reference_or_simple_type{ v }); - } - }, t); - } - - std::visit(fn, aggregate_storage); - } -} - -std::shared_ptr ifcopenshell::parse_context::construct(ifcopenshell::file* owner, std::optional name, unresolved_references& references_to_resolve, const ifcopenshell::declaration* decl, std::optional expected_size, int resolve_reference_index, bool coerce_attribute_count) { - std::vector parameter_types; - std::unique_ptr transient_named_type; - - if ((decl != nullptr) && (decl->as_type_declaration() != nullptr)) { - parameter_types = { decl->as_type_declaration()->declared_type() }; - } else if ((decl != nullptr) && (decl->as_enumeration_type() != nullptr)) { - transient_named_type.reset(new ifcopenshell::named_type(const_cast(decl))); - parameter_types = { &*transient_named_type }; - } else if ((decl != nullptr) && (decl->as_entity() != nullptr)) { - const auto& entity_attrs = decl->as_entity()->all_attributes(); - std::transform( - entity_attrs.begin(), - entity_attrs.end(), - std::back_inserter(parameter_types), - [](auto* attr) { - return attr->type_of_attribute(); - } - ); - } - - if (((decl != nullptr) && (tokens_.size() != parameter_types.size())) || - expected_size && *expected_size != tokens_.size()) - { - size_t expected = expected_size ? *expected_size : parameter_types.size(); - if (decl != nullptr && decl->schema() == &Header_section_schema::get_schema()) { - logger::warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + " for header entity " + decl->name()); - } else { - logger::warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + (name ? std::string(" for instance #" + std::to_string(*name)) : std::string(""))); - } - } - - if (tokens_.empty()) { - return std::make_shared(owner, decl, name.value_or(0), in_memory_attribute_storage(0)); - } - - in_memory_attribute_storage storage(coerce_attribute_count - ? (decl != nullptr - ? (std::min)(parameter_types.size(), tokens_.size()) - : tokens_.size()) - : tokens_.size() - ); - - auto it = tokens_.begin(); - auto kt = parameter_types.begin(); - for (; it != tokens_.end() && ((decl == nullptr) || kt != parameter_types.end()); ++it) { - auto& token = *it; - // @todo coerce to expected type, e.g empty -> std::vector, bool -> logical - const ifcopenshell::parameter_type* param_type = nullptr; - if (decl != nullptr) { - param_type = *kt; - } - - auto index = (uint8_t) std::distance(tokens_.begin(), it); - - std::visit([this, &storage, name, &references_to_resolve, index, param_type, resolve_reference_index](const auto& v) { - if constexpr (std::is_same_v, ifcopenshell::token>) { - dispatch_token(name, index, v, param_type && param_type->as_named_type() ? param_type->as_named_type()->declared_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](auto v) { - if constexpr (std::is_same_v, ifcopenshell::reference_or_simple_type>) { - if (name) { - references_to_resolve.push_back(std::make_pair( - // @todo previously this was storage but apparently the - // pointer is not constant with the moving and temporary nature - // maybe it ought to be and in that case a pointer is more direct - mutable_attribute_value{ (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index }, - v - )); - } - } else { - storage.set(index, v); - } - }); - } else if constexpr (std::is_same_v, ifcopenshell::parse_context*>) { - const auto *pt = param_type; - if (pt) { - while (pt->as_named_type() && pt->as_named_type()->declared_type()->as_type_declaration()) { - pt = pt->as_named_type()->declared_type()->as_type_declaration()->declared_type(); - } - } - construct_<0>(name, index, *v, pt ? pt->as_aggregation_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](const auto& v) { - if constexpr (std::is_same_v, std::vector>) { - if (name) { - references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v }); - } - } else if constexpr (std::is_same_v, std::vector>>) { - if (name) { - references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v }); - } - } else { - storage.set(index, v); - } - }); - } else { - storage.set(index, v); - } - }, token); - - if (decl != nullptr) { - ++kt; - } - } - - return std::make_shared(owner, decl, (decl && decl->as_entity()) ? name.value_or(0) : 0, std::move(storage)); -} - /* ifcopenshell::IfcBaseClass* ifcopenshell::impl::rocks_db_file_storage::rocksdb_instance_iterator::operator*() const { auto it = storage_->byid_.find(*read_id_()); @@ -391,7 +61,7 @@ express::Base ifcopenshell::impl::rocks_db_file_storage::assert_existance(size_t } // @nb note that in case of type declarations we pass the identity as the number so // that we can read back the attributes from the db (we cannot assign to identity). - auto data = std::make_shared(file, decl, number, rocks_db_attribute_storage{}); + auto data = ifcopenshell::make_pointer_type(file, decl, number, rocks_db_attribute_storage{}); if (r == ifcopenshell::impl::rocks_db_file_storage::entityinstance_ref) { instance_cache_.insert({number, data}); } else { @@ -674,7 +344,7 @@ express::Base ifcopenshell::impl::in_memory_file_storage::create(const ifcopensh } else { throw std::runtime_error("Requires and entity or type declaration"); } - auto data = std::make_shared(file, decl, instance_name, decl->as_entity() ? in_memory_attribute_storage(decl->as_entity()->attribute_count()) : in_memory_attribute_storage(1)); + auto data = ifcopenshell::make_pointer_type(file, decl, instance_name, decl->as_entity() ? in_memory_attribute_storage(decl->as_entity()->attribute_count()) : in_memory_attribute_storage(1)); if (instance_name) { byid_.insert({instance_name, data}); } else { diff --git a/src/ifcparse/file.h b/src/ifcparse/file.h index 9be7803a6f..0a720c914e 100644 --- a/src/ifcparse/file.h +++ b/src/ifcparse/file.h @@ -107,6 +107,7 @@ private: bool yield_header_instances_ = true; std::vector types_to_bypass_; std::vector bypassed_instances_; + std::vector types_to_bypass_materialized_; void initialize_header(); spf_header& ensure_header(); @@ -143,7 +144,7 @@ private: return storage_.byref_excl_; } - std::vector> steal_instances() { + std::vector steal_instances() { return storage_.steal_instances(); } @@ -171,7 +172,7 @@ private: ~instance_streamer() = default; - std::optional>> read_instance(); + std::optional> read_instance(); }; class uninitialized_tag {}; diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index a60adb883f..1b02dc3cc5 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #ifdef USE_MMAP #include @@ -48,60 +49,6 @@ using namespace ifcopenshell; -// A static locale for the real number parser. strtod() is locale-dependent, causing issues -// in locales that have ',' as a decimal separator. Therefore the non standard _strtod_l() / -// strtod_l() is used and a reference to the "C" locale is obtained here. The alternative is -// to use std::istringstream::imbue(std::locale::classic()), but there are subtleties in -// parsing in MSVC2010 and it appears to be much slower. -#if defined(_MSC_VER) - -static _locale_t locale = (_locale_t)0; -void init_locale() { - if (locale == (_locale_t)0) { - locale = _create_locale(LC_NUMERIC, "C"); - } -} - -#else - -#if defined(__MINGW64__) || defined(__MINGW32__) -#include -#include - -typedef void* locale_t; -static locale_t locale = (locale_t)0; - -void init_locale() {} - -double strtod_l(const char* start, char** end, locale_t loc) { - double d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - size_t nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} - -#else - -#ifdef __APPLE__ -#include -#endif -#include - -static locale_t locale = (locale_t)0; -void init_locale() { - if (locale == (locale_t)0) { - locale = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0); - } -} - -#endif - -#endif - template spf_lexer::spf_lexer(Reader* stream_) { stream = stream_; @@ -174,27 +121,22 @@ std::string& spf_lexer::get_temp_string() const { namespace { -bool parse_int_(const char* pStart, int& val) { - char* pEnd; - long result = strtol(pStart, &pEnd, 10); - if (*pEnd != 0) { +template +bool parse_num_(const char* pStart, size_t size, T& val) { + if (size == 0) { return false; } - val = (int)result; - return true; -} - -bool parse_float_(const char* pStart, double& val) { - char* pEnd; -#ifdef _MSC_VER - double result = _strtod_l(pStart, &pEnd, locale); -#else - double result = strtod_l(pStart, &pEnd, locale); -#endif - if (*pEnd != 0) { + if (*pStart == '+') { + ++pStart; + --size; + if (size == 0) { + return false; + } + } + auto re = std::from_chars(pStart, pStart + size, val); + if (re.ec != std::errc() || re.ptr != pStart + size) { return false; } - val = result; return true; } @@ -382,7 +324,7 @@ token spf_lexer::next() { return token(pos, token::Token_BOOL, str[0]); } else if (ttype == token::Token_IDENTIFIER) { int int_val; - if (!parse_int_(str.c_str(), int_val)) { + if (!parse_num_(str.c_str(), str.size(), int_val)) { throw invalid_token_exception(pos, str, "instance name"); } pop_pool_entry(); @@ -394,11 +336,11 @@ token spf_lexer::next() { if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) { ttype = token::Token_KEYWORD; return token(pos, ttype, str); - } else if (parse_int_(str.c_str(), int_val)) { + } else if (parse_num_(str.c_str(), str.size(), int_val)) { ttype = token::Token_INT; pop_pool_entry(); return token(pos, ttype, int_val); - } else if (parse_float_(str.c_str(), float_val)) { + } else if (parse_num_(str.c_str(), str.size(), float_val)) { ttype = token::Token_FLOAT; pop_pool_entry(); return token(pos, float_val); @@ -572,61 +514,490 @@ std::string token::to_string() { return result; } -// -// Reads the arguments from a list of token -// Aditionally, registers the ids (i.e. #[\d]+) in the inverse map -// -template -void ifcopenshell::impl::in_memory_file_storage::load(ifcopenshell::spf_lexer* tokens, std::optional entity_instance_name, const ifcopenshell::entity* entity, parse_context& context, int attribute_index) { - token next = tokens->next(); +namespace { - size_t attribute_index_within_data = 0; - size_t return_value = 0; +template +struct is_type_in_variant; + +template +struct is_type_in_variant, T> +{ + static constexpr bool value = std::is_same::value || is_type_in_variant, T>::value; +}; + +template +struct is_type_in_variant, T> +{ + static constexpr bool value = std::is_same::value; +}; + +template +constexpr bool is_type_in_variant_v = is_type_in_variant::value; + +class parameter_type_view { + const ifcopenshell::declaration* declaration_; + const std::vector* attributes_; + std::unique_ptr transient_named_type_; + +public: + parameter_type_view(const ifcopenshell::declaration* declaration) + : declaration_(declaration) + , attributes_(nullptr) + { + if (declaration_ && declaration_->as_entity()) { + attributes_ = &declaration_->as_entity()->all_attributes(); + } else if (declaration_ && declaration_->as_enumeration_type()) { + transient_named_type_.reset(new ifcopenshell::named_type(const_cast(declaration_))); + } + } + + size_t size() const { + if (attributes_) { + return attributes_->size(); + } + return declaration_ ? 1 : 0; + } + + const ifcopenshell::parameter_type* operator[](size_t index) const { + if (attributes_) { + return index < attributes_->size() ? (*attributes_)[index]->type_of_attribute() : nullptr; + } + if (index != 0 || !declaration_) { + return nullptr; + } + if (auto* type_declaration = declaration_->as_type_declaration()) { + return type_declaration->declared_type(); + } + if (declaration_->as_enumeration_type()) { + return transient_named_type_.get(); + } + return nullptr; + } +}; + +const ifcopenshell::parameter_type* unwrap_type_declarations(const ifcopenshell::parameter_type* parameter_type) { + while (parameter_type && parameter_type->as_named_type() && + parameter_type->as_named_type()->declared_type()->as_type_declaration()) { + parameter_type = parameter_type->as_named_type()->declared_type()->as_type_declaration()->declared_type(); + } + return parameter_type; +} + +ifcopenshell::declaration* declared_type(const ifcopenshell::parameter_type* parameter_type) { + parameter_type = unwrap_type_declarations(parameter_type); + return parameter_type && parameter_type->as_named_type() ? parameter_type->as_named_type()->declared_type() : nullptr; +} + +const ifcopenshell::aggregation_type* aggregate_parameter_type(const ifcopenshell::parameter_type* parameter_type) { + parameter_type = unwrap_type_declarations(parameter_type); + return parameter_type ? parameter_type->as_aggregation_type() : nullptr; +} + +const ifcopenshell::aggregation_type* nested_aggregation_type(const ifcopenshell::aggregation_type* aggregate_type) { + return aggregate_type ? aggregate_parameter_type(aggregate_type->type_of_element()) : nullptr; +} + +void warn_attribute_count( + const ifcopenshell::declaration* declaration, + std::optional instance_name, + size_t expected_size, + size_t actual_size +) { + if (!declaration || expected_size == actual_size) { + return; + } + if (declaration->schema() == &Header_section_schema::get_schema()) { + logger::warning("Expected " + std::to_string(expected_size) + " attribute values, found " + std::to_string(actual_size) + " for header entity " + declaration->name()); + } else { + logger::warning("Expected " + std::to_string(expected_size) + " attribute values, found " + std::to_string(actual_size) + (instance_name ? std::string(" for instance #" + std::to_string(*instance_name)) : std::string(""))); + } +} + +template +void dispatch_token_direct(ifcopenshell::token token, ifcopenshell::declaration* declaration, int attribute_index, Fn&& fn) { + if (token.is_binary()) { + fn(token.as_binary()); + } else if (token.is_bool()) { + fn(token.as_bool()); + } else if (token.is_logical()) { + fn(token.as_logical()); + } else if (token.is_enumeration()) { + const auto& value = token.as_string(); + if (declaration && declaration->as_enumeration_type()) { + try { + fn(enumeration_reference(declaration->as_enumeration_type(), declaration->as_enumeration_type()->lookup_enum_offset(value))); + } catch (ifcopenshell::exception&) { + logger::error("An enumeration literal '" + value + "' is not valid for type '" + declaration->name() + "' at offset " + std::to_string(token.start_pos)); + } + } else { + logger::error("An enumeration literal '" + value + "' is not expected at attribute index '" + std::to_string(attribute_index) + "' at offset " + std::to_string(token.start_pos)); + } + } else if (token.is_int()) { + fn(token.as_int()); + } else if (token.is_float()) { + fn(token.as_float()); + } else if (token.is_identifier()) { + fn(ifcopenshell::reference_or_simple_type{ifcopenshell::instance_reference{(int) token.as_identifier(), token.start_pos}}); + } else if (token.is_string()) { + fn(token.as_string()); + } else if (token.is_operator('*')) { + fn(derived{}); + } +} + +typedef std::variant< + blank, + + std::vector, + std::vector, + std::vector, + std::vector>, + std::vector, + + std::vector>, + std::vector>, + std::vector> +> direct_aggregate_storage; + +struct direct_aggregate { + direct_aggregate_storage storage; + size_t pending_empty_aggregates = 0; + size_t values = 0; + + template + void append(const T& value) { + ++values; + if constexpr (is_type_in_variant_v>>) { + if constexpr ( + std::is_same_v, std::vector> || + std::is_same_v, std::vector> || + std::is_same_v, std::vector> + ) { + if (storage.index() == 0 && pending_empty_aggregates) { + append_promoted(value); + return; + } + } + if (pending_empty_aggregates) { + logger::error("Inconsistent aggregate valuation while attempting to append " + std::string(typeid(T).name()) + " after an empty nested aggregate"); + pending_empty_aggregates = 0; + } + if (storage.index() == 0) { + storage = std::vector>{value}; + } else if (auto* vector = std::get_if>>(&storage)) { + vector->push_back(value); + } else { + append_promoted(value); + } + } else { + logger::error(std::string("Aggregates of ") + typeid(T).name() + " are not supported in the IfcOpenShell parser"); + } + } + + void append_empty_nested() { + ++values; + if (auto* int_vector = std::get_if>>(&storage)) { + int_vector->emplace_back(); + } else if (auto* double_vector = std::get_if>>(&storage)) { + double_vector->emplace_back(); + } else if (auto* reference_vector = std::get_if>>(&storage)) { + reference_vector->emplace_back(); + } else if (storage.index() == 0) { + ++pending_empty_aggregates; + } else { + logger::error("Inconsistent aggregate valuation while attempting to append an empty nested aggregate"); + } + } + +private: + template + void append_promoted(const T& value) { + if constexpr (std::is_same_v, int>) { + if (auto* vector = std::get_if>(&storage)) { + vector->push_back((double) value); + return; + } + } + if constexpr (std::is_same_v, double>) { + if (auto* vector = std::get_if>(&storage)) { + std::vector promoted(vector->begin(), vector->end()); + promoted.push_back(value); + storage = std::move(promoted); + return; + } + } + if constexpr (std::is_same_v, std::vector>) { + if (storage.index() == 0) { + std::vector> promoted(pending_empty_aggregates); + pending_empty_aggregates = 0; + promoted.push_back(value); + storage = std::move(promoted); + return; + } + if (auto* vector = std::get_if>>(&storage)) { + vector->push_back(value); + return; + } + if (auto* vector = std::get_if>>(&storage)) { + std::vector promoted(value.begin(), value.end()); + vector->push_back(std::move(promoted)); + return; + } + } + if constexpr (std::is_same_v, std::vector>) { + if (storage.index() == 0) { + std::vector> promoted(pending_empty_aggregates); + pending_empty_aggregates = 0; + promoted.push_back(value); + storage = std::move(promoted); + return; + } + if (auto* vector = std::get_if>>(&storage)) { + vector->push_back(value); + return; + } + if (auto* vector = std::get_if>>(&storage)) { + std::vector> promoted; + promoted.reserve(vector->size() + 1); + for (const auto& nested : *vector) { + promoted.emplace_back(nested.begin(), nested.end()); + } + promoted.push_back(value); + storage = std::move(promoted); + return; + } + } + if constexpr (std::is_same_v, std::vector>) { + if (storage.index() == 0) { + std::vector> promoted(pending_empty_aggregates); + pending_empty_aggregates = 0; + promoted.push_back(value); + storage = std::move(promoted); + return; + } + if (auto* vector = std::get_if>>(&storage)) { + vector->push_back(value); + return; + } + } + + auto current = std::visit([](auto v) { + if constexpr (!std::is_same_v) { + return std::string(typeid(typename decltype(v)::value_type).name()); + } else { + return std::string{}; + } + }, storage); + logger::error("Inconsistent aggregate valuation while attempting to append " + std::string(typeid(T).name()) + " to an aggregate of " + current); + } +}; + +void append_empty_direct_aggregate(const ifcopenshell::aggregation_type* aggregate_type, direct_aggregate& target) { + if (!aggregate_type) { + target.append_empty_nested(); + return; + } + + auto argument_type = ifcopenshell::make_aggregate(ifcopenshell::from_parameter_type(aggregate_type->type_of_element())); + if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_INT) { + target.storage = std::vector{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_DOUBLE) { + target.storage = std::vector{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_STRING) { + target.storage = std::vector{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_BINARY) { + target.storage = std::vector>{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_ENTITY_INSTANCE) { + target.storage = std::vector{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_AGGREGATE_OF_INT) { + target.storage = std::vector>{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_AGGREGATE_OF_DOUBLE) { + target.storage = std::vector>{}; + } else if (argument_type == ifcopenshell::Argument_AGGREGATE_OF_AGGREGATE_OF_ENTITY_INSTANCE) { + target.storage = std::vector>{}; + } else { + target.append_empty_nested(); + } +} + +template +void set_direct_attribute( + in_memory_attribute_storage& storage, + std::optional instance_name, + ifcopenshell::unresolved_references* references_to_resolve, + size_t attribute_index, + int resolve_reference_index, + const T& value +) { + if constexpr (std::is_same_v, ifcopenshell::reference_or_simple_type>) { + if (instance_name && references_to_resolve) { + references_to_resolve->push_back(std::make_pair( + mutable_attribute_value{(uint32_t) *instance_name, resolve_reference_index == -1 ? (uint8_t) attribute_index : (uint8_t) resolve_reference_index}, + value + )); + } + } else if constexpr (std::is_same_v, std::vector>) { + if (instance_name && references_to_resolve) { + references_to_resolve->push_back({{(uint32_t) *instance_name, resolve_reference_index == -1 ? (uint8_t) attribute_index : (uint8_t) resolve_reference_index}, value}); + } + } else if constexpr (std::is_same_v, std::vector>>) { + if (instance_name && references_to_resolve) { + references_to_resolve->push_back({{(uint32_t) *instance_name, resolve_reference_index == -1 ? (uint8_t) attribute_index : (uint8_t) resolve_reference_index}, value}); + } + } else { + storage.set(attribute_index, value); + } +} + +template +void skip_aggregate(ifcopenshell::spf_lexer* tokens) { + size_t depth = 1; + while (depth) { + token next = tokens->next(); + if (!next) { + break; + } + if (next.is_operator('(')) { + ++depth; + } else if (next.is_operator(')')) { + --depth; + } + } +} + +template +direct_aggregate read_direct_aggregate( + ifcopenshell::impl::in_memory_file_storage& storage, + ifcopenshell::spf_lexer* tokens, + std::optional entity_instance_name, + const ifcopenshell::entity* entity, + int attribute_index, + const ifcopenshell::aggregation_type* aggregate_type +) { + direct_aggregate aggregate; + token next = tokens->next(); while (next) { if (next.is_operator(',')) { - if (attribute_index == -1) { - attribute_index_within_data += 1; - } } else if (next.is_operator(')')) { break; } else if (next.is_operator('(')) { - return_value++; - load(tokens, entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); - } else { - return_value++; - if (next.is_identifier() && entity && entity_instance_name) { - register_inverse(*entity_instance_name, entity, next.value_int, attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); + auto nested = read_direct_aggregate(storage, tokens, entity_instance_name, entity, attribute_index, nested_aggregation_type(aggregate_type)); + if (nested.values == 0 && nested.storage.index() == 0) { + aggregate.append_empty_nested(); + } else { + std::visit([&aggregate](const auto& value) { + if constexpr (!std::is_same_v, blank>) { + aggregate.append(value); + } + }, nested.storage); } + } else if (next.is_keyword()) { + try { + const auto* declaration = (storage.schema ? storage.schema : storage.file->schema())->declaration_by_name(next.as_string()); + tokens->next(); + auto data = storage.load(tokens, entity_instance_name, declaration, entity, attribute_index); + storage.read_simple_type_instances.push_back(data); + aggregate.append(ifcopenshell::reference_or_simple_type{express::Base(data)}); + } catch (exception& e) { + logger::message(logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(next.start_pos)); + } + } else { + if (next.is_identifier() && entity && entity_instance_name) { + storage.register_inverse((unsigned)*entity_instance_name, entity, next.value_int, attribute_index); + } + dispatch_token_direct(next, aggregate_type && aggregate_type->type_of_element()->as_named_type() ? aggregate_type->type_of_element()->as_named_type()->declared_type() : nullptr, attribute_index, [&aggregate](const auto& value) { + aggregate.append(value); + }); + } + next = tokens->next(); + } - if (next.is_keyword()) { + if (aggregate.values == 0) { + append_empty_direct_aggregate(aggregate_type, aggregate); + } + + return aggregate; +} + +} // namespace + +// +// Reads the arguments from a list of tokens directly into instance_data storage. +// Additionally, registers the ids (i.e. #[\d]+) in the inverse map. +// +template +shared_pointer_type ifcopenshell::impl::in_memory_file_storage::load( + ifcopenshell::spf_lexer* tokens, + std::optional entity_instance_name, + const ifcopenshell::declaration* declaration, + const ifcopenshell::entity* entity, + int attribute_index, + bool coerce_attribute_count +) { + static_cast(coerce_attribute_count); + + parameter_type_view parameter_types(declaration); + const size_t expected_size = parameter_types.size(); + in_memory_attribute_storage storage(expected_size); + + token next = tokens->next(); + size_t attribute_index_within_data = 0; + size_t values_read = 0; + + while (next) { + if (next.is_operator(',')) { + ++attribute_index_within_data; + } else if (next.is_operator(')')) { + break; + } else { + ++values_read; + const bool retain_value = attribute_index_within_data < expected_size; + const ifcopenshell::parameter_type* parameter_type = retain_value ? parameter_types[attribute_index_within_data] : nullptr; + const int reference_attribute_index = attribute_index == -1 ? (int) attribute_index_within_data : attribute_index; + + if (next.is_operator('(')) { + if (retain_value) { + auto aggregate = read_direct_aggregate(*this, tokens, entity_instance_name, entity, reference_attribute_index, aggregate_parameter_type(parameter_type)); + std::visit([&](const auto& value) { + if constexpr (!std::is_same_v, blank>) { + set_direct_attribute(storage, entity_instance_name, references_to_resolve, attribute_index_within_data, attribute_index, value); + } + }, aggregate.storage); + } else { + skip_aggregate(tokens); + } + } else if (next.is_keyword()) { try { - const auto* decl = (schema ? schema : file->schema())->declaration_by_name(next.as_string()); - parse_context ps; + const auto* simple_declaration = (schema ? schema : file->schema())->declaration_by_name(next.as_string()); tokens->next(); - // The only case we know where a defined type contains entity - // instance references is IfcPropertySetDefinitionSet. For - // that purpose we propagate the entity_instance_name to - // register inverses to the host entity (and not the defined - // type) and to be able to actually register the references in - // the 2nd pass. - load(tokens, entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index); - express::Base simple_type_instance(read_simple_type_instances.emplace_back( - ps.construct(file, entity_instance_name, *references_to_resolve, decl, std::nullopt, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index)) - ); - // @todo do we need express::Base here? Or should we just push instance_data? - context.push(simple_type_instance); + if (retain_value) { + auto data = load(tokens, entity_instance_name, simple_declaration, entity, reference_attribute_index); + read_simple_type_instances.push_back(data); + storage.set(attribute_index_within_data, express::Base(data)); + } else { + skip_aggregate(tokens); + } } catch (exception& e) { logger::message(logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(next.start_pos)); - // #4070 We didn't actually capture an aggregate entry, undo length increment. - return_value--; + --values_read; } } else { - context.push(next); + if (next.is_identifier() && entity && entity_instance_name) { + register_inverse((unsigned)*entity_instance_name, entity, next.value_int, reference_attribute_index); + } + if (retain_value) { + dispatch_token_direct(next, declared_type(parameter_type), (int) attribute_index_within_data, [&](const auto& value) { + set_direct_attribute(storage, entity_instance_name, references_to_resolve, attribute_index_within_data, attribute_index, value); + }); + } } } next = tokens->next(); } + + warn_attribute_count(declaration, entity_instance_name, expected_size, values_read); + return ifcopenshell::make_pointer_type(file, declaration, (declaration && declaration->as_entity()) ? (uint32_t)entity_instance_name.value_or(0) : 0, std::move(storage)); } template @@ -640,17 +1011,13 @@ void ifcopenshell::impl::in_memory_file_storage::try_read_semicolon(ifcopenshell void ifcopenshell::impl::in_memory_file_storage::register_inverse(unsigned id_from, const ifcopenshell::entity* from_entity, int inst_id, int attribute_index) { // Assume a check on token type has already been performed - byref_excl_[inst_id][{from_entity->index_in_schema(), attribute_index}].push_back(id_from); + byref_excl_.add((uint32_t)inst_id, (uint32_t)id_from, (uint16_t)from_entity->index_in_schema(), attribute_index); } void ifcopenshell::impl::in_memory_file_storage::unregister_inverse(unsigned id_from, const ifcopenshell::entity* from_entity, const express::Base& inst, int attribute_index) { - auto& ids = byref_excl_[inst.id()][{from_entity->index_in_schema(), attribute_index}]; - auto iter = std::find(ids.begin(), ids.end(), id_from); - if (iter == ids.end()) { + if (!byref_excl_.remove((uint32_t)inst.id(), (uint32_t)id_from, (uint16_t)from_entity->index_in_schema(), attribute_index)) { // @todo inverses also need to be populated when multiple instances are added to a new file. // throw ifcopenshell::exception("Instance not found among inverses"); - } else { - ids.erase(iter); } } @@ -1382,17 +1749,16 @@ void read_terminal(spf_lexer& lexer, const std::string& term, bool trail } template -std::shared_ptr read_header_entity( +shared_pointer_type read_header_entity( ifcopenshell::file* file, ifcopenshell::impl::in_memory_file_storage& storage, spf_lexer& lexer, ifcopenshell::unresolved_references& references_to_resolve, const ifcopenshell::entity& decl) { - parse_context pc; lexer.next(); - storage.load(&lexer, std::nullopt, nullptr, pc, -1); - auto result = pc.construct(file, std::nullopt, references_to_resolve, &decl, decl.attribute_count(), -1); - return result; + storage.file = file; + storage.references_to_resolve = &references_to_resolve; + return storage.load(&lexer, std::nullopt, &decl, nullptr, -1); } template @@ -1480,6 +1846,20 @@ void ifcopenshell::instance_streamer::initialize_header() { } storage_.schema = schema_; + + types_to_bypass_materialized_.resize(schema_->declarations().size(), false); + for (auto& bp : types_to_bypass_) { + std::function mark; + mark = [&](const ifcopenshell::entity* e) { + types_to_bypass_materialized_[e->index_in_schema()] = true; + for (auto& subtype : e->subtypes()) { + mark(subtype); + } + }; + if (auto* e = bp->as_entity()) { + mark(e); + } + } } template @@ -1546,8 +1926,6 @@ ifcopenshell::instance_streamer::instance_streamer(ifcopenshell::file* f , schema_(nullptr) , progress_(0) { - init_locale(); - if constexpr (std::is_same_v>) { owned_stream_ = std::make_unique(caller_fed_tag{}); } else if constexpr (std::is_same_v>) { @@ -1570,10 +1948,9 @@ ifcopenshell::instance_streamer::instance_streamer(const std::string& fn , owner_(f) , token_stream_(3, token{}) , schema_(nullptr) - , progress_(0) { - init_locale(); - - if constexpr (std::is_same_v>) { + , progress_(0) +{ + if constexpr (std::is_same_v>) { (void)mmap; owned_stream_ = std::make_unique(fn); #ifdef USE_MMAP @@ -1600,8 +1977,6 @@ ifcopenshell::instance_streamer::instance_streamer(void* data, int lengt , schema_(nullptr) , progress_(0) { - init_locale(); - if constexpr (std::is_same_v>) { owned_stream_ = std::make_unique(std::string((char*)data, length), caller_fed_tag{}); } else if constexpr (std::is_same_v>) { @@ -1623,9 +1998,8 @@ ifcopenshell::instance_streamer::instance_streamer(Reader* stream, ifcop , owner_(f) , token_stream_(3, token{}) , schema_(nullptr) - , progress_(0) { - init_locale(); - + , progress_(0) +{ lexer_ = std::make_unique>(stream_); good_ = file_open_status::NO_HEADER; initialize_header(); @@ -1643,25 +2017,37 @@ void ifcopenshell::instance_streamer::bypass_types(const std::set -std::optional>> ifcopenshell::instance_streamer::read_instance() { - std::optional>> return_value; +std::optional> ifcopenshell::instance_streamer::read_instance() { + std::optional> return_value; if (yield_header_instances_ && header_ && yielded_header_instances_ < 3) { if (yielded_header_instances_ == 0) { return_value.emplace( 0, &header_->file_description().declaration(), +#ifdef IFOPSH_SAFE_INSTANCE header_->file_description().data_weak().lock()); +#else + header_->file_description().data_weak()); +#endif } else if (yielded_header_instances_ == 1) { return_value.emplace( 0, &header_->file_name().declaration(), +#ifdef IFOPSH_SAFE_INSTANCE header_->file_name().data_weak().lock()); +#else + header_->file_name().data_weak()); +#endif } else if (yielded_header_instances_ == 2) { return_value.emplace( 0, &header_->file_schema().declaration(), +#ifdef IFOPSH_SAFE_INSTANCE header_->file_schema().data_weak().lock()); +#else + header_->file_schema().data_weak()); +#endif } yielded_header_instances_ += 1; return return_value; @@ -1688,36 +2074,31 @@ std::optionalis(*ty)) { - bypassed_instances_.push_back(current_id); - current_id = 0; - goto advance; - } + if (types_to_bypass_materialized_[entity_type->index_in_schema()]) { + bypassed_instances_.push_back(current_id); + current_id = 0; + goto advance; } - parse_context ps; lexer_->next(); try { - storage_.load(lexer_.get(), current_id, entity_type->as_entity(), ps, -1); + auto data = storage_.load(lexer_.get(), current_id, entity_type, entity_type->as_entity(), -1, coerce_attribute_count); + + if (((++progress_) % 1000) == 0) { + std::stringstream ss; + ss << "\r#" << current_id; + logger::status(ss.str(), false); + } + + return_value.emplace( + (size_t)current_id, + entity_type, + data); } catch (const invalid_token_exception& e) { good_ = file_open_status::INVALID_SYNTAX; logger::error(e); break; } - - if (((++progress_) % 1000) == 0) { - std::stringstream ss; - ss << "\r#" << current_id; - logger::status(ss.str(), false); - } - - auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count); - - return_value.emplace( - (size_t)current_id, - entity_type, - data); } advance: token next_token; @@ -1747,8 +2128,6 @@ template class IFC_PARSE_API ifcopenshell::instance_streamer void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass) { - init_locale(); - schema = nullptr; if (!s->size() || s->eof()) { @@ -1831,7 +2210,8 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con } good_ = streamer.status(); - byref_excl_ = streamer.inverses(); + byref_excl_ = std::move(streamer.inverses()); + byref_excl_.sort(); read_simple_type_instances = streamer.steal_instances(); logger::status("\rDone scanning file "); @@ -1861,7 +2241,11 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con express::Base inst = storage->get_attribute_value(attr_index); if (!inst.declaration().as_entity()) { // Probably a case of IfcPropertySetDefinitionSet, divert storage of reference to the simply type instance +#ifdef IFOPSH_SAFE_INSTANCE storage = inst.data_weak().lock(); +#else + storage = inst.data_weak(); +#endif attr_index = 0; } } @@ -1901,7 +2285,11 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con express::Base inst = storage->get_attribute_value(attr_index); if (!inst.declaration().as_entity()) { // Probably a case of IfcPropertySetDefinitionSet, divert storage of reference to the simply type instance +#ifdef IFOPSH_SAFE_INSTANCE storage = inst.data_weak().lock(); +#else + storage = inst.data_weak(); +#endif attr_index = 0; } } @@ -1939,7 +2327,11 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con express::Base inst = storage->get_attribute_value(attr_index); if (!inst.declaration().as_entity()) { // Probably a case of IfcPropertySetDefinitionSet, divert storage of reference to the simply type instance +#ifdef IFOPSH_SAFE_INSTANCE storage = inst.data_weak().lock(); +#else + storage = inst.data_weak(); +#endif attr_index = 0; } } @@ -2362,28 +2754,7 @@ void ifcopenshell::impl::in_memory_file_storage::process_deletion_inverse(const // Delete inverses into entity byref_excl_.erase(id); - - // This is based on traversal which needs instances to still be contained in the map. - // another option would be to keep byid intact for the remainder of this loop - auto entity_attributes = traverse(entity, 1); - for (auto it = entity_attributes.begin(); it != entity_attributes.end(); ++it) { - auto entity_attribute = *it; - if (entity_attribute == entity) { - continue; - } - const unsigned int name = entity_attribute.id(); - // Do not update inverses for simple types (which have id()==0 in IfcOpenShell). - if (name != 0) { - // Find instances entity -> other - // and update inverses from entity into other - auto submap = byref_excl_.find(name); - if (submap != byref_excl_.end()) { - for (auto& [key, ids] : submap->second) { - ids.erase(std::remove(ids.begin(), ids.end(), id), ids.end()); - } - } - } - } + byref_excl_.remove_source(id); } namespace { @@ -2453,14 +2824,10 @@ std::vector file::instances_by_reference(int t) { std::vector ret; std::visit([this, t, &ret](auto& x) { if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - auto submap = x.byref_excl_.find(t); - if (submap == x.byref_excl_.end()) { - return; - } - for (auto& [key, ids] : submap->second) { - for (auto& i : ids) { - ret.push_back(instance_by_id(i)); - } + auto range = x.byref_excl_.equal_range((uint32_t)t); + ret.reserve(ret.size() + (size_t)std::distance(range.first, range.second)); + for (auto it = range.first; it != range.second; ++it) { + ret.push_back(instance_by_id(it->source_id)); } } #ifdef IFOPSH_WITH_ROCKSDB @@ -2611,18 +2978,16 @@ std::vector file::get_inverse_indices_by_id(int instance_id) { // Mapping of instance id to attribute offset. std::map> mapping; + bool handled = false; - std::visit([&mapping, instance_id](const auto& x) { + std::visit([&mapping, &return_value, &handled, instance_id](auto& x) { if constexpr (std::is_same_v, std::monostate>) { } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - auto submap = x.byref_excl_.find(instance_id); - if (submap == x.byref_excl_.end()) { - return; - } - for (auto& [key, ids] : submap->second) { - for (auto& i : ids) { - mapping[i].push_back(std::get<1>(key)); - } + handled = true; + auto range = x.byref_excl_.equal_range((uint32_t)instance_id); + return_value.reserve((size_t)std::distance(range.first, range.second)); + for (auto it = range.first; it != range.second; ++it) { + return_value.push_back(it->attribute_index); } } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { #ifdef IFOPSH_WITH_ROCKSDB @@ -2643,6 +3008,10 @@ std::vector file::get_inverse_indices_by_id(int instance_id) { } }, storage_); + if (handled) { + return return_value; + } + auto refs = instances_by_reference(instance_id); for (const auto& ref : refs) { @@ -2677,29 +3046,19 @@ std::vector file::get_inverse(int instance_id, const ifcopenshe return return_value; } - std::visit([&return_value, this, attribute_index, instance_id, type](const auto& x) { + std::visit([&return_value, this, attribute_index, instance_id, type](auto& x) { if constexpr (std::is_same_v, std::monostate>) { } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - auto submap = x.byref_excl_.find(instance_id); - if (submap != x.byref_excl_.end()) { - visit_subtypes(type->as_entity(), [this, attribute_index, instance_id, &return_value, &submap](const ifcopenshell::declaration* ent) { - if (attribute_index == -1) { - auto lower = submap->second.lower_bound({ent->index_in_schema(), std::numeric_limits::min()}); - auto upper = submap->second.upper_bound({ent->index_in_schema(), std::numeric_limits::max()}); - for (auto it = lower; it != upper; ++it) { - for (auto& i : it->second) { - return_value.push_back(instance_by_id(i).template as()); - } - } - } else { - auto it = submap->second.find({ent->index_in_schema(), attribute_index}); - if (it != submap->second.end()) { - for (auto& i : it->second) { - return_value.push_back(instance_by_id(i).template as()); - } - } - } - }); + std::vector source_types(schema()->declarations().size(), 0); + visit_subtypes(type->as_entity(), [&source_types](const ifcopenshell::declaration* ent) { + source_types[ent->index_in_schema()] = 1; + }); + auto range = x.byref_excl_.equal_range((uint32_t)instance_id); + for (auto it = range.first; it != range.second; ++it) { + if (it->source_entity < source_types.size() && source_types[it->source_entity] && + (attribute_index == -1 || it->attribute_index == attribute_index)) { + return_value.push_back(instance_by_id(it->source_id).template as()); + } } } #ifdef IFOPSH_WITH_ROCKSDB @@ -2737,17 +3096,12 @@ std::vector file::get_inverse(int instance_id, const ifcopenshe size_t file::get_total_inverses(int instance_id) { std::set counted_ids; - std::visit([&counted_ids, instance_id](const auto& x) { + std::visit([&counted_ids, instance_id](auto& x) { if constexpr (std::is_same_v, std::monostate>) { } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - auto submap = x.byref_excl_.find(instance_id); - if (submap == x.byref_excl_.end()) { - return; - } - for (auto& [key, ids] : submap->second) { - for (auto& i : ids) { - counted_ids.insert(i); - } + auto range = x.byref_excl_.equal_range((uint32_t)instance_id); + for (auto it = range.first; it != range.second; ++it) { + counted_ids.insert(it->source_id); } } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { // @todo @@ -2846,7 +3200,7 @@ void ifcopenshell::file::build_inverses_(const express::Base& inst) { std::visit([entity_attribute_id, decl, idx, inst](auto& x) { if constexpr (std::is_same_v, std::monostate>) { } else if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - x.byref_excl_[entity_attribute_id][{decl->index_in_schema(), idx}].push_back(inst.id()); + x.byref_excl_.add(entity_attribute_id, inst.id(), (uint16_t)decl->index_in_schema(), idx); } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { // @todo } diff --git a/src/ifcparse/spf_header.cpp b/src/ifcparse/spf_header.cpp index d3ccff3c4d..4f9b2e0b1c 100644 --- a/src/ifcparse/spf_header.cpp +++ b/src/ifcparse/spf_header.cpp @@ -11,16 +11,16 @@ using namespace ifcopenshell; namespace { -std::shared_ptr make_header_entity(ifcopenshell::file* file, const ifcopenshell::entity& decl) { +shared_pointer_type make_header_entity(ifcopenshell::file* file, const ifcopenshell::entity& decl) { const bool in_memory = file == nullptr || std::visit([](auto& storage) { return std::is_same_v, ifcopenshell::impl::in_memory_file_storage>; }, file->storage_); if (in_memory) { - return std::make_shared(file, &decl, 0, in_memory_attribute_storage(decl.attribute_count())); + return ifcopenshell::make_pointer_type(file, &decl, 0, in_memory_attribute_storage(decl.attribute_count())); } - return std::make_shared(file, &decl, 0, rocks_db_attribute_storage{}); + return ifcopenshell::make_pointer_type(file, &decl, 0, rocks_db_attribute_storage{}); } } // namespace @@ -60,15 +60,15 @@ void ifcopenshell::spf_header::owner_file(ifcopenshell::file* file) { file_ = file; } -void ifcopenshell::spf_header::set_file_description(const std::shared_ptr& data) { +void ifcopenshell::spf_header::set_file_description(const shared_pointer_type& data) { header_entities_[0] = data; } -void ifcopenshell::spf_header::set_file_name(const std::shared_ptr& data) { +void ifcopenshell::spf_header::set_file_name(const shared_pointer_type& data) { header_entities_[1] = data; } -void ifcopenshell::spf_header::set_file_schema(const std::shared_ptr& data) { +void ifcopenshell::spf_header::set_file_schema(const shared_pointer_type& data) { header_entities_[2] = data; } diff --git a/src/ifcparse/spf_header.h b/src/ifcparse/spf_header.h index efd6304553..4c73deff68 100644 --- a/src/ifcparse/spf_header.h +++ b/src/ifcparse/spf_header.h @@ -32,7 +32,7 @@ class IFC_PARSE_API spf_header { private: ifcopenshell::file* file_; - std::array, 3> header_entities_; + std::array header_entities_; public: explicit spf_header(ifcopenshell::file* owner_file); @@ -43,9 +43,9 @@ class IFC_PARSE_API spf_header { ifcopenshell::file* owner_file() { return file_; } void owner_file(ifcopenshell::file* file); - void set_file_description(const std::shared_ptr& description_data); - void set_file_name(const std::shared_ptr& name_data); - void set_file_schema(const std::shared_ptr& schema_data); + void set_file_description(const shared_pointer_type& description_data); + void set_file_name(const shared_pointer_type& name_data); + void set_file_schema(const shared_pointer_type& schema_data); const Header_section_schema::file_description file_description() const; const Header_section_schema::file_name file_name() const; diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h index 50cf35a890..6b339bb8bb 100644 --- a/src/ifcparse/storage.h +++ b/src/ifcparse/storage.h @@ -28,7 +28,11 @@ namespace rocksdb { #include #include +#include +#include #include +#include +#include #include #include #include @@ -37,6 +41,7 @@ namespace rocksdb { #include #include #include +#include #ifndef SWIG @@ -131,7 +136,7 @@ namespace ifcopenshell { }; typedef std::variant reference_or_simple_type; - typedef std::list, std::vector>>>> unresolved_references; + typedef std::vector, std::vector>>>> unresolved_references; class file; template @@ -205,38 +210,206 @@ namespace ifcopenshell { } }; - struct IFC_PARSE_API parse_context { - std::vector< - std::variant< - express::Base, - token, - parse_context* - >> tokens_; - - parse_context() {} - ~parse_context(); - - parse_context(const parse_context& other) = delete; - parse_context& operator=(const parse_context& other) = delete; - - parse_context(parse_context&& other) = default; - parse_context& operator=(parse_context&& other) = default; - - parse_context& push(); - - void push(token next_token); - - void push(const express::Base& instance); - - std::shared_ptr construct(ifcopenshell::file* owner_file, std::optional instance_name, unresolved_references& references_to_resolve, const ifcopenshell::declaration* declaration, std::optional expected_size, int resolve_reference_index, bool coerce_attribute_count = true); - }; - namespace impl { + struct inverse_record { + uint32_t referenced_id; + uint32_t source_id; + uint16_t source_entity; + int16_t attribute_index; + }; + + class inverse_index { + public: + typedef std::map, std::vector> legacy_bucket_t; + typedef std::unordered_map legacy_map_t; + typedef legacy_map_t::key_type key_type; + typedef legacy_map_t::mapped_type mapped_type; + typedef legacy_map_t::value_type value_type; + typedef legacy_map_t::iterator iterator; + typedef legacy_map_t::const_iterator const_iterator; + typedef std::vector::const_iterator record_iterator; + + private: + mutable std::vector records_; + mutable bool sorted_ = true; + mutable std::unique_ptr materialized_; + + static bool record_less(const inverse_record& a, const inverse_record& b) { + if (a.referenced_id != b.referenced_id) { + return a.referenced_id < b.referenced_id; + } + if (a.source_entity != b.source_entity) { + return a.source_entity < b.source_entity; + } + if (a.attribute_index != b.attribute_index) { + return a.attribute_index < b.attribute_index; + } + return a.source_id < b.source_id; + } + + static bool referenced_less(const inverse_record& a, uint32_t referenced_id) { + return a.referenced_id < referenced_id; + } + + static bool referenced_less(uint32_t referenced_id, const inverse_record& a) { + return referenced_id < a.referenced_id; + } + + void invalidate_materialized() const { + materialized_.reset(); + } + + legacy_map_t& materialize() const { + if (!materialized_) { + materialized_ = std::make_unique(); + materialized_->reserve(records_.size()); + for (const auto& record : records_) { + (*materialized_)[(int)record.referenced_id][{(short)record.source_entity, (short)record.attribute_index}].push_back(record.source_id); + } + } + return *materialized_; + } + + public: + inverse_index() = default; + + inverse_index(const inverse_index& other) + : records_(other.records_) + , sorted_(other.sorted_) + {} + + inverse_index& operator=(const inverse_index& other) { + if (this != &other) { + records_ = other.records_; + sorted_ = other.sorted_; + materialized_.reset(); + } + return *this; + } + + inverse_index(inverse_index&&) noexcept = default; + inverse_index& operator=(inverse_index&&) noexcept = default; + + void reserve(size_t size) { + records_.reserve(size); + } + + void add(uint32_t referenced_id, uint32_t source_id, uint16_t source_entity, int attribute_index) { + records_.push_back({referenced_id, source_id, source_entity, (int16_t)attribute_index}); + sorted_ = false; + invalidate_materialized(); + } + + bool remove(uint32_t referenced_id, uint32_t source_id, uint16_t source_entity, int attribute_index) { + const inverse_record needle{referenced_id, source_id, source_entity, (int16_t)attribute_index}; + auto it = std::find_if(records_.begin(), records_.end(), [&needle](const inverse_record& record) { + return record.referenced_id == needle.referenced_id && + record.source_id == needle.source_id && + record.source_entity == needle.source_entity && + record.attribute_index == needle.attribute_index; + }); + if (it == records_.end()) { + return false; + } + records_.erase(it); + invalidate_materialized(); + return true; + } + + void remove_source(uint32_t source_id) { + records_.erase(std::remove_if(records_.begin(), records_.end(), [source_id](const inverse_record& record) { + return record.source_id == source_id; + }), records_.end()); + invalidate_materialized(); + } + + void sort() const { + if (!sorted_) { + std::sort(records_.begin(), records_.end(), record_less); + sorted_ = true; + invalidate_materialized(); + } + } + + std::pair equal_range(uint32_t referenced_id) const { + sort(); + return std::equal_range(records_.begin(), records_.end(), referenced_id, [](const auto& a, const auto& b) { + if constexpr (std::is_same_v, inverse_record>) { + return referenced_less(a, b); + } else { + return referenced_less(a, b); + } + }); + } + + const std::vector& records() const { + sort(); + return records_; + } + + bool empty() const { + return records_.empty(); + } + + size_t size() const { + return records_.size(); + } + + void clear() { + records_.clear(); + sorted_ = true; + materialized_.reset(); + } + + iterator begin() { + return materialize().begin(); + } + + iterator end() { + return materialize().end(); + } + + const_iterator begin() const { + return materialize().begin(); + } + + const_iterator end() const { + return materialize().end(); + } + + iterator find(const key_type& key) { + return materialize().find(key); + } + + const_iterator find(const key_type& key) const { + return materialize().find(key); + } + + size_t erase(const key_type& key) { + const auto old_size = records_.size(); + records_.erase(std::remove_if(records_.begin(), records_.end(), [key](const inverse_record& record) { + return record.referenced_id == (uint32_t)key; + }), records_.end()); + invalidate_materialized(); + return old_size - records_.size(); + } + + std::pair insert(const value_type& value) { + for (const auto& bucket : value.second) { + for (auto source_id : bucket.second) { + add((uint32_t)value.first, source_id, (uint16_t)std::get<0>(bucket.first), std::get<1>(bucket.first)); + } + } + auto it = find(value.first); + return {it, true}; + } + }; + struct IFC_PARSE_API in_memory_file_storage { - std::vector> read_simple_type_instances; - std::vector> steal_instances() { - return read_simple_type_instances; + std::vector read_simple_type_instances; + std::vector steal_instances() { + return std::move(read_simple_type_instances); } // Either one of these needs to be set @@ -246,14 +419,14 @@ namespace ifcopenshell { unresolved_references* references_to_resolve = nullptr; typedef std::map> entities_by_type_t; - typedef boost::unordered_map> entity_instance_by_name_storage_t; - typedef map_transformer)>> entity_instance_by_name_t; - typedef boost::unordered_map> type_instance_by_name_t; + typedef boost::unordered_map entity_instance_by_name_storage_t; + typedef map_transformer> entity_instance_by_name_t; + typedef boost::unordered_map type_instance_by_name_t; typedef std::map entity_instance_by_guid_t; - typedef std::unordered_map, std::vector>> entities_by_ref_t; + typedef inverse_index entities_by_ref_t; typedef entity_instance_by_name_t::iterator iterator; - in_memory_file_storage(ifcopenshell::file* owner_file = nullptr) : file(owner_file), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr& data) { return express::Base(data); }) {}; + in_memory_file_storage(ifcopenshell::file* owner_file = nullptr) : file(owner_file), schema(nullptr), byid_read_(&byid_, [this](const shared_pointer_type& data) { return express::Base(data); }) {}; in_memory_file_storage(const in_memory_file_storage& other) = delete; in_memory_file_storage(const in_memory_file_storage&& other) = delete; @@ -299,7 +472,7 @@ namespace ifcopenshell { entity_instance_by_name_t byid_read_; template - void load(ifcopenshell::spf_lexer* tokens, std::optional entity_instance_name, const ifcopenshell::entity* entity, parse_context& context, int attribute_index = -1); + shared_pointer_type load(ifcopenshell::spf_lexer* tokens, std::optional entity_instance_name, const ifcopenshell::declaration* declaration, const ifcopenshell::entity* entity, int attribute_index = -1, bool coerce_attribute_count = true); template void try_read_semicolon(ifcopenshell::spf_lexer* tokens) const; @@ -353,7 +526,7 @@ namespace ifcopenshell { // to make sure that instance pointer are constant during file lifetime // cache instances because we want stable pointers // @todo this is silly, but we cannot have the same type, this should be just a pointer then on the file side? - typedef std::map> entity_by_iden_cache_t; + typedef std::map entity_by_iden_cache_t; entity_by_iden_cache_t instance_cache_, type_instance_cache_; std::mutex instance_cache_mutex_; diff --git a/src/plugin/plugin.cpp b/src/plugin/plugin.cpp index 288622d4c7..bc4d06e460 100644 --- a/src/plugin/plugin.cpp +++ b/src/plugin/plugin.cpp @@ -54,10 +54,14 @@ namespace { } void plugin_debug(const std::string& message) { +#ifdef IFOPSH_PLUGIN_DEBUG #if defined(_MSC_VER) && defined(_UNICODE) std::wcerr << "[ifcopenshell.plugin] " << message.c_str() << std::endl; #else std::cerr << "[ifcopenshell.plugin] " << message << std::endl; +#endif +#else + static_cast(message); #endif } diff --git a/src/pyodide/demo-app/index.html b/src/pyodide/demo-app/index.html index 53d5462f31..b689287e15 100644 --- a/src/pyodide/demo-app/index.html +++ b/src/pyodide/demo-app/index.html @@ -78,14 +78,11 @@ await micropip.install("typing-extensions"); document.querySelector("#status2").innerHTML = "Loading IfcOpenShell"; - // await micropip.install("wheels/ifcopenshell-0.8.6-cp313-cp313-emscripten_4_0_9_wasm32.whl"); - - await micropip.install("wheels/modular/0.8.6-b1899b1/ifcopenshell-0.8.6+b1899b1-cp313-cp313-pyodide_2025_0_wasm32.whl"); - await micropip.install("wheels/modular/0.8.6-b1899b1/ifcopenshell_parse_schema_ifc4-0.8.6+b1899b1-cp313-cp313-pyodide_2025_0_wasm32.whl"); - await micropip.install("wheels/modular/0.8.6-b1899b1/ifcopenshell_geometry_mapping_ifc4-0.8.6+b1899b1-cp313-cp313-pyodide_2025_0_wasm32.whl"); - await micropip.install("wheels/modular/0.8.6-b1899b1/ifcopenshell_pure_python-0.8.6+b1899b1-py3-none-any.whl"); - await micropip.install("wheels/modular/0.8.6-b1899b1/ifcopenshell_geometry_kernel_cgalsimple-0.8.6+b1899b1-cp313-cp313-pyodide_2025_0_wasm32.whl"); - await micropip.install("wheels/modular/0.8.6-b1899b1/ifcopenshell_geometry_kernel_opencascade-0.8.6+b1899b1-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/modular/0.8.6+424e70a/ifcopenshell-0.8.6+424e70a-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/modular/0.8.6+424e70a/ifcopenshell_parse_schema_ifc4-0.8.6+424e70a-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/modular/0.8.6+424e70a/ifcopenshell_geometry_mapping_ifc4-0.8.6+424e70a-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/modular/0.8.6+424e70a/ifcopenshell_pure_python-0.8.6+424e70a-py3-none-any.whl"); + await micropip.install("wheels/modular/0.8.6+424e70a/ifcopenshell_geometry_kernel_manifold-0.8.6+424e70a-cp313-cp313-pyodide_2025_0_wasm32.whl"); document.body.className = ''; @@ -295,7 +292,7 @@ 'settings': s, 'file_or_filename': ifc, 'exclude': ['IfcSpace', 'IfcOpeningElement'], - 'geometry_library': 'hybrid-cgal-simple-opencascade' + 'geometry_library': 'manifold' }); let last_mesh_id = null; @@ -372,7 +369,7 @@ addObjToScene(ifcopenshell_geom.create_shape.callKwargs({ 'settings': s, 'inst': el, - 'geometry_library': 'hybrid-cgal-simple-opencascade' + 'geometry_library': 'manifold' })); }