From 0494bd9677f557cd7ac282bc9187acd97bd2095e Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Fri, 24 Oct 2025 12:06:59 +0200 Subject: [PATCH] Option to bypass storing types when opening model --- src/ifcconvert/IfcConvert.cpp | 37 ++++++++--- .../ifcopenshell/__init__.py | 21 ++++++- src/ifcopenshell-python/ifcopenshell/file.py | 6 +- ...st_streaming_rocksdb_and_simpletyperefs.py | 27 ++++++++ src/ifcparse/IfcFile.cpp | 20 ++++++ src/ifcparse/IfcFile.h | 33 +++++++++- src/ifcparse/IfcParse.cpp | 61 +++++++++++++------ src/ifcparse/storage.h | 2 +- 8 files changed, 174 insertions(+), 33 deletions(-) diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index 376373006f..c41c1eab80 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -207,7 +207,7 @@ size_t read_filters_from_file(const std::string&, inclusion_filter&, inclusion_t void parse_filter(geom_filter &, const std::vector&); std::vector setup_filters(const std::vector&, const std::string&); -bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap); +bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap, bool bypass_properties=false); // from https://stackoverflow.com/questions/31696328/boost-program-options-using-zero-parameter-options-multiple-times struct verbosity_counter { @@ -965,7 +965,10 @@ int main(int argc, char** argv) { time_t start,end; time(&start); - if (!init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) { + // @nb last argument true -> bypass_properties which are not read by any of the geometry serializers + // XML, RocksDB, IFC are already special-cased above + // SVG requires properties for IfcAnnotation/DRAWING properties + if (!init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap, output_extension != SVG)) { write_log(!quiet); serializer.reset(); IfcUtil::path::delete_file(IfcUtil::path::to_utf8(output_temp_filename)); /**< @todo Windows Unicode support */ @@ -1336,7 +1339,7 @@ void write_log(bool header) { #include -bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap) { +bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap, bool bypass_properties) { time_t start, end; // Prevent IfcFile::Init() prints by setting output to null temporarily @@ -1344,20 +1347,36 @@ bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, time(&start); + bool requires_init = false; + #ifdef WITH_IFCXML if (boost::ends_with(boost::to_lower_copy(filename), ".ifcxml")) { ifc_file = IfcParse::parse_ifcxml(filename); - } else + } else #endif + { + ifc_file = new IfcParse::IfcFile(IfcParse::uninitialized_tag{}); + requires_init = true; + } - { + ifc_file->bypass_type("IfcRelDefinesByProperties"); + ifc_file->bypass_type("IfcPropertySetDefinition"); + ifc_file->bypass_type("IfcProperty"); + ifc_file->bypass_type("IfcMaterialProperties"); + ifc_file->bypass_type("IfcProfileProperties"); + ifc_file->bypass_type("IfcPhysicalQuantity"); + #ifdef USE_MMAP - ifc_file = new IfcParse::IfcFile(filename, mmap); + if (mmap) { + ifc_file->initialize(filename, mmap); + requires_init = false; + } #else - (void)mmap; - ifc_file = new IfcParse::IfcFile(filename); + (void)mmap; #endif - } + if (requires_init) { + ifc_file->initialize(filename); + } if (!ifc_file || !ifc_file->good()) { Logger::Error("Unable to parse input file '" + filename + "'"); diff --git a/src/ifcopenshell-python/ifcopenshell/__init__.py b/src/ifcopenshell-python/ifcopenshell/__init__.py index 006ba7acb8..5e04f006d7 100644 --- a/src/ifcopenshell-python/ifcopenshell/__init__.py +++ b/src/ifcopenshell-python/ifcopenshell/__init__.py @@ -59,7 +59,7 @@ import sys import zipfile import tempfile from pathlib import Path -from typing import Optional, Union, TYPE_CHECKING, Any, overload, Literal +from typing import Optional, Sequence, Union, TYPE_CHECKING, Any, overload, Literal if TYPE_CHECKING: import ifcopenshell.express.schema_class @@ -138,7 +138,12 @@ def open( path: Union[os.PathLike, str], format: Optional[str] = None, *, should_stream: bool = False, readonly: bool = False ) -> Union[_file, sqlite, _stream]: ... def open( - path: Union[os.PathLike, str], format: Optional[str] = None, should_stream: bool = False, readonly: bool = False + path: Union[os.PathLike, str], + format: Optional[str] = None, + should_stream: bool = False, + readonly: bool = False, + mmap: bool = False, + bypass_types: Optional[Sequence[str]] = None, ) -> Union[_file, sqlite, _stream]: """Loads an IFC dataset from a filepath @@ -186,7 +191,17 @@ def open( if should_stream: return stream(path) if readonly: # Temporary conditional see #7131. Remove once newer builds don't segfault on Linux. - f = ifcopenshell_wrapper.open(str(path.absolute()), readonly) + f = ifcopenshell_wrapper.open(str(path.absolute()), readonly=readonly) + elif bypass_types: + f = ifcopenshell_wrapper.file(ifcopenshell_wrapper.uninitialized_tag()) + for ty in bypass_types: + f.bypass_type(ty) + if mmap: + f.initialize(str(path.absolute()), mmap=mmap) + else: + f.initialize(str(path.absolute())) + elif mmap: + f = ifcopenshell_wrapper.open(str(path.absolute()), mmap=mmap) else: f = ifcopenshell_wrapper.open(str(path.absolute())) return file(f) diff --git a/src/ifcopenshell-python/ifcopenshell/file.py b/src/ifcopenshell-python/ifcopenshell/file.py index a93c9b5dea..e2784f9b49 100644 --- a/src/ifcopenshell-python/ifcopenshell/file.py +++ b/src/ifcopenshell-python/ifcopenshell/file.py @@ -248,6 +248,7 @@ READ_ERROR = ifcopenshell_wrapper.file_open_status.READ_ERROR NO_HEADER = ifcopenshell_wrapper.file_open_status.NO_HEADER UNSUPPORTED_SCHEMA = ifcopenshell_wrapper.file_open_status.UNSUPPORTED_SCHEMA INVALID_SYNTAX = ifcopenshell_wrapper.file_open_status.INVALID_SYNTAX +UNKNOWN = ifcopenshell_wrapper.file_open_status.UNKNOWN import struct @@ -586,8 +587,11 @@ class file: "Unsupported schema: %s" % ",".join(self.header.file_schema.schema_identifiers), ), INVALID_SYNTAX: lambda: (Error, "Syntax error during parse, check logs"), + # This is the case when passing uninitialized_tag + UNKNOWN: lambda: (None, None), }[f.good().value()]() - raise exc(msg) + if exc is not None: + raise exc(msg) else: args = filter(None, [schema]) args = map(ifcopenshell_wrapper.schema_by_name, args) diff --git a/src/ifcopenshell-python/test/test_streaming_rocksdb_and_simpletyperefs.py b/src/ifcopenshell-python/test/test_streaming_rocksdb_and_simpletyperefs.py index 59da95808d..0462ca954b 100644 --- a/src/ifcopenshell-python/test/test_streaming_rocksdb_and_simpletyperefs.py +++ b/src/ifcopenshell-python/test/test_streaming_rocksdb_and_simpletyperefs.py @@ -23,6 +23,11 @@ import pytest import ifcopenshell import tempfile +try: + import psutil +except ImportError: + psutil = None + fn = os.path.join(os.path.dirname(__file__), "fixtures/ColumnPSetsOfSets.ifc") @@ -32,18 +37,40 @@ def test_stream(): "value": ({"ref": 136}, {"ref": 138}), } + def test_chunked_stream(): assert list(ifcopenshell.stream2(fn)) == list(ifcopenshell.stream2(fn, page_size=1024)) + def test_mmaped_stream(): assert list(ifcopenshell.stream2(fn)) == list(ifcopenshell.stream2(fn, mmap=True)) + def test_file(): f = ifcopenshell.open(fn) assert f[139].RelatingPropertyDefinition.is_a("IfcPropertySetDefinitionSet") assert {x.id() for x in f[139].RelatingPropertyDefinition[0]} == {136, 138} +def test_partial_open(): + f = ifcopenshell.open(fn) + assert len(f.by_type("ifccartesianpoint")) + f = ifcopenshell.open(fn, bypass_types=("IfcRepresentationItem",)) + assert len(f.by_type("ifccartesianpoint")) == 0 + + +@pytest.mark.skipif(psutil is None, reason="psutil not installed") +def test_memusage_partial_open(): + m0 = psutil.Process().memory_info().rss + f = ifcopenshell.open(fn) + m1 = psutil.Process().memory_info().rss + g = ifcopenshell.open(fn, bypass_types=("IfcRepresentationItem",)) + m2 = psutil.Process().memory_info().rss + # arbitrary... + expected_ratio = 0.75 + assert (m2 - m1) < (m1 - m0) * expected_ratio + + def test_rocks(): with tempfile.TemporaryDirectory() as d: rfn = os.path.join(d, os.path.basename(fn)) diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index c5afdefc71..93525f3bcf 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -651,6 +651,17 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) { } } +void IfcParse::InstanceStreamer::bypassTypes(const std::set& type_names) { + for (auto& name : type_names) { + try { + types_to_bypass_.push_back(schema_->declaration_by_name(name)); + } catch (const IfcException&) { + continue; + } + } + } + + std::optional> IfcParse::InstanceStreamer::readInstance() { std::optional> return_value; @@ -698,6 +709,15 @@ std::optionalis(*ty)) { + bypassed_instances_.push_back(current_id); + // Why is this a conditional clause in the loop? + current_id = 0; + goto advance; + } + } + parse_context ps; lexer_->Next(); try { diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index 4ff9cfad93..4adde443b3 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -98,8 +98,10 @@ private: int progress_; IfcParse::unresolved_references references_to_resolve_; int yielded_header_instances_ = 0; + std::vector types_to_bypass_; + std::vector bypassed_instances_; -public: + public: bool coerce_attribute_count = true; operator bool() const { @@ -118,6 +120,11 @@ public: return references_to_resolve_; } + const std::vector& bypassed_instances() { + std::sort(bypassed_instances_.begin(), bypassed_instances_.end()); + return bypassed_instances_; + } + const IfcParse::impl::in_memory_file_storage::entities_by_ref_t& inverses() const { return storage_.byref_excl_; } @@ -142,6 +149,8 @@ public: InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer); + void bypassTypes(const std::set& type_names); + ~InstanceStreamer() { delete stream_; if (stream_) { @@ -153,6 +162,9 @@ public: std::optional> readInstance(); }; +class uninitialized_tag {}; + + /// This class provides access to the entity instances in an IFC file /// The file takes ownership of instances added to this file and deletes them when the file is deleted. class IFC_PARSE_API IfcFile { @@ -179,7 +191,10 @@ public: // @todo temporarily public for header storage_t storage_; -private: + + std::set types_to_bypass_loading_; + + private: file_open_status good_ = file_open_status::SUCCESS; const IfcParse::schema_definition* schema_; @@ -246,6 +261,20 @@ private: /// The file system path to the IFC file. Defaults to an empty string. IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = FT_AUTODETECT, const std::string& path = ""); + /// + /// Constructs an unitialized IfcFile object. Call initialize() later on. Allows to specify which types to bypass during load. + /// + IfcFile(const uninitialized_tag&); + + bool initialize(const std::string& path, filetype ty = FT_AUTODETECT, bool readonly = false); +#ifdef USE_MMAP + bool initialize(const std::string& path, bool mmap); +#endif + + /// @brief Bypass loading of all instances of the specified type name. Only applies to parsed IFC-SPF files. + /// @param type_name case insensitive name of the type to bypass + void bypass_type(const std::string& type_name); + ~IfcFile(); IfcParse::file_open_status good() const { return good_; } diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index cbad1c7a21..09816a10c2 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -1178,15 +1178,19 @@ IfcUtil::IfcBaseClass::set_attribute_value(const std::string& s, const T& t) { // #ifdef USE_MMAP IfcFile::IfcFile(const std::string& fn, bool mmap) { - std::unique_ptr s; + initialize(fn, mmap); +} + +bool IfcParse::IfcFile::initialize(const std::string& fn, bool mmap) { + std::unique_ptr s; if (mmap) { s = std::make_unique(fn, FileReader::mmap_tag{}); } else { s = std::make_unique(fn); - } + } storage_.emplace<1>(this); - std::get(storage_).read_from_stream(&*s, schema_, max_id_); + std::get(storage_).read_from_stream(&*s, schema_, max_id_, types_to_bypass_loading_); if ((good_ = std::get(storage_).good_)) { // @todo unify these names, it's already confusing enough as it stands @@ -1199,25 +1203,23 @@ IfcFile::IfcFile(const std::string& fn, bool mmap) { } #endif -IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly) - : schema_(nullptr) - , max_id_(0) - , _header(this) -{ - // @todo allow for rocksdb from path +IfcFile::IfcFile(const uninitialized_tag&) + : schema_(nullptr), max_id_(0), _header(this), good_(file_open_status::UNKNOWN), ifcroot_type_(nullptr) {} + +bool IfcParse::IfcFile::initialize(const std::string& path, filetype ty, bool readonly) { if (ty == FT_AUTODETECT) { ty = guess_file_type(path); } if (ty == FT_IFCSPF) { FileReader s(path); storage_.emplace<1>(this); - std::get(storage_).read_from_stream(&s, schema_, max_id_); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); if ((good_ = std::get(storage_).good_)) { // @todo unify these names, it's already confusing enough as it stands byid_ = decltype(byid_)(&std::get(storage_).byid_); byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); - byguid_ = decltype(byguid_)(&std::get(storage_).byguid_); + byguid_ = decltype(byguid_)(&std::get(storage_).byguid_); } // byidentity_ = decltype(byidentity_)(&std::get(storage_).byidentity_); } else if (ty == FT_ROCKSDB) { @@ -1246,6 +1248,19 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly) // throw std::runtime_error("Unsupported file format"); } ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; + return good_ == file_open_status::SUCCESS; +} + +void IfcParse::IfcFile::bypass_type(const std::string& type_name) { + types_to_bypass_loading_.insert(type_name); +} + +IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly) + : schema_(nullptr) + , max_id_(0) + , _header(this) +{ + initialize(path, ty, readonly); } IfcFile::IfcFile(std::istream& stream, int length) @@ -1260,7 +1275,7 @@ IfcFile::IfcFile(std::istream& stream, int length) s.pushNextPage(string_data); storage_.emplace<1>(this); - std::get(storage_).read_from_stream(&s, schema_, max_id_); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); good_ = std::get(storage_).good_; ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; @@ -1276,7 +1291,7 @@ IfcFile::IfcFile(void* data, int length) FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{}); storage_.emplace<1>(this); - std::get(storage_).read_from_stream(&s, schema_, max_id_); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); good_ = std::get(storage_).good_; ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; @@ -1290,7 +1305,7 @@ IfcFile::IfcFile(IfcParse::FileReader* s) , max_id_(0) { storage_.emplace<1>(this); - std::get(storage_).read_from_stream(s, schema_, max_id_); + std::get(storage_).read_from_stream(s, schema_, max_id_, types_to_bypass_loading_); good_ = std::get(storage_).good_; ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; @@ -1330,8 +1345,7 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s setDefaultHeaderValues(); } -bool IfcParse::InstanceStreamer::hasSemicolon() const -{ +bool IfcParse::InstanceStreamer::hasSemicolon() const { auto local_stream = stream_->clone(); auto local_lexer = IfcSpfLexer(&local_stream); Token t; @@ -1452,7 +1466,7 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* storage_.references_to_resolve = &references_to_resolve_; } -void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) { +void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass) { // Initialize a "C" locale for locale-independent // number parsing. See comment above on line 41. init_locale(); @@ -1499,6 +1513,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead auto ifcroot_type_ = schema->declaration_by_name("IfcRoot"); InstanceStreamer streamer(schema, tokens); + streamer.bypassTypes(typed_to_bypass); Logger::Status("Scanning file..."); @@ -1510,6 +1525,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead // No more instances to read break; } + auto current_id = std::get<0>(*inst); auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst))); @@ -1569,11 +1585,16 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead return; } + const auto& bypassed = streamer.bypassed_instances(); + for (const auto& p : streamer.references()) { const auto& ref = p.first.name_; const auto& refattr = p.first.index_; if (auto* v = std::get_if(&p.second)) { if (auto* name = std::get_if(v)) { + if (std::binary_search(bypassed.begin(), bypassed.end(), *name)) { + continue; + } auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset)); @@ -1604,6 +1625,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead instances->reserve(vv->size()); for (const auto& vi : *vv) { if (auto* name = std::get_if(&vi)) { + if (std::binary_search(bypassed.begin(), bypassed.end(), *name)) { + continue; + } auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset)); @@ -1638,6 +1662,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead std::vector inner; for (const auto& vii : vi) { if (auto* name = std::get_if(&vii)) { + if (std::binary_search(bypassed.begin(), bypassed.end(), *name)) { + continue; + } auto it = byid_.find(*name); if (it == byid_.end()) { Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset)); diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h index 5729d54a39..60378c982e 100644 --- a/src/ifcparse/storage.h +++ b/src/ifcparse/storage.h @@ -268,7 +268,7 @@ namespace IfcParse { // @todo is this still used IfcEntityInstanceData read(unsigned int index); - void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id); + void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); file_open_status good_ = file_open_status::SUCCESS;