Option to bypass storing types when opening model

src/ifcconvert/IfcConvert.cpp

@@ -207,7 +207,7 @@ size_t read_filters_from_file(const std::string&, inclusion_filter&, inclusion_t
 void parse_filter(geom_filter &, const std::vector<std::string>&);
 std::vector<IfcGeom::filter_t> setup_filters(const std::vector<geom_filter>&, const std::string&);
 
-bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap);
+bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap, bool bypass_properties=false);
 
 // from https://stackoverflow.com/questions/31696328/boost-program-options-using-zero-parameter-options-multiple-times
 struct verbosity_counter {
@@ -965,7 +965,10 @@ int main(int argc, char** argv) {
 	time_t start,end;
 	time(&start);
 	
-    if (!init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) {
+	// @nb last argument true -> bypass_properties which are not read by any of the geometry serializers
+    // XML, RocksDB, IFC are already special-cased above
+    // SVG requires properties for IfcAnnotation/DRAWING properties
+    if (!init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap, output_extension != SVG)) {
         write_log(!quiet);
 		serializer.reset();
         IfcUtil::path::delete_file(IfcUtil::path::to_utf8(output_temp_filename)); /**< @todo Windows Unicode support */
@@ -1336,7 +1339,7 @@ void write_log(bool header) {
 
 #include <boost/algorithm/string/predicate.hpp>
 
-bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap) {
+bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file, bool no_progress, bool mmap, bool bypass_properties) {
     time_t start, end;
 
     // Prevent IfcFile::Init() prints by setting output to null temporarily
@@ -1344,20 +1347,36 @@ bool init_input_file(const std::string& filename, IfcParse::IfcFile*& ifc_file,
 
     time(&start);
 
+	bool requires_init = false;
+
 #ifdef WITH_IFCXML
 	if (boost::ends_with(boost::to_lower_copy(filename), ".ifcxml")) {
 		ifc_file = IfcParse::parse_ifcxml(filename);
-	} else
+    } else
 #endif
+    {
+        ifc_file = new IfcParse::IfcFile(IfcParse::uninitialized_tag{});
+        requires_init = true;
+    }
 
-	{
+	ifc_file->bypass_type("IfcRelDefinesByProperties");
+    ifc_file->bypass_type("IfcPropertySetDefinition");
+    ifc_file->bypass_type("IfcProperty");
+    ifc_file->bypass_type("IfcMaterialProperties");
+    ifc_file->bypass_type("IfcProfileProperties");
+    ifc_file->bypass_type("IfcPhysicalQuantity");
+    
 #ifdef USE_MMAP
-		ifc_file = new IfcParse::IfcFile(filename, mmap);
+    if (mmap) {
+        ifc_file->initialize(filename, mmap);
+        requires_init = false;
+    }
 #else
-		(void)mmap;
-		ifc_file = new IfcParse::IfcFile(filename);
+    (void)mmap;
 #endif
-	}
+    if (requires_init) {
+        ifc_file->initialize(filename);
+    }
 
 	if (!ifc_file || !ifc_file->good()) {
         Logger::Error("Unable to parse input file '" + filename + "'");

src/ifcopenshell-python/ifcopenshell/__init__.py

@@ -59,7 +59,7 @@ import sys
 import zipfile
 import tempfile
 from pathlib import Path
-from typing import Optional, Union, TYPE_CHECKING, Any, overload, Literal
+from typing import Optional, Sequence, Union, TYPE_CHECKING, Any, overload, Literal
 
 if TYPE_CHECKING:
     import ifcopenshell.express.schema_class
@@ -138,7 +138,12 @@ def open(
     path: Union[os.PathLike, str], format: Optional[str] = None, *, should_stream: bool = False, readonly: bool = False
 ) -> Union[_file, sqlite, _stream]: ...
 def open(
-    path: Union[os.PathLike, str], format: Optional[str] = None, should_stream: bool = False, readonly: bool = False
+    path: Union[os.PathLike, str],
+    format: Optional[str] = None,
+    should_stream: bool = False,
+    readonly: bool = False,
+    mmap: bool = False,
+    bypass_types: Optional[Sequence[str]] = None,
 ) -> Union[_file, sqlite, _stream]:
     """Loads an IFC dataset from a filepath
 
@@ -186,7 +191,17 @@ def open(
     if should_stream:
         return stream(path)
     if readonly:  # Temporary conditional see #7131. Remove once newer builds don't segfault on Linux.
-        f = ifcopenshell_wrapper.open(str(path.absolute()), readonly)
+        f = ifcopenshell_wrapper.open(str(path.absolute()), readonly=readonly)
+    elif bypass_types:
+        f = ifcopenshell_wrapper.file(ifcopenshell_wrapper.uninitialized_tag())
+        for ty in bypass_types:
+            f.bypass_type(ty)
+        if mmap:
+            f.initialize(str(path.absolute()), mmap=mmap)
+        else:
+            f.initialize(str(path.absolute()))
+    elif mmap:
+        f = ifcopenshell_wrapper.open(str(path.absolute()), mmap=mmap)
     else:
         f = ifcopenshell_wrapper.open(str(path.absolute()))
     return file(f)

src/ifcopenshell-python/ifcopenshell/file.py

@@ -248,6 +248,7 @@ READ_ERROR = ifcopenshell_wrapper.file_open_status.READ_ERROR
 NO_HEADER = ifcopenshell_wrapper.file_open_status.NO_HEADER
 UNSUPPORTED_SCHEMA = ifcopenshell_wrapper.file_open_status.UNSUPPORTED_SCHEMA
 INVALID_SYNTAX = ifcopenshell_wrapper.file_open_status.INVALID_SYNTAX
+UNKNOWN = ifcopenshell_wrapper.file_open_status.UNKNOWN
 
 import struct
 
@@ -586,8 +587,11 @@ class file:
                         "Unsupported schema: %s" % ",".join(self.header.file_schema.schema_identifiers),
                     ),
                     INVALID_SYNTAX: lambda: (Error, "Syntax error during parse, check logs"),
+                    # This is the case when passing uninitialized_tag
+                    UNKNOWN: lambda: (None, None),
                 }[f.good().value()]()
-                raise exc(msg)
+                if exc is not None:
+                    raise exc(msg)
         else:
             args = filter(None, [schema])
             args = map(ifcopenshell_wrapper.schema_by_name, args)

src/ifcopenshell-python/test/test_streaming_rocksdb_and_simpletyperefs.py

@@ -23,6 +23,11 @@ import pytest
 import ifcopenshell
 import tempfile
 
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
 fn = os.path.join(os.path.dirname(__file__), "fixtures/ColumnPSetsOfSets.ifc")
 
 
@@ -32,18 +37,40 @@ def test_stream():
         "value": ({"ref": 136}, {"ref": 138}),
     }
 
+
 def test_chunked_stream():
     assert list(ifcopenshell.stream2(fn)) == list(ifcopenshell.stream2(fn, page_size=1024))
 
+
 def test_mmaped_stream():
     assert list(ifcopenshell.stream2(fn)) == list(ifcopenshell.stream2(fn, mmap=True))
 
+
 def test_file():
     f = ifcopenshell.open(fn)
     assert f[139].RelatingPropertyDefinition.is_a("IfcPropertySetDefinitionSet")
     assert {x.id() for x in f[139].RelatingPropertyDefinition[0]} == {136, 138}
 
 
+def test_partial_open():
+    f = ifcopenshell.open(fn)
+    assert len(f.by_type("ifccartesianpoint"))
+    f = ifcopenshell.open(fn, bypass_types=("IfcRepresentationItem",))
+    assert len(f.by_type("ifccartesianpoint")) == 0
+
+
+@pytest.mark.skipif(psutil is None, reason="psutil not installed")
+def test_memusage_partial_open():
+    m0 = psutil.Process().memory_info().rss
+    f = ifcopenshell.open(fn)
+    m1 = psutil.Process().memory_info().rss
+    g = ifcopenshell.open(fn, bypass_types=("IfcRepresentationItem",))
+    m2 = psutil.Process().memory_info().rss
+    # arbitrary...
+    expected_ratio = 0.75
+    assert (m2 - m1) < (m1 - m0) * expected_ratio
+
+
 def test_rocks():
     with tempfile.TemporaryDirectory() as d:
         rfn = os.path.join(d, os.path.basename(fn))

src/ifcparse/IfcFile.cpp

@@ -651,6 +651,17 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
     }
 }
 
+void IfcParse::InstanceStreamer::bypassTypes(const std::set<std::string>& type_names) {
+    for (auto& name : type_names) {
+        try {
+            types_to_bypass_.push_back(schema_->declaration_by_name(name));
+        } catch (const IfcException&) {
+            continue;
+        }
+    }
+ }
+
+
 std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> IfcParse::InstanceStreamer::readInstance() {
     std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> return_value;
 
@@ -698,6 +709,15 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstance
                 goto advance;
             }
 
+            for (auto& ty : types_to_bypass_) {
+                if (entity_type->is(*ty)) {
+                    bypassed_instances_.push_back(current_id);
+                    // Why is this a conditional clause in the loop?
+                    current_id = 0;
+                    goto advance;
+                }
+            }
+
             parse_context ps;
             lexer_->Next();
             try {

src/ifcparse/IfcFile.h

@@ -98,8 +98,10 @@ private:
     int progress_;
     IfcParse::unresolved_references references_to_resolve_;
     int yielded_header_instances_ = 0;
+    std::vector<const declaration*> types_to_bypass_;
+    std::vector<unsigned> bypassed_instances_;
 
-public:
+  public:
 	bool coerce_attribute_count = true;
 
     operator bool() const {
@@ -118,6 +120,11 @@ public:
         return references_to_resolve_;
     }
 
+    const std::vector<unsigned>& bypassed_instances() {
+        std::sort(bypassed_instances_.begin(), bypassed_instances_.end());
+        return bypassed_instances_;
+    }
+
     const IfcParse::impl::in_memory_file_storage::entities_by_ref_t& inverses() const {
         return storage_.byref_excl_;
     }
@@ -142,6 +149,8 @@ public:
 
     InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer);
 
+    void bypassTypes(const std::set<std::string>& type_names);
+
     ~InstanceStreamer() {
         delete stream_;
         if (stream_) {
@@ -153,6 +162,9 @@ public:
     std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> readInstance();
 };
 
+class uninitialized_tag {};
+
+
 /// This class provides access to the entity instances in an IFC file
 /// The file takes ownership of instances added to this file and deletes them when the file is deleted.
 class IFC_PARSE_API IfcFile {
@@ -179,7 +191,10 @@ public:
 
     // @todo temporarily public for header
     storage_t storage_;
-private:
+
+    std::set<std::string> types_to_bypass_loading_;
+
+  private:
     file_open_status good_ = file_open_status::SUCCESS;
 
     const IfcParse::schema_definition* schema_;
@@ -246,6 +261,20 @@ private:
     /// <param name="path">The file system path to the IFC file. Defaults to an empty string.</param>
     IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = FT_AUTODETECT, const std::string& path = "");
 
+    /// <summary>
+    /// Constructs an unitialized IfcFile object. Call initialize() later on. Allows to specify which types to bypass during load.
+    /// </summary>
+    IfcFile(const uninitialized_tag&);
+
+    bool initialize(const std::string& path, filetype ty = FT_AUTODETECT, bool readonly = false);
+#ifdef USE_MMAP
+    bool initialize(const std::string& path, bool mmap);
+#endif
+
+    /// @brief Bypass loading of all instances of the specified type name. Only applies to parsed IFC-SPF files.
+    /// @param type_name case insensitive name of the type to bypass
+    void bypass_type(const std::string& type_name);
+
     ~IfcFile();
 
     IfcParse::file_open_status good() const { return good_; }

src/ifcparse/IfcParse.cpp

@@ -1178,15 +1178,19 @@ IfcUtil::IfcBaseClass::set_attribute_value(const std::string& s, const T& t) {
 //
 #ifdef USE_MMAP
 IfcFile::IfcFile(const std::string& fn, bool mmap) {
-	std::unique_ptr<FileReader> s;
+    initialize(fn, mmap);
+}
+
+bool IfcParse::IfcFile::initialize(const std::string& fn, bool mmap) {
+    std::unique_ptr<FileReader> s;
     if (mmap) {
         s = std::make_unique<FileReader>(fn, FileReader::mmap_tag{});
     } else {
         s = std::make_unique<FileReader>(fn);
-	}   
+    }
 
     storage_.emplace<1>(this);
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&*s, schema_, max_id_);
+    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&*s, schema_, max_id_, types_to_bypass_loading_);
 
     if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
         // @todo unify these names, it's already confusing enough as it stands
@@ -1199,25 +1203,23 @@ IfcFile::IfcFile(const std::string& fn, bool mmap) {
 }
 #endif
 
-IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
-    : schema_(nullptr)
-    , max_id_(0)
-    , _header(this)
-{
-    // @todo allow for rocksdb from path
+IfcFile::IfcFile(const uninitialized_tag&)
+    : schema_(nullptr), max_id_(0), _header(this), good_(file_open_status::UNKNOWN), ifcroot_type_(nullptr) {}
+
+bool IfcParse::IfcFile::initialize(const std::string& path, filetype ty, bool readonly) {
     if (ty == FT_AUTODETECT) {
         ty = guess_file_type(path);
     }
     if (ty == FT_IFCSPF) {
         FileReader s(path);
         storage_.emplace<1>(this);
-        std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
+        std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
 
         if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
             // @todo unify these names, it's already confusing enough as it stands
             byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
             byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
-			byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
+            byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
         }
         // byidentity_ = decltype(byidentity_)(&std::get<impl::in_memory_file_storage>(storage_).byidentity_);
     } else if (ty == FT_ROCKSDB) {
@@ -1246,6 +1248,19 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
         // throw std::runtime_error("Unsupported file format");
     }
     ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
+    return good_ == file_open_status::SUCCESS;
+}
+
+void IfcParse::IfcFile::bypass_type(const std::string& type_name) {
+    types_to_bypass_loading_.insert(type_name);
+}
+
+IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
+    : schema_(nullptr)
+    , max_id_(0)
+    , _header(this)
+{
+    initialize(path, ty, readonly);
 }
 
 IfcFile::IfcFile(std::istream& stream, int length)
@@ -1260,7 +1275,7 @@ IfcFile::IfcFile(std::istream& stream, int length)
     s.pushNextPage(string_data);
 
     storage_.emplace<1>(this);
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
+    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
     good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
     ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
 
@@ -1276,7 +1291,7 @@ IfcFile::IfcFile(void* data, int length)
 	FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{});
     
     storage_.emplace<1>(this);
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
+    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
     good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
     ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
 
@@ -1290,7 +1305,7 @@ IfcFile::IfcFile(IfcParse::FileReader* s)
     , max_id_(0)
 {
     storage_.emplace<1>(this);
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(s, schema_, max_id_);
+    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(s, schema_, max_id_, types_to_bypass_loading_);
     good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
     ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
 
@@ -1330,8 +1345,7 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
     setDefaultHeaderValues();
 }
 
-bool IfcParse::InstanceStreamer::hasSemicolon() const
-{
+bool IfcParse::InstanceStreamer::hasSemicolon() const {
     auto local_stream = stream_->clone();
 	auto local_lexer = IfcSpfLexer(&local_stream);
     Token t;
@@ -1452,7 +1466,7 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition*
     storage_.references_to_resolve = &references_to_resolve_;
 }
 
-void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) {
+void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass) {
     // Initialize a "C" locale for locale-independent
     // number parsing. See comment above on line 41.
     init_locale();
@@ -1499,6 +1513,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
     auto ifcroot_type_ = schema->declaration_by_name("IfcRoot");
 
 	InstanceStreamer streamer(schema, tokens);
+    streamer.bypassTypes(typed_to_bypass);
 
     Logger::Status("Scanning file...");
 
@@ -1510,6 +1525,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
             // No more instances to read
             break;
 		}
+
         auto current_id = std::get<0>(*inst);
 
         auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst)));
@@ -1569,11 +1585,16 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
         return;
     }
 
+    const auto& bypassed = streamer.bypassed_instances();
+
     for (const auto& p : streamer.references()) {
         const auto& ref = p.first.name_;
         const auto& refattr = p.first.index_;
         if (auto* v = std::get_if<reference_or_simple_type>(&p.second)) {
             if (auto* name = std::get_if<InstanceReference>(v)) {
+                if (std::binary_search(bypassed.begin(), bypassed.end(), *name)) {
+                    continue;
+                }
                 auto it = byid_.find(*name);
                 if (it == byid_.end()) {
                     Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset));
@@ -1604,6 +1625,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
             instances->reserve(vv->size());
             for (const auto& vi : *vv) {
                 if (auto* name = std::get_if<InstanceReference>(&vi)) {
+                    if (std::binary_search(bypassed.begin(), bypassed.end(), *name)) {
+                        continue;
+                    }
                     auto it = byid_.find(*name);
                     if (it == byid_.end()) {
                         Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset));
@@ -1638,6 +1662,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
                 std::vector<IfcUtil::IfcBaseClass*> inner;
                 for (const auto& vii : vi) {
                     if (auto* name = std::get_if<InstanceReference>(&vii)) {
+                        if (std::binary_search(bypassed.begin(), bypassed.end(), *name)) {
+                            continue;
+                        }
                         auto it = byid_.find(*name);
                         if (it == byid_.end()) {
                             Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset));

src/ifcparse/storage.h

@@ -268,7 +268,7 @@ namespace IfcParse {
 
             // @todo is this still used
             IfcEntityInstanceData read(unsigned int index);
-            void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
+            void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
 
             file_open_status good_ = file_open_status::SUCCESS;