diff --git a/src/ifcopenshell-python/ifcopenshell/__init__.py b/src/ifcopenshell-python/ifcopenshell/__init__.py
index b0f52f654d..4b5ede368d 100644
--- a/src/ifcopenshell-python/ifcopenshell/__init__.py
+++ b/src/ifcopenshell-python/ifcopenshell/__init__.py
@@ -195,12 +195,19 @@ def open(
mmap: bool = False,
bypass_types: Optional[Sequence[str]] = None,
logger: Optional[logger] = None,
+ lazy: bool = False,
) -> Union[file, sqlite, _stream]:
"""Loads an IFC dataset from a filepath
:param should_stream: Whether to open the file in streaming mode. Could be useful
for reading large files.
:param logger: Logger that receives native parser messages.
+ :param lazy: Index the file with one quick pass and parse each instance's
+ attributes only when they are first read. Opening is then faster and
+ memory stays proportional to what is accessed; reading every attribute
+ of every instance costs about the same as a normal open, spread over
+ the reads. Falls back to a normal open if the file uses syntax the
+ index pass does not handle.
You can specify a file format. If no format is given, it is guessed from
its extension.
@@ -242,10 +249,12 @@ def open(
return stream(path)
if readonly: # Temporary conditional see #7131. Remove once newer builds don't segfault on Linux.
f = ifcopenshell_wrapper.open(str(path.absolute()), readonly, *optional_logger_args(logger))
- elif bypass_types:
+ elif bypass_types or lazy:
f = ifcopenshell_wrapper.file.create_uninitialized(*optional_logger_args(logger))
- for ty in bypass_types:
+ for ty in bypass_types or ():
f.bypass_type(ty)
+ if lazy:
+ f.lazy_loading(True)
if mmap:
# mmap parameter is only available for builds with USE_MMAP, not used in our main builds
f.initialize(str(path.absolute()), mmap=mmap) # ty: ignore[unknown-argument]
diff --git a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi
index 446c4b7448..e50b76ac7c 100644
--- a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi
+++ b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi
@@ -274,6 +274,7 @@ class geometry_serializer:
class instance_streamer:
def __init__(self, *args): ...
def bypass_types(self, type_names): ...
+ def resolve_references_in_place(self, value): ...
def bypassed_instances(self): ...
coerce_attribute_count: bool
def has_semicolon(self): ...
@@ -924,6 +925,9 @@ class file(file_mixin):
...
def get_max_id(self) -> int: ...
+ def lazy_loading(self, *args: bool) -> bool:
+ """Get, or with an argument set, whether ``initialize()`` indexes the file with one pass and parses each instance's attributes on first access. Set before ``initialize()``."""
+ ...
def get_inverse_indices_by_id(self, instance_id: int) -> tuple[int, ...]: ...
def _get_inverse(self, e: entity_instance) -> tuple[entity_instance, ...]: ...
def _get_inverse_indices(self, *args: Union[entity_instance, int]) -> tuple[int, ...]:
diff --git a/src/ifcopenshell-python/test/test_lazy.py b/src/ifcopenshell-python/test/test_lazy.py
new file mode 100644
index 0000000000..58cf31e37c
--- /dev/null
+++ b/src/ifcopenshell-python/test/test_lazy.py
@@ -0,0 +1,89 @@
+# IfcOpenShell - IFC toolkit and geometry engine
+# Copyright (C) 2026 IfcOpenShell contributors
+#
+# This file is part of IfcOpenShell.
+#
+# IfcOpenShell is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# IfcOpenShell is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with IfcOpenShell. If not, see .
+
+# This file was generated with the assistance of an AI coding tool.
+
+import os
+
+import pytest
+
+import ifcopenshell
+import ifcopenshell.api.pset
+import ifcopenshell.api.root
+import ifcopenshell.util.element
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+FILES = ["ColumnPSetsOfSets.ifc"]
+
+
+@pytest.mark.parametrize("name", FILES)
+def test_lazy_open_matches_full_parse(name):
+ ifcopenshell.get_log() # the log buffer is global: drop what earlier modules logged
+ strict = ifcopenshell.open(os.path.join(FIXTURES, name))
+ lazy = ifcopenshell.open(os.path.join(FIXTURES, name), lazy=True)
+ assert lazy.schema == strict.schema
+ strict_ids = sorted(e.id() for e in strict)
+ assert sorted(e.id() for e in lazy) == strict_ids
+ for e in strict:
+ l = lazy.by_id(e.id())
+ assert l.is_a() == e.is_a()
+ assert str(l) == str(e)
+ assert len(lazy.get_inverse(l)) == len(strict.get_inverse(e))
+ for e in strict.by_type("IfcRoot"):
+ assert lazy.by_guid(e.GlobalId).id() == e.id()
+ assert len(lazy.by_type("IfcWall")) == len(strict.by_type("IfcWall"))
+ assert ifcopenshell.get_log() == ""
+
+
+def test_lazy_file_can_be_edited_and_written(tmp_path):
+ lazy = ifcopenshell.open(os.path.join(FIXTURES, FILES[0]), lazy=True)
+ existing = lazy.by_type("IfcProduct")[0]
+ existing.Name = "Renamed"
+ wall = ifcopenshell.api.root.create_entity(lazy, ifc_class="IfcWall")
+ pset = ifcopenshell.api.pset.add_pset(lazy, product=wall, name="Pset_LazyTest")
+ ifcopenshell.api.pset.edit_pset(lazy, pset=pset, properties={"Answer": 42})
+ out = tmp_path / "lazy.ifc"
+ lazy.write(str(out))
+ reread = ifcopenshell.open(str(out))
+ assert reread.by_id(existing.id()).Name == "Renamed"
+ assert ifcopenshell.util.element.get_psets(reread.by_id(wall.id()))["Pset_LazyTest"]["Answer"] == 42
+ assert len(list(reread)) == len(list(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])))) + len(list(lazy)) - len(
+ list(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])))
+ )
+
+
+def test_lazy_open_passes_over_a_stray_keyword(tmp_path):
+ src = open(os.path.join(FIXTURES, FILES[0]), "rb").read()
+ data_at = src.index(b"DATA;")
+ patched = src[: data_at + 5] + b"\nSTRAY;" + src[data_at + 5 :]
+ path = tmp_path / "stray.ifc"
+ path.write_bytes(patched)
+ f = ifcopenshell.open(str(path), lazy=True)
+ assert f.lazy_loading()
+ assert len(f.by_type("IfcRoot")) == len(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])).by_type("IfcRoot"))
+
+
+def test_lazy_open_falls_back_on_unsupported_syntax(tmp_path):
+ src = open(os.path.join(FIXTURES, FILES[0]), "rb").read()
+ data_at = src.index(b"DATA;")
+ patched = src[: data_at + 5] + b"\n#999999=IFCCARTESIANPOINT((0.;0.));" + src[data_at + 5 :]
+ path = tmp_path / "semicolon.ifc"
+ path.write_bytes(patched)
+ f = ifcopenshell.open(str(path), lazy=True)
+ assert not f.lazy_loading()
+ assert len(f.by_type("IfcRoot")) > 0
diff --git a/src/ifcparse/file.h b/src/ifcparse/file.h
index 8da94b4741..03dfff068e 100644
--- a/src/ifcparse/file.h
+++ b/src/ifcparse/file.h
@@ -213,6 +213,7 @@ public:
std::set types_to_bypass_loading_;
private:
+ bool lazy_loading_ = false;
file_open_status good_ = file_open_status::SUCCESS;
std::reference_wrapper logger_;
@@ -282,6 +283,12 @@ public:
file(const uninitialized_tag& tag, ifcopenshell::logger& logger = ifcopenshell::logger::root());
bool initialize(const std::string& path, filetype type = FT_AUTODETECT, bool read_only = false);
+ // Index the file with one pass and parse each instance's attributes on
+ // first access instead of parsing everything up front. Set before
+ // initialize(). Falls back to the full parse if the index pass finds
+ // anything it does not handle.
+ void lazy_loading(bool value) { lazy_loading_ = value; }
+ bool lazy_loading() const { return lazy_loading_; }
#ifdef USE_MMAP
bool initialize(const std::string& path, bool use_mmap);
#endif
diff --git a/src/ifcparse/instance_data.h b/src/ifcparse/instance_data.h
index c507e2fdf2..f7acf5112f 100644
--- a/src/ifcparse/instance_data.h
+++ b/src/ifcparse/instance_data.h
@@ -534,7 +534,20 @@ class IFC_PARSE_API instance_data {
public:
// Since rocks_db_attribute_storage has no members this is not a variant but an optional in_memory storage, where an empty optional means a rocks_db_attribute_storage is constructed on the fly given the context from instance data.
- std::optional storage_;
+ mutable std::optional storage_;
+
+ // Lazy loading: parses the attributes from the file's retained source
+ // if this instance was indexed lazily and has not been accessed yet.
+ // No-op otherwise (loaded, created, or RocksDB-backed).
+ void ensure_loaded() const;
+ // A lazily indexed instance: the attributes, including the derived
+ // markers, are filled in by the file storage on first access.
+ struct lazy_tag {};
+ instance_data(ifcopenshell::file* file, const ifcopenshell::declaration* declaration, uint32_t id, lazy_tag)
+ : file_(file), declaration_(declaration), identity_(counter_++), id_(id), storage_(std::nullopt)
+ {
+ }
+ friend struct ifcopenshell::impl::in_memory_file_storage;
const ifcopenshell::declaration* declaration() const {
return declaration_;
@@ -597,6 +610,7 @@ class IFC_PARSE_API instance_data {
template
void set_attribute_value(std::size_t attribute_index, T&& value) {
+ ensure_loaded();
if (storage_) {
storage_->set(attribute_index, value);
return;
@@ -612,6 +626,7 @@ class IFC_PARSE_API instance_data {
template
bool has_attribute_value(std::size_t attribute_index) const {
+ ensure_loaded();
if (storage_) {
return storage_->has(attribute_index);
}
diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp
index e3d0e5f864..4a593ce199 100644
--- a/src/ifcparse/parse.cpp
+++ b/src/ifcparse/parse.cpp
@@ -43,6 +43,7 @@
#include
#include
#include
+#include
// Apple clang's libc++ has no floating-point std::from_chars overload (it's
// =deleted), so on macOS doubles are parsed via strtod_l with a cached "C"
@@ -1072,10 +1073,21 @@ shared_pointer_type ifcopenshell::impl::in_memory_file_storage::load(
bool coerce_attribute_count
) {
static_cast(coerce_attribute_count);
+ auto storage = load_attributes(tokens, entity_instance_name, declaration, entity, attribute_index);
+ return ifcopenshell::make_pointer_type(file, declaration, (declaration && declaration->as_entity()) ? (uint32_t)entity_instance_name.value_or(0) : 0, std::move(storage));
+}
+template
+Storage ifcopenshell::impl::in_memory_file_storage::load_attributes(
+ ifcopenshell::spf_lexer* tokens,
+ std::optional entity_instance_name,
+ const ifcopenshell::declaration* declaration,
+ const ifcopenshell::entity* entity,
+ int attribute_index
+) {
parameter_type_view parameter_types(declaration);
const size_t expected_size = parameter_types.size();
- in_memory_attribute_storage storage(expected_size);
+ Storage storage(expected_size);
token next = tokens->next();
size_t attribute_index_within_data = 0;
@@ -1133,7 +1145,7 @@ shared_pointer_type ifcopenshell::impl::in_memory_file_storage::load(
}
warn_attribute_count(declaration, entity_instance_name, expected_size, values_read, logger_.get());
- return ifcopenshell::make_pointer_type(file, declaration, (declaration && declaration->as_entity()) ? (uint32_t)entity_instance_name.value_or(0) : 0, std::move(storage));
+ return storage;
}
template
@@ -1146,6 +1158,9 @@ void ifcopenshell::impl::in_memory_file_storage::try_read_semicolon(ifcopenshell
}
void ifcopenshell::impl::in_memory_file_storage::register_inverse(unsigned id_from, const ifcopenshell::entity* from_entity, int inst_id, int attribute_index) {
+ if (!register_inverses_) {
+ return;
+ }
// Assume a check on token type has already been performed
byref_excl_.add((uint32_t)inst_id, (uint32_t)id_from, (uint16_t)from_entity->index_in_schema(), attribute_index);
}
@@ -1914,10 +1929,24 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r
ty = guess_file_type(path);
}
if (ty == FT_IFCSPF) {
- file_reader s(path);
storage_.emplace<1>(this, logger_.get());
header_.reset(new spf_header(this, &logger_.get()));
- std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+ bool indexed = false;
+ if (lazy_loading_) {
+ indexed = std::get(storage_).index_lazily(path, schema_, max_id_, types_to_bypass_loading_);
+ if (!indexed) {
+ // Start over with the full parser.
+ storage_.emplace<1>(this, logger_.get());
+ header_.reset(new spf_header(this, &logger_.get()));
+ schema_ = nullptr;
+ max_id_ = 0;
+ lazy_loading_ = false;
+ }
+ }
+ if (!indexed) {
+ file_reader s(path);
+ std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+ }
if ((good_ = std::get(storage_).good_)) {
// @todo unify these names, it's already confusing enough as it stands
@@ -2523,6 +2552,258 @@ void ifcopenshell::impl::in_memory_file_storage::resolve_instance_references(con
}
}
+struct ifcopenshell::impl::in_memory_file_storage::lazy_source {
+ file_reader reader;
+ spf_lexer> lexer;
+ lazy_source(const std::string& path, ifcopenshell::logger& logger)
+ : reader(path, 64 << 10, 64), lexer(&reader, logger) {}
+};
+
+namespace {
+void delete_lazy_source(ifcopenshell::impl::in_memory_file_storage::lazy_source* source) {
+ delete source;
+}
+}
+
+void instance_data::ensure_loaded() const {
+ if (storage_ || file_ == nullptr) {
+ return;
+ }
+ if (auto* storage = std::get_if(&file_->storage_)) {
+ if (storage->lazy_) {
+ storage->materialize(const_cast(this));
+ }
+ }
+}
+
+void ifcopenshell::impl::in_memory_file_storage::materialize(instance_data* data) {
+ const auto offset = std::lower_bound(lazy_offsets_.begin(), lazy_offsets_.end(), std::make_pair(data->id(), (uint64_t)0), [](const auto& a, const auto& b) { return a.first < b.first; });
+ if (offset == lazy_offsets_.end() || offset->first != data->id()) {
+ return;
+ }
+ auto& lexer = lazy_source_->lexer;
+ lexer.stream->seek((size_t)offset->second);
+ const auto* declaration = data->declaration();
+ register_inverses_ = false;
+ const size_t simple_type_instances_before = read_simple_type_instances.size();
+ auto storage = load_attributes, in_memory_attribute_storage>(&lexer, (size_t)data->id(), declaration, declaration->as_entity(), -1);
+ lexer.reset_pool();
+ register_inverses_ = true;
+ data->storage_.emplace(std::move(storage));
+ data->populate_derived_();
+ auto self = byid_.find(data->id());
+ if (self != byid_.end()) {
+ resolve_instance_references(self->second, lazy_bypassed_);
+ }
+ for (size_t i = simple_type_instances_before; i < read_simple_type_instances.size(); ++i) {
+ resolve_instance_references(read_simple_type_instances[i], lazy_bypassed_);
+ }
+}
+
+bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& path, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& types_to_bypass) {
+ lazy_source_ = decltype(lazy_source_)(new lazy_source(path, logger_.get()), &delete_lazy_source);
+ auto& reader = lazy_source_->reader;
+ auto& lexer = lazy_source_->lexer;
+ if (!reader.size()) {
+ return false;
+ }
+
+ // The header, the schema and the bypass set, exactly as the full parse
+ // derives them; the streamer leaves the reader at the DATA section.
+ {
+ instance_streamer> streamer(&reader, file, logger_.get());
+ schema = streamer.schema();
+ }
+ if (schema == nullptr) {
+ return false;
+ }
+ std::vector bypassed_types(schema->declarations().size(), 0);
+ for (const auto& name : types_to_bypass) {
+ const ifcopenshell::declaration* declaration = nullptr;
+ try {
+ declaration = schema->declaration_by_name(name);
+ } catch (const ifcopenshell::exception&) {
+ continue;
+ }
+ std::function mark = [&](const ifcopenshell::entity* e) {
+ bypassed_types[e->index_in_schema()] = 1;
+ for (const auto* subtype : e->subtypes()) {
+ mark(subtype);
+ }
+ };
+ if (const auto* e = declaration->as_entity()) {
+ mark(e);
+ }
+ }
+ const auto* ifcroot = schema->declaration_by_name("IfcRoot");
+
+ this->schema = schema;
+ resolve_references_in_place = true;
+ lazy_ = true;
+ byref_excl_.reserve(reader.size() / 32);
+
+ // One pass over the DATA section with the tokenizer's index policy:
+ // the instance headers as tokens, then the attribute list as tokens
+ // with only the parentheses, commas and names looked at. Nothing is
+ // decoded. A keyword where an instance should start, or a token the
+ // tokenizer rejects, stops the index and the caller parses in full.
+ std::unordered_map declarations;
+ const char* failure = nullptr;
+ size_t failure_offset = 0;
+ bool in_data = false;
+ try {
+ while (failure == nullptr) {
+ token first = lexer.next();
+ if (!first) {
+ break;
+ }
+ if (first.is_keyword()) {
+ const std::string keyword = first.as_string();
+ if (keyword == "ENDSEC") {
+ if (in_data) {
+ break;
+ }
+ in_data = true; // the header's; DATA follows
+ } else if (keyword != "DATA") {
+ failure = "keyword where an instance should start";
+ failure_offset = first.start_pos;
+ }
+ lexer.reset_pool();
+ continue;
+ }
+ if (!first.is_identifier()) {
+ continue;
+ }
+ in_data = true;
+ const uint32_t name = (uint32_t)first.as_identifier();
+ if (!lexer.next().is_operator('=')) {
+ continue;
+ }
+ token keyword = lexer.next();
+ if (!keyword.is_keyword()) {
+ continue;
+ }
+ const ifcopenshell::declaration* declaration = nullptr;
+ const std::string keyword_text = keyword.as_string();
+ lexer.reset_pool();
+ auto found = declarations.find(keyword_text);
+ if (found == declarations.end()) {
+ try {
+ declaration = schema->declaration_by_name(keyword_text);
+ if (declaration->as_entity() == nullptr) {
+ logger_.get().message(ifcopenshell::logger::LOG_ERROR, "Non-entity type " + declaration->name() + " at offset " + std::to_string(keyword.start_pos));
+ declaration = nullptr;
+ }
+ } catch (const exception& e) {
+ logger_.get().message(ifcopenshell::logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(keyword.start_pos));
+ }
+ declarations.emplace(keyword_text, declaration);
+ } else {
+ declaration = found->second;
+ }
+ if (!lexer.next().is_operator('(')) {
+ failure = "expected ( after the type";
+ failure_offset = keyword.start_pos;
+ break;
+ }
+ const uint64_t attributes_offset = reader.tell();
+ const bool bypassed = declaration != nullptr && bypassed_types[declaration->index_in_schema()];
+ const bool indexed = declaration != nullptr && !bypassed;
+ const uint16_t type_index = indexed ? (uint16_t)declaration->index_in_schema() : 0;
+ int depth = 1;
+ int attribute = 0;
+ bool first_value = true;
+ size_t guid_begin = 0, guid_end = 0;
+ while (depth > 0) {
+ token t = lexer.next();
+ if (!t) {
+ failure = "file ends inside an instance";
+ failure_offset = attributes_offset;
+ break;
+ }
+ if (t.is_operator()) {
+ if (t.value_char == '(') {
+ ++depth;
+ } else if (t.value_char == ')') {
+ --depth;
+ } else if (t.value_char == ',' && depth == 1) {
+ ++attribute;
+ } else if (t.value_char == ';') {
+ failure = "; inside an instance";
+ failure_offset = t.start_pos;
+ break;
+ }
+ } else if (t.is_identifier()) {
+ if (indexed) {
+ byref_excl_.add((uint32_t)t.as_identifier(), name, type_index, attribute);
+ }
+ } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) {
+ guid_begin = t.start_pos + 1;
+ guid_end = reader.tell() - 1;
+ }
+ if (depth == 1) {
+ first_value = false;
+ }
+ lexer.reset_pool();
+ }
+ if (failure != nullptr) {
+ break;
+ }
+ if (!lexer.next().is_operator(';')) {
+ failure = "expected ; after )";
+ failure_offset = reader.tell();
+ break;
+ }
+ if (bypassed) {
+ lazy_bypassed_.push_back(name);
+ }
+ if (!indexed) {
+ continue;
+ }
+ auto data = ifcopenshell::make_pointer_type(file, declaration, name, instance_data::lazy_tag{});
+ if (!byid_.insert({name, data}).second) {
+ logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Overwriting instance with name #" + std::to_string(name));
+ byid_.erase(name);
+ byid_.insert({name, data});
+ }
+ lazy_offsets_.push_back({name, attributes_offset});
+ express::base instance(data);
+ bytype_excl_[declaration].push_back(instance);
+ max_id = (std::max)(max_id, (unsigned int)name);
+ if (guid_end > guid_begin && declaration->is(*ifcroot)) {
+ std::string guid;
+ guid.reserve(guid_end - guid_begin);
+ for (size_t at = guid_begin; at < guid_end; ++at) {
+ guid.push_back(reader.get(at));
+ }
+ if (guid.find('\\') != std::string::npos || guid.find("''") != std::string::npos) {
+ guid = ifcopenshell::decode_spf_string(guid);
+ }
+ std::array key;
+ if (guid_key(guid, key)) {
+ if (byguid_.count(key) != 0) {
+ logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Instance encountered with non-unique GlobalId " + guid);
+ }
+ byguid_[key] = instance;
+ }
+ }
+ }
+ } catch (const invalid_token_exception&) {
+ failure = "invalid token";
+ failure_offset = reader.tell();
+ }
+ if (failure != nullptr) {
+ logger_.get().message(ifcopenshell::logger::LOG_NOTICE, std::string("Lazy loading not possible (") + failure + " at offset " + std::to_string(failure_offset) + "), parsing the file in full");
+ return false;
+ }
+
+ std::sort(lazy_bypassed_.begin(), lazy_bypassed_.end());
+ std::sort(lazy_offsets_.begin(), lazy_offsets_.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
+ byref_excl_.sort();
+ good_ = file_open_status::SUCCESS;
+ return true;
+}
+
template
void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass) {
schema = nullptr;
@@ -3732,6 +4013,7 @@ instance_data::instance_data(const instance_data& data)
attribute_value instance_data::get_attribute_value(size_t index) const
{
+ ensure_loaded();
if (storage_) {
return attribute_value(&*storage_, (uint8_t)index);
} else {
diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h
index 7f47de644f..62d6d8e851 100644
--- a/src/ifcparse/storage.h
+++ b/src/ifcparse/storage.h
@@ -585,6 +585,25 @@ namespace ifcopenshell {
// on; streaming consumers of references() leave it off.
bool resolve_references_in_place = false;
+ // Lazy loading (index_lazily): the file was read once through the
+ // tokenizer's index policy to build the instance shells, the
+ // inverse index, the GlobalId map and the by-type lists, and each
+ // instance's attributes are parsed from the retained paged source
+ // the first time they are accessed (instance_data::ensure_loaded).
+ // The offset of each instance's attribute list lives here, not in
+ // the instance, so a full parse pays nothing for it. Inverses were
+ // registered by the index, so materialisation must not register
+ // them again. Materialising from several threads at once is not
+ // safe.
+ struct lazy_source;
+ bool lazy_ = false;
+ bool register_inverses_ = true;
+ std::unique_ptr lazy_source_{nullptr, nullptr};
+ std::vector lazy_bypassed_;
+ std::vector> lazy_offsets_;
+ bool index_lazily(const std::string& path, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& types_to_bypass);
+ void materialize(instance_data* data);
+
typedef std::map> entities_by_type;
typedef std::unordered_map entity_instance_by_name_storage;
typedef map_transformer> entity_instance_by_name;
@@ -658,6 +677,13 @@ namespace ifcopenshell {
shared_pointer_type load(ifcopenshell::spf_lexer* tokens, std::optional entity_instance_name, const ifcopenshell::declaration* declaration, const ifcopenshell::entity* entity, int attribute_index = -1, bool coerce_attribute_count = true);
template
void try_read_semicolon(ifcopenshell::spf_lexer* tokens) const;
+ // The attribute-reading half of load(): the tokens after the
+ // opening parenthesis into a fresh attribute array. Storage is
+ // always in_memory_attribute_storage; it is a template parameter
+ // only because that type is defined in a header that includes
+ // this one.
+ template
+ Storage load_attributes(ifcopenshell::spf_lexer* tokens, std::optional entity_instance_name, const ifcopenshell::declaration* declaration, const ifcopenshell::entity* entity, int attribute_index = -1);
// Replaces the names left in `data`'s attribute slots by in-place
// reference storage with the instances they name; a name that is
// missing or bypassed becomes null in a scalar and is dropped
diff --git a/src/ifcparse/tests/test_ifcopenshell_parse.cpp b/src/ifcparse/tests/test_ifcopenshell_parse.cpp
index 1f7a7eeac7..13f1cb4fc7 100644
--- a/src/ifcparse/tests/test_ifcopenshell_parse.cpp
+++ b/src/ifcparse/tests/test_ifcopenshell_parse.cpp
@@ -434,3 +434,68 @@ TEST_CASE("References to bypassed instances are dropped from slots and from mixe
CHECK(control_points[0].empty());
CHECK(control_points[1].empty());
}
+
+namespace {
+void check_lazy_matches_strict(const std::string& path) {
+ ifcopenshell::file strict(path);
+ REQUIRE(strict.good());
+ ifcopenshell::file lazy(ifcopenshell::uninitialized_tag{});
+ lazy.lazy_loading(true);
+ REQUIRE(lazy.initialize(path));
+ REQUIRE(lazy.lazy_loading());
+ REQUIRE(lazy.schema() == strict.schema());
+
+ size_t strict_count = 0;
+ for (auto it = strict.begin(); it != strict.end(); ++it) {
+ const express::base a = it->second;
+ const express::base b = lazy.instance_by_id((int)a.id());
+ REQUIRE(b);
+ REQUIRE(&b.declaration() == &a.declaration());
+ REQUIRE(lazy.instances_by_reference((int)a.id()).size() == strict.instances_by_reference((int)a.id()).size());
+ std::ostringstream sa, sb;
+ a.to_string(sa);
+ b.to_string(sb);
+ REQUIRE(sb.str() == sa.str());
+ ++strict_count;
+ }
+ size_t lazy_count = 0;
+ for (auto it = lazy.begin(); it != lazy.end(); ++it) {
+ ++lazy_count;
+ }
+ CHECK(lazy_count == strict_count);
+
+ for (const auto& rooted : strict.instances_by_type("IfcRoot")) {
+ const std::string guid = rooted.get_attribute_value(0);
+ REQUIRE(lazy.instance_by_guid(guid).id() == rooted.id());
+ }
+}
+}
+
+TEST_CASE("Lazy loading yields the same instances, attributes, inverses and GlobalIds as a full parse", "[ifcparse]") {
+ check_lazy_matches_strict(std::string(IFCOPENSHELL_TEST_FIXTURES) + "/ColumnPSetsOfSets.ifc");
+
+ const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_lazy_loading_test.ifc";
+ {
+ std::ofstream out(path);
+ out << reference_resolution_spf;
+ }
+ check_lazy_matches_strict(path.string());
+ std::filesystem::remove(path);
+}
+
+TEST_CASE("Lazy loading falls back to the full parser on syntax the index pass does not handle", "[ifcparse]") {
+ const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_lazy_fallback_test.ifc";
+ {
+ std::ofstream out(path);
+ // A stray keyword between instances is something the full parser skips past, but the index pass gives up on.
+ std::string spf(reference_resolution_spf);
+ spf.replace(spf.find("#8=IFCWALL"), 0, "STRAY;\n");
+ out << spf;
+ }
+ ifcopenshell::file lazy(ifcopenshell::uninitialized_tag{});
+ lazy.lazy_loading(true);
+ lazy.initialize(path.string());
+ std::filesystem::remove(path);
+ CHECK_FALSE(lazy.lazy_loading());
+ CHECK(lazy.instance_by_id(4));
+}
diff --git a/src/serializers/rocks_db_serializer.cpp b/src/serializers/rocks_db_serializer.cpp
index b593c41805..ec1d018291 100644
--- a/src/serializers/rocks_db_serializer.cpp
+++ b/src/serializers/rocks_db_serializer.cpp
@@ -150,6 +150,7 @@ void RocksDbSerializer::write_streaming_() {
std::vector simple_type_instances;
+ data->ensure_loaded();
for (size_t i = 0; i < data->storage_->size(); i++) {
auto val = data->get_attribute_value(i);
val.apply_visitor([&](const auto& t) {
@@ -291,6 +292,7 @@ void RocksDbSerializer::write_streaming_() {
}
if (decl->is(*ifcroot_type)) {
// @nb attribute counts are not coerced, so the attribute may be absent
+ data->ensure_loaded();
const bool has_guid = data->storage_->size() > 0 && data->get_attribute_value(0).type() == ifcopenshell::Argument_STRING;
if (has_guid) {
size_t v = name;