ifcparse: lazy loading, opt in with file::lazy_loading() / open(lazy=True)

A lazy open reads the DATA section once with the tokenizer's index
policy and builds what indexes the file: a shell per instance (name and
declaration, no attribute array), the complete inverse index with
attribute indices, the GlobalId map and the by-type lists. No attribute
value is decoded. The first time an instance's attributes are touched,
ensure_loaded() seeks the retained paged reader to the instance and runs
the same load_attributes() the full parse runs, with inverse registration
off, then resolves that instance's references from its own slots. A
modified instance is materialised first, so writing works.

There is no scanner of its own: the index pass consumes next<index_tokens>()
and counts parentheses and commas on the operator tokens; a keyword where
an instance should start, or a token the tokenizer rejects, stops the
index and the file is parsed in full. The offset of each instance's
attribute list is kept in one sorted vector that exists only in lazy
mode, so a full parse pays nothing for it. Materialising from several
threads at once is not safe.

TXG 58 MB / 210_King 147 MB / OKgate22 231 MB, single thread: lazy open
0.61 / 1.73 / 2.86 s against the full parse's 1.05 / 2.69 / 4.99 s, at
141 / 374 / 534 MB against 274 / 654 / 1036 MB; reading one attribute of
every instance afterwards costs a further 0.56 / 1.44 / 4.86 s.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL

src/ifcopenshell-python/ifcopenshell/__init__.py

@@ -195,12 +195,19 @@ def open(
     mmap: bool = False,
     bypass_types: Optional[Sequence[str]] = None,
     logger: Optional[logger] = None,
+    lazy: bool = False,
 ) -> Union[file, sqlite, _stream]:
     """Loads an IFC dataset from a filepath
 
     :param should_stream: Whether to open the file in streaming mode. Could be useful
         for reading large files.
     :param logger: Logger that receives native parser messages.
+    :param lazy: Index the file with one quick pass and parse each instance's
+        attributes only when they are first read. Opening is then faster and
+        memory stays proportional to what is accessed; reading every attribute
+        of every instance costs about the same as a normal open, spread over
+        the reads. Falls back to a normal open if the file uses syntax the
+        index pass does not handle.
 
     You can specify a file format. If no format is given, it is guessed from
     its extension.
@@ -242,10 +249,12 @@ def open(
         return stream(path)
     if readonly:  # Temporary conditional see #7131. Remove once newer builds don't segfault on Linux.
         f = ifcopenshell_wrapper.open(str(path.absolute()), readonly, *optional_logger_args(logger))
-    elif bypass_types:
+    elif bypass_types or lazy:
         f = ifcopenshell_wrapper.file.create_uninitialized(*optional_logger_args(logger))
-        for ty in bypass_types:
+        for ty in bypass_types or ():
             f.bypass_type(ty)
+        if lazy:
+            f.lazy_loading(True)
         if mmap:
             # mmap parameter is only available for builds with USE_MMAP, not used in our main builds
             f.initialize(str(path.absolute()), mmap=mmap)  # ty: ignore[unknown-argument]

src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi

@@ -274,6 +274,7 @@ class geometry_serializer:
 class instance_streamer:
     def __init__(self, *args): ...
     def bypass_types(self, type_names): ...
+    def resolve_references_in_place(self, value): ...
     def bypassed_instances(self): ...
     coerce_attribute_count: bool
     def has_semicolon(self): ...
@@ -924,6 +925,9 @@ class file(file_mixin):
         ...
 
     def get_max_id(self) -> int: ...
+    def lazy_loading(self, *args: bool) -> bool:
+        """Get, or with an argument set, whether ``initialize()`` indexes the file with one pass and parses each instance's attributes on first access. Set before ``initialize()``."""
+        ...
     def get_inverse_indices_by_id(self, instance_id: int) -> tuple[int, ...]: ...
     def _get_inverse(self, e: entity_instance) -> tuple[entity_instance, ...]: ...
     def _get_inverse_indices(self, *args: Union[entity_instance, int]) -> tuple[int, ...]:

src/ifcopenshell-python/test/test_lazy.py

@@ -0,0 +1,89 @@
+# IfcOpenShell - IFC toolkit and geometry engine
+# Copyright (C) 2026 IfcOpenShell contributors
+#
+# This file is part of IfcOpenShell.
+#
+# IfcOpenShell is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# IfcOpenShell is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with IfcOpenShell.  If not, see <http://www.gnu.org/licenses/>.
+
+# This file was generated with the assistance of an AI coding tool.
+
+import os
+
+import pytest
+
+import ifcopenshell
+import ifcopenshell.api.pset
+import ifcopenshell.api.root
+import ifcopenshell.util.element
+
+FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
+FILES = ["ColumnPSetsOfSets.ifc"]
+
+
+@pytest.mark.parametrize("name", FILES)
+def test_lazy_open_matches_full_parse(name):
+    ifcopenshell.get_log()  # the log buffer is global: drop what earlier modules logged
+    strict = ifcopenshell.open(os.path.join(FIXTURES, name))
+    lazy = ifcopenshell.open(os.path.join(FIXTURES, name), lazy=True)
+    assert lazy.schema == strict.schema
+    strict_ids = sorted(e.id() for e in strict)
+    assert sorted(e.id() for e in lazy) == strict_ids
+    for e in strict:
+        l = lazy.by_id(e.id())
+        assert l.is_a() == e.is_a()
+        assert str(l) == str(e)
+        assert len(lazy.get_inverse(l)) == len(strict.get_inverse(e))
+    for e in strict.by_type("IfcRoot"):
+        assert lazy.by_guid(e.GlobalId).id() == e.id()
+    assert len(lazy.by_type("IfcWall")) == len(strict.by_type("IfcWall"))
+    assert ifcopenshell.get_log() == ""
+
+
+def test_lazy_file_can_be_edited_and_written(tmp_path):
+    lazy = ifcopenshell.open(os.path.join(FIXTURES, FILES[0]), lazy=True)
+    existing = lazy.by_type("IfcProduct")[0]
+    existing.Name = "Renamed"
+    wall = ifcopenshell.api.root.create_entity(lazy, ifc_class="IfcWall")
+    pset = ifcopenshell.api.pset.add_pset(lazy, product=wall, name="Pset_LazyTest")
+    ifcopenshell.api.pset.edit_pset(lazy, pset=pset, properties={"Answer": 42})
+    out = tmp_path / "lazy.ifc"
+    lazy.write(str(out))
+    reread = ifcopenshell.open(str(out))
+    assert reread.by_id(existing.id()).Name == "Renamed"
+    assert ifcopenshell.util.element.get_psets(reread.by_id(wall.id()))["Pset_LazyTest"]["Answer"] == 42
+    assert len(list(reread)) == len(list(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])))) + len(list(lazy)) - len(
+        list(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])))
+    )
+
+
+def test_lazy_open_passes_over_a_stray_keyword(tmp_path):
+    src = open(os.path.join(FIXTURES, FILES[0]), "rb").read()
+    data_at = src.index(b"DATA;")
+    patched = src[: data_at + 5] + b"\nSTRAY;" + src[data_at + 5 :]
+    path = tmp_path / "stray.ifc"
+    path.write_bytes(patched)
+    f = ifcopenshell.open(str(path), lazy=True)
+    assert f.lazy_loading()
+    assert len(f.by_type("IfcRoot")) == len(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])).by_type("IfcRoot"))
+
+
+def test_lazy_open_falls_back_on_unsupported_syntax(tmp_path):
+    src = open(os.path.join(FIXTURES, FILES[0]), "rb").read()
+    data_at = src.index(b"DATA;")
+    patched = src[: data_at + 5] + b"\n#999999=IFCCARTESIANPOINT((0.;0.));" + src[data_at + 5 :]
+    path = tmp_path / "semicolon.ifc"
+    path.write_bytes(patched)
+    f = ifcopenshell.open(str(path), lazy=True)
+    assert not f.lazy_loading()
+    assert len(f.by_type("IfcRoot")) > 0

src/ifcparse/file.h

@@ -213,6 +213,7 @@ public:
     std::set<std::string> types_to_bypass_loading_;
 
   private:
+    bool lazy_loading_ = false;
     file_open_status good_ = file_open_status::SUCCESS;
     std::reference_wrapper<ifcopenshell::logger> logger_;
 
@@ -282,6 +283,12 @@ public:
     file(const uninitialized_tag& tag, ifcopenshell::logger& logger = ifcopenshell::logger::root());
 
     bool initialize(const std::string& path, filetype type = FT_AUTODETECT, bool read_only = false);
+    // Index the file with one pass and parse each instance's attributes on
+    // first access instead of parsing everything up front. Set before
+    // initialize(). Falls back to the full parse if the index pass finds
+    // anything it does not handle.
+    void lazy_loading(bool value) { lazy_loading_ = value; }
+    bool lazy_loading() const { return lazy_loading_; }
 #ifdef USE_MMAP
     bool initialize(const std::string& path, bool use_mmap);
 #endif

src/ifcparse/instance_data.h

@@ -534,7 +534,20 @@ class IFC_PARSE_API instance_data {
 
   public:
       // Since rocks_db_attribute_storage has no members this is not a variant<in_memory, rocks> but an optional in_memory storage, where an empty optional means a rocks_db_attribute_storage is constructed on the fly given the context from instance data.
-      std::optional<in_memory_attribute_storage> storage_;
+      mutable std::optional<in_memory_attribute_storage> storage_;
+
+      // Lazy loading: parses the attributes from the file's retained source
+      // if this instance was indexed lazily and has not been accessed yet.
+      // No-op otherwise (loaded, created, or RocksDB-backed).
+      void ensure_loaded() const;
+      // A lazily indexed instance: the attributes, including the derived
+      // markers, are filled in by the file storage on first access.
+      struct lazy_tag {};
+      instance_data(ifcopenshell::file* file, const ifcopenshell::declaration* declaration, uint32_t id, lazy_tag)
+          : file_(file), declaration_(declaration), identity_(counter_++), id_(id), storage_(std::nullopt)
+      {
+      }
+      friend struct ifcopenshell::impl::in_memory_file_storage;
 
       const ifcopenshell::declaration* declaration() const {
           return declaration_;
@@ -597,6 +610,7 @@ class IFC_PARSE_API instance_data {
 
     template<typename T>
     void set_attribute_value(std::size_t attribute_index, T&& value) {
+        ensure_loaded();
         if (storage_) {
             storage_->set(attribute_index, value);
             return;
@@ -612,6 +626,7 @@ class IFC_PARSE_API instance_data {
 
     template<typename T>
     bool has_attribute_value(std::size_t attribute_index) const {
+        ensure_loaded();
         if (storage_) {
             return storage_->has<T>(attribute_index);
         }

src/ifcparse/parse.cpp

@@ -43,6 +43,7 @@
 #include <charconv>
 #include <type_traits>
 #include <unordered_map>
+#include <functional>
 
 // Apple clang's libc++ has no floating-point std::from_chars overload (it's
 // =deleted), so on macOS doubles are parsed via strtod_l with a cached "C"
@@ -1072,10 +1073,21 @@ shared_pointer_type ifcopenshell::impl::in_memory_file_storage::load(
     bool coerce_attribute_count
 ) {
     static_cast<void>(coerce_attribute_count);
+    auto storage = load_attributes<Reader, in_memory_attribute_storage>(tokens, entity_instance_name, declaration, entity, attribute_index);
+    return ifcopenshell::make_pointer_type<instance_data>(file, declaration, (declaration && declaration->as_entity()) ? (uint32_t)entity_instance_name.value_or(0) : 0, std::move(storage));
+}
 
+template <typename Reader, typename Storage>
+Storage ifcopenshell::impl::in_memory_file_storage::load_attributes(
+    ifcopenshell::spf_lexer<Reader>* tokens,
+    std::optional<size_t> entity_instance_name,
+    const ifcopenshell::declaration* declaration,
+    const ifcopenshell::entity* entity,
+    int attribute_index
+) {
     parameter_type_view parameter_types(declaration);
     const size_t expected_size = parameter_types.size();
-    in_memory_attribute_storage storage(expected_size);
+    Storage storage(expected_size);
 
     token next = tokens->next();
     size_t attribute_index_within_data = 0;
@@ -1133,7 +1145,7 @@ shared_pointer_type ifcopenshell::impl::in_memory_file_storage::load(
     }
 
     warn_attribute_count(declaration, entity_instance_name, expected_size, values_read, logger_.get());
-    return ifcopenshell::make_pointer_type<instance_data>(file, declaration, (declaration && declaration->as_entity()) ? (uint32_t)entity_instance_name.value_or(0) : 0, std::move(storage));
+    return storage;
 }
 
 template <typename Reader>
@@ -1146,6 +1158,9 @@ void ifcopenshell::impl::in_memory_file_storage::try_read_semicolon(ifcopenshell
 }
 
 void ifcopenshell::impl::in_memory_file_storage::register_inverse(unsigned id_from, const ifcopenshell::entity* from_entity, int inst_id, int attribute_index) {
+    if (!register_inverses_) {
+        return;
+    }
     // Assume a check on token type has already been performed
     byref_excl_.add((uint32_t)inst_id, (uint32_t)id_from, (uint16_t)from_entity->index_in_schema(), attribute_index);
 }
@@ -1914,10 +1929,24 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r
         ty = guess_file_type(path);
     }
     if (ty == FT_IFCSPF) {
-        file_reader<full_buffer_impl> s(path);
         storage_.emplace<1>(this, logger_.get());
         header_.reset(new spf_header(this, &logger_.get()));
-        std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+        bool indexed = false;
+        if (lazy_loading_) {
+            indexed = std::get<impl::in_memory_file_storage>(storage_).index_lazily(path, schema_, max_id_, types_to_bypass_loading_);
+            if (!indexed) {
+                // Start over with the full parser.
+                storage_.emplace<1>(this, logger_.get());
+                header_.reset(new spf_header(this, &logger_.get()));
+                schema_ = nullptr;
+                max_id_ = 0;
+                lazy_loading_ = false;
+            }
+        }
+        if (!indexed) {
+            file_reader<full_buffer_impl> s(path);
+            std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+        }
 
         if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
             // @todo unify these names, it's already confusing enough as it stands
@@ -2523,6 +2552,258 @@ void ifcopenshell::impl::in_memory_file_storage::resolve_instance_references(con
     }
 }
 
+struct ifcopenshell::impl::in_memory_file_storage::lazy_source {
+    file_reader<paged_file_impl> reader;
+    spf_lexer<file_reader<paged_file_impl>> lexer;
+    lazy_source(const std::string& path, ifcopenshell::logger& logger)
+        : reader(path, 64 << 10, 64), lexer(&reader, logger) {}
+};
+
+namespace {
+void delete_lazy_source(ifcopenshell::impl::in_memory_file_storage::lazy_source* source) {
+    delete source;
+}
+}
+
+void instance_data::ensure_loaded() const {
+    if (storage_ || file_ == nullptr) {
+        return;
+    }
+    if (auto* storage = std::get_if<ifcopenshell::impl::in_memory_file_storage>(&file_->storage_)) {
+        if (storage->lazy_) {
+            storage->materialize(const_cast<instance_data*>(this));
+        }
+    }
+}
+
+void ifcopenshell::impl::in_memory_file_storage::materialize(instance_data* data) {
+    const auto offset = std::lower_bound(lazy_offsets_.begin(), lazy_offsets_.end(), std::make_pair(data->id(), (uint64_t)0), [](const auto& a, const auto& b) { return a.first < b.first; });
+    if (offset == lazy_offsets_.end() || offset->first != data->id()) {
+        return;
+    }
+    auto& lexer = lazy_source_->lexer;
+    lexer.stream->seek((size_t)offset->second);
+    const auto* declaration = data->declaration();
+    register_inverses_ = false;
+    const size_t simple_type_instances_before = read_simple_type_instances.size();
+    auto storage = load_attributes<file_reader<paged_file_impl>, in_memory_attribute_storage>(&lexer, (size_t)data->id(), declaration, declaration->as_entity(), -1);
+    lexer.reset_pool();
+    register_inverses_ = true;
+    data->storage_.emplace(std::move(storage));
+    data->populate_derived_();
+    auto self = byid_.find(data->id());
+    if (self != byid_.end()) {
+        resolve_instance_references(self->second, lazy_bypassed_);
+    }
+    for (size_t i = simple_type_instances_before; i < read_simple_type_instances.size(); ++i) {
+        resolve_instance_references(read_simple_type_instances[i], lazy_bypassed_);
+    }
+}
+
+bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& path, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& types_to_bypass) {
+    lazy_source_ = decltype(lazy_source_)(new lazy_source(path, logger_.get()), &delete_lazy_source);
+    auto& reader = lazy_source_->reader;
+    auto& lexer = lazy_source_->lexer;
+    if (!reader.size()) {
+        return false;
+    }
+
+    // The header, the schema and the bypass set, exactly as the full parse
+    // derives them; the streamer leaves the reader at the DATA section.
+    {
+        instance_streamer<file_reader<paged_file_impl>> streamer(&reader, file, logger_.get());
+        schema = streamer.schema();
+    }
+    if (schema == nullptr) {
+        return false;
+    }
+    std::vector<char> bypassed_types(schema->declarations().size(), 0);
+    for (const auto& name : types_to_bypass) {
+        const ifcopenshell::declaration* declaration = nullptr;
+        try {
+            declaration = schema->declaration_by_name(name);
+        } catch (const ifcopenshell::exception&) {
+            continue;
+        }
+        std::function<void(const ifcopenshell::entity*)> mark = [&](const ifcopenshell::entity* e) {
+            bypassed_types[e->index_in_schema()] = 1;
+            for (const auto* subtype : e->subtypes()) {
+                mark(subtype);
+            }
+        };
+        if (const auto* e = declaration->as_entity()) {
+            mark(e);
+        }
+    }
+    const auto* ifcroot = schema->declaration_by_name("IfcRoot");
+
+    this->schema = schema;
+    resolve_references_in_place = true;
+    lazy_ = true;
+    byref_excl_.reserve(reader.size() / 32);
+
+    // One pass over the DATA section with the tokenizer's index policy:
+    // the instance headers as tokens, then the attribute list as tokens
+    // with only the parentheses, commas and names looked at. Nothing is
+    // decoded. A keyword where an instance should start, or a token the
+    // tokenizer rejects, stops the index and the caller parses in full.
+    std::unordered_map<std::string, const ifcopenshell::declaration*> declarations;
+    const char* failure = nullptr;
+    size_t failure_offset = 0;
+    bool in_data = false;
+    try {
+        while (failure == nullptr) {
+            token first = lexer.next<index_tokens>();
+            if (!first) {
+                break;
+            }
+            if (first.is_keyword()) {
+                const std::string keyword = first.as_string();
+                if (keyword == "ENDSEC") {
+                    if (in_data) {
+                        break;
+                    }
+                    in_data = true;  // the header's; DATA follows
+                } else if (keyword != "DATA") {
+                    failure = "keyword where an instance should start";
+                    failure_offset = first.start_pos;
+                }
+                lexer.reset_pool();
+                continue;
+            }
+            if (!first.is_identifier()) {
+                continue;
+            }
+            in_data = true;
+            const uint32_t name = (uint32_t)first.as_identifier();
+            if (!lexer.next<index_tokens>().is_operator('=')) {
+                continue;
+            }
+            token keyword = lexer.next<index_tokens>();
+            if (!keyword.is_keyword()) {
+                continue;
+            }
+            const ifcopenshell::declaration* declaration = nullptr;
+            const std::string keyword_text = keyword.as_string();
+            lexer.reset_pool();
+            auto found = declarations.find(keyword_text);
+            if (found == declarations.end()) {
+                try {
+                    declaration = schema->declaration_by_name(keyword_text);
+                    if (declaration->as_entity() == nullptr) {
+                        logger_.get().message(ifcopenshell::logger::LOG_ERROR, "Non-entity type " + declaration->name() + " at offset " + std::to_string(keyword.start_pos));
+                        declaration = nullptr;
+                    }
+                } catch (const exception& e) {
+                    logger_.get().message(ifcopenshell::logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(keyword.start_pos));
+                }
+                declarations.emplace(keyword_text, declaration);
+            } else {
+                declaration = found->second;
+            }
+            if (!lexer.next<index_tokens>().is_operator('(')) {
+                failure = "expected ( after the type";
+                failure_offset = keyword.start_pos;
+                break;
+            }
+            const uint64_t attributes_offset = reader.tell();
+            const bool bypassed = declaration != nullptr && bypassed_types[declaration->index_in_schema()];
+            const bool indexed = declaration != nullptr && !bypassed;
+            const uint16_t type_index = indexed ? (uint16_t)declaration->index_in_schema() : 0;
+            int depth = 1;
+            int attribute = 0;
+            bool first_value = true;
+            size_t guid_begin = 0, guid_end = 0;
+            while (depth > 0) {
+                token t = lexer.next<index_tokens>();
+                if (!t) {
+                    failure = "file ends inside an instance";
+                    failure_offset = attributes_offset;
+                    break;
+                }
+                if (t.is_operator()) {
+                    if (t.value_char == '(') {
+                        ++depth;
+                    } else if (t.value_char == ')') {
+                        --depth;
+                    } else if (t.value_char == ',' && depth == 1) {
+                        ++attribute;
+                    } else if (t.value_char == ';') {
+                        failure = "; inside an instance";
+                        failure_offset = t.start_pos;
+                        break;
+                    }
+                } else if (t.is_identifier()) {
+                    if (indexed) {
+                        byref_excl_.add((uint32_t)t.as_identifier(), name, type_index, attribute);
+                    }
+                } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) {
+                    guid_begin = t.start_pos + 1;
+                    guid_end = reader.tell() - 1;
+                }
+                if (depth == 1) {
+                    first_value = false;
+                }
+                lexer.reset_pool();
+            }
+            if (failure != nullptr) {
+                break;
+            }
+            if (!lexer.next<index_tokens>().is_operator(';')) {
+                failure = "expected ; after )";
+                failure_offset = reader.tell();
+                break;
+            }
+            if (bypassed) {
+                lazy_bypassed_.push_back(name);
+            }
+            if (!indexed) {
+                continue;
+            }
+            auto data = ifcopenshell::make_pointer_type<instance_data>(file, declaration, name, instance_data::lazy_tag{});
+            if (!byid_.insert({name, data}).second) {
+                logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Overwriting instance with name #" + std::to_string(name));
+                byid_.erase(name);
+                byid_.insert({name, data});
+            }
+            lazy_offsets_.push_back({name, attributes_offset});
+            express::base instance(data);
+            bytype_excl_[declaration].push_back(instance);
+            max_id = (std::max)(max_id, (unsigned int)name);
+            if (guid_end > guid_begin && declaration->is(*ifcroot)) {
+                std::string guid;
+                guid.reserve(guid_end - guid_begin);
+                for (size_t at = guid_begin; at < guid_end; ++at) {
+                    guid.push_back(reader.get(at));
+                }
+                if (guid.find('\\') != std::string::npos || guid.find("''") != std::string::npos) {
+                    guid = ifcopenshell::decode_spf_string(guid);
+                }
+                std::array<char, 22> key;
+                if (guid_key(guid, key)) {
+                    if (byguid_.count(key) != 0) {
+                        logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Instance encountered with non-unique GlobalId " + guid);
+                    }
+                    byguid_[key] = instance;
+                }
+            }
+        }
+    } catch (const invalid_token_exception&) {
+        failure = "invalid token";
+        failure_offset = reader.tell();
+    }
+    if (failure != nullptr) {
+        logger_.get().message(ifcopenshell::logger::LOG_NOTICE, std::string("Lazy loading not possible (") + failure + " at offset " + std::to_string(failure_offset) + "), parsing the file in full");
+        return false;
+    }
+
+    std::sort(lazy_bypassed_.begin(), lazy_bypassed_.end());
+    std::sort(lazy_offsets_.begin(), lazy_offsets_.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
+    byref_excl_.sort();
+    good_ = file_open_status::SUCCESS;
+    return true;
+}
+
 template <typename Reader>
 void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass) {
     schema = nullptr;
@@ -3732,6 +4013,7 @@ instance_data::instance_data(const instance_data& data)
 
 attribute_value instance_data::get_attribute_value(size_t index) const
 {
+    ensure_loaded();
     if (storage_) {
         return attribute_value(&*storage_, (uint8_t)index);
     } else {

src/ifcparse/storage.h

@@ -585,6 +585,25 @@ namespace ifcopenshell {
             // on; streaming consumers of references() leave it off.
             bool resolve_references_in_place = false;
 
+            // Lazy loading (index_lazily): the file was read once through the
+            // tokenizer's index policy to build the instance shells, the
+            // inverse index, the GlobalId map and the by-type lists, and each
+            // instance's attributes are parsed from the retained paged source
+            // the first time they are accessed (instance_data::ensure_loaded).
+            // The offset of each instance's attribute list lives here, not in
+            // the instance, so a full parse pays nothing for it. Inverses were
+            // registered by the index, so materialisation must not register
+            // them again. Materialising from several threads at once is not
+            // safe.
+            struct lazy_source;
+            bool lazy_ = false;
+            bool register_inverses_ = true;
+            std::unique_ptr<lazy_source, void (*)(lazy_source*)> lazy_source_{nullptr, nullptr};
+            std::vector<unsigned> lazy_bypassed_;
+            std::vector<std::pair<uint32_t, uint64_t>> lazy_offsets_;
+            bool index_lazily(const std::string& path, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& types_to_bypass);
+            void materialize(instance_data* data);
+
             typedef std::map<const ifcopenshell::declaration*, std::vector<express::base>> entities_by_type;
             typedef std::unordered_map<uint32_t, shared_pointer_type> entity_instance_by_name_storage;
             typedef map_transformer<entity_instance_by_name_storage, std::function<express::base(shared_pointer_type)>> entity_instance_by_name;
@@ -658,6 +677,13 @@ namespace ifcopenshell {
             shared_pointer_type load(ifcopenshell::spf_lexer<Reader>* tokens, std::optional<size_t> entity_instance_name, const ifcopenshell::declaration* declaration, const ifcopenshell::entity* entity, int attribute_index = -1, bool coerce_attribute_count = true);
             template <typename Reader>
             void try_read_semicolon(ifcopenshell::spf_lexer<Reader>* tokens) const;
+            // The attribute-reading half of load(): the tokens after the
+            // opening parenthesis into a fresh attribute array. Storage is
+            // always in_memory_attribute_storage; it is a template parameter
+            // only because that type is defined in a header that includes
+            // this one.
+            template <typename Reader, typename Storage>
+            Storage load_attributes(ifcopenshell::spf_lexer<Reader>* tokens, std::optional<size_t> entity_instance_name, const ifcopenshell::declaration* declaration, const ifcopenshell::entity* entity, int attribute_index = -1);
             // Replaces the names left in `data`'s attribute slots by in-place
             // reference storage with the instances they name; a name that is
             // missing or bypassed becomes null in a scalar and is dropped

src/ifcparse/tests/test_ifcopenshell_parse.cpp

@@ -434,3 +434,68 @@ TEST_CASE("References to bypassed instances are dropped from slots and from mixe
     CHECK(control_points[0].empty());
     CHECK(control_points[1].empty());
 }
+
+namespace {
+void check_lazy_matches_strict(const std::string& path) {
+    ifcopenshell::file strict(path);
+    REQUIRE(strict.good());
+    ifcopenshell::file lazy(ifcopenshell::uninitialized_tag{});
+    lazy.lazy_loading(true);
+    REQUIRE(lazy.initialize(path));
+    REQUIRE(lazy.lazy_loading());
+    REQUIRE(lazy.schema() == strict.schema());
+
+    size_t strict_count = 0;
+    for (auto it = strict.begin(); it != strict.end(); ++it) {
+        const express::base a = it->second;
+        const express::base b = lazy.instance_by_id((int)a.id());
+        REQUIRE(b);
+        REQUIRE(&b.declaration() == &a.declaration());
+        REQUIRE(lazy.instances_by_reference((int)a.id()).size() == strict.instances_by_reference((int)a.id()).size());
+        std::ostringstream sa, sb;
+        a.to_string(sa);
+        b.to_string(sb);
+        REQUIRE(sb.str() == sa.str());
+        ++strict_count;
+    }
+    size_t lazy_count = 0;
+    for (auto it = lazy.begin(); it != lazy.end(); ++it) {
+        ++lazy_count;
+    }
+    CHECK(lazy_count == strict_count);
+
+    for (const auto& rooted : strict.instances_by_type("IfcRoot")) {
+        const std::string guid = rooted.get_attribute_value(0);
+        REQUIRE(lazy.instance_by_guid(guid).id() == rooted.id());
+    }
+}
+}
+
+TEST_CASE("Lazy loading yields the same instances, attributes, inverses and GlobalIds as a full parse", "[ifcparse]") {
+    check_lazy_matches_strict(std::string(IFCOPENSHELL_TEST_FIXTURES) + "/ColumnPSetsOfSets.ifc");
+
+    const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_lazy_loading_test.ifc";
+    {
+        std::ofstream out(path);
+        out << reference_resolution_spf;
+    }
+    check_lazy_matches_strict(path.string());
+    std::filesystem::remove(path);
+}
+
+TEST_CASE("Lazy loading falls back to the full parser on syntax the index pass does not handle", "[ifcparse]") {
+    const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_lazy_fallback_test.ifc";
+    {
+        std::ofstream out(path);
+        // A stray keyword between instances is something the full parser skips past, but the index pass gives up on.
+        std::string spf(reference_resolution_spf);
+        spf.replace(spf.find("#8=IFCWALL"), 0, "STRAY;\n");
+        out << spf;
+    }
+    ifcopenshell::file lazy(ifcopenshell::uninitialized_tag{});
+    lazy.lazy_loading(true);
+    lazy.initialize(path.string());
+    std::filesystem::remove(path);
+    CHECK_FALSE(lazy.lazy_loading());
+    CHECK(lazy.instance_by_id(4));
+}

src/serializers/rocks_db_serializer.cpp

@@ -150,6 +150,7 @@ void RocksDbSerializer::write_streaming_() {
 
 			std::vector<express::base> simple_type_instances;
 
+			data->ensure_loaded();
 			for (size_t i = 0; i < data->storage_->size(); i++) {
 				auto val = data->get_attribute_value(i);
 				val.apply_visitor([&](const auto& t) {
@@ -291,6 +292,7 @@ void RocksDbSerializer::write_streaming_() {
 				}
 				if (decl->is(*ifcroot_type)) {
 					// @nb attribute counts are not coerced, so the attribute may be absent
+					data->ensure_loaded();
 					const bool has_guid = data->storage_->size() > 0 && data->get_attribute_value(0).type() == ifcopenshell::Argument_STRING;
 					if (has_guid) {
 						size_t v = name;