ifcparse: lazy loading, opt in with file::lazy_loading() / open(lazy=True)

A lazy open reads the DATA section once with the tokenizer's index
policy and builds what indexes the file: a shell per instance (name and
declaration, no attribute array), the complete inverse index with
attribute indices, the GlobalId map and the by-type lists. No attribute
value is decoded. The first time an instance's attributes are touched,
ensure_loaded() seeks the retained paged reader to the instance and runs
the same load_attributes() the full parse runs, with inverse registration
off, then resolves that instance's references from its own slots. A
modified instance is materialised first, so writing works.

There is no scanner of its own: the index pass consumes next<index_tokens>()
and counts parentheses and commas on the operator tokens; a keyword where
an instance should start, or a token the tokenizer rejects, stops the
index and the file is parsed in full. The offset of each instance's
attribute list is kept in one sorted vector that exists only in lazy
mode, so a full parse pays nothing for it. Materialising from several
threads at once is not safe.

TXG 58 MB / 210_King 147 MB / OKgate22 231 MB, single thread: lazy open
0.61 / 1.73 / 2.86 s against the full parse's 1.05 / 2.69 / 4.99 s, at
141 / 374 / 534 MB against 274 / 654 / 1036 MB; reading one attribute of
every instance afterwards costs a further 0.56 / 1.44 / 4.86 s.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL

src/ifcparse/tests/test_ifcopenshell_parse.cpp

@@ -434,3 +434,68 @@ TEST_CASE("References to bypassed instances are dropped from slots and from mixe
     CHECK(control_points[0].empty());
     CHECK(control_points[1].empty());
 }
+
+namespace {
+void check_lazy_matches_strict(const std::string& path) {
+    ifcopenshell::file strict(path);
+    REQUIRE(strict.good());
+    ifcopenshell::file lazy(ifcopenshell::uninitialized_tag{});
+    lazy.lazy_loading(true);
+    REQUIRE(lazy.initialize(path));
+    REQUIRE(lazy.lazy_loading());
+    REQUIRE(lazy.schema() == strict.schema());
+
+    size_t strict_count = 0;
+    for (auto it = strict.begin(); it != strict.end(); ++it) {
+        const express::base a = it->second;
+        const express::base b = lazy.instance_by_id((int)a.id());
+        REQUIRE(b);
+        REQUIRE(&b.declaration() == &a.declaration());
+        REQUIRE(lazy.instances_by_reference((int)a.id()).size() == strict.instances_by_reference((int)a.id()).size());
+        std::ostringstream sa, sb;
+        a.to_string(sa);
+        b.to_string(sb);
+        REQUIRE(sb.str() == sa.str());
+        ++strict_count;
+    }
+    size_t lazy_count = 0;
+    for (auto it = lazy.begin(); it != lazy.end(); ++it) {
+        ++lazy_count;
+    }
+    CHECK(lazy_count == strict_count);
+
+    for (const auto& rooted : strict.instances_by_type("IfcRoot")) {
+        const std::string guid = rooted.get_attribute_value(0);
+        REQUIRE(lazy.instance_by_guid(guid).id() == rooted.id());
+    }
+}
+}
+
+TEST_CASE("Lazy loading yields the same instances, attributes, inverses and GlobalIds as a full parse", "[ifcparse]") {
+    check_lazy_matches_strict(std::string(IFCOPENSHELL_TEST_FIXTURES) + "/ColumnPSetsOfSets.ifc");
+
+    const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_lazy_loading_test.ifc";
+    {
+        std::ofstream out(path);
+        out << reference_resolution_spf;
+    }
+    check_lazy_matches_strict(path.string());
+    std::filesystem::remove(path);
+}
+
+TEST_CASE("Lazy loading falls back to the full parser on syntax the index pass does not handle", "[ifcparse]") {
+    const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_lazy_fallback_test.ifc";
+    {
+        std::ofstream out(path);
+        // A stray keyword between instances is something the full parser skips past, but the index pass gives up on.
+        std::string spf(reference_resolution_spf);
+        spf.replace(spf.find("#8=IFCWALL"), 0, "STRAY;\n");
+        out << spf;
+    }
+    ifcopenshell::file lazy(ifcopenshell::uninitialized_tag{});
+    lazy.lazy_loading(true);
+    lazy.initialize(path.string());
+    std::filesystem::remove(path);
+    CHECK_FALSE(lazy.lazy_loading());
+    CHECK(lazy.instance_by_id(4));
+}