ifcparse: opt in to running the full parse through the paged reader

file::paged_reading(true), set before initialize(), runs the full parse
(serial or parallel) through the paged reader with 64 KB pages and a
4 MB cache instead of reading the whole file into memory; the whole
file is then never held. Every stage already reads through the reader,
so nothing else changes. The equality test now runs the same file paged,
serially and with five workers each holding its own page cache.

TXG 58 MB / 210_King 147 MB / OKgate22 231 MB: one thread 1.09 / 2.89 /
5.25 s against 1.07 / 2.75 / 5.12 s in memory, twelve threads 0.48 /
1.27 / 1.99 s against 0.44 / 1.24 / 2.01 s; peak memory 311 / 727 /
1119 MB against 365 / 871 / 1347 MB, that is, down by the size of the
file. Whether this should become the default is a decision the numbers
on the PR are meant to inform.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL

src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi

@@ -928,12 +928,19 @@ class file(file_mixin):
     def parse_threads(self, *args: int) -> int:
         """Get, or with an argument set, the number of threads ``initialize()`` parses instances with; 0 uses one per core (capped at 16) or honours ``IFCOPENSHELL_PARSE_THREADS``."""
         ...
+
     def effective_parse_threads(self) -> int:
         """The thread count ``initialize()`` will use given ``parse_threads()`` and the environment."""
         ...
+
+    def paged_reading(self, *args: bool) -> bool:
+        """Get, or with an argument set, whether ``initialize()`` reads the file through the paged reader instead of loading it whole. Set before ``initialize()``."""
+        ...
+
     def lazy_loading(self, *args: bool) -> bool:
         """Get, or with an argument set, whether ``initialize()`` indexes the file with one pass and parses each instance's attributes on first access. Set before ``initialize()``."""
         ...
+
     def get_inverse_indices_by_id(self, instance_id: int) -> tuple[int, ...]: ...
     def _get_inverse(self, e: entity_instance) -> tuple[entity_instance, ...]: ...
     def _get_inverse_indices(self, *args: Union[entity_instance, int]) -> tuple[int, ...]:

src/ifcparse/file.h

@@ -215,6 +215,7 @@ public:
   private:
     bool lazy_loading_ = false;
     unsigned parse_threads_ = 0;
+    bool paged_reading_ = false;
     file_open_status good_ = file_open_status::SUCCESS;
     std::reference_wrapper<ifcopenshell::logger> logger_;
 
@@ -296,6 +297,12 @@ public:
     void parse_threads(unsigned value) { parse_threads_ = value; }
     unsigned parse_threads() const { return parse_threads_; }
     unsigned effective_parse_threads() const;
+    // Read the file through the paged reader (64 KB pages, 4 MB cache)
+    // instead of loading it into memory as a whole. Set before
+    // initialize(). Applies to the full parse; lazy loading always reads
+    // in pages.
+    void paged_reading(bool value) { paged_reading_ = value; }
+    bool paged_reading() const { return paged_reading_; }
 #ifdef USE_MMAP
     bool initialize(const std::string& path, bool use_mmap);
 #endif

src/ifcparse/parse.cpp

@@ -1948,9 +1948,14 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r
             }
         }
         if (!indexed) {
-            file_reader<full_buffer_impl> s(path);
             std::get<impl::in_memory_file_storage>(storage_).parse_threads = effective_parse_threads();
-            std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+            if (paged_reading_) {
+                file_reader<paged_file_impl> s(path, 64 << 10, 64);
+                std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+            } else {
+                file_reader<full_buffer_impl> s(path);
+                std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
+            }
         }
 
         if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
@@ -3380,6 +3385,7 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con
 }
 
 template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader<full_buffer_impl>* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
+template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader<paged_file_impl>* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
 template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader<pushed_sequential_impl>* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
 #ifdef USE_MMAP
 template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader<mmap_impl>* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);

src/ifcparse/tests/test_ifcopenshell_parse.cpp

@@ -508,7 +508,7 @@ TEST_CASE("Lazy loading passes over a stray keyword like the full parser and fal
     CHECK(lazy.instance_by_id(8));
 }
 
-TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and GlobalIds as serial parsing, comments in DATA included", "[ifcparse]") {
+TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inverses and GlobalIds as serial in-memory parsing, comments in DATA included", "[ifcparse]") {
     // The fixture is small, so the threshold would keep it serial; write a
     // file big enough to be chunked by repeating its DATA section under new
     // names, with a comment and a string holding '/*' between the copies.
@@ -560,6 +560,16 @@ TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and
     ifcopenshell::file parallel(ifcopenshell::uninitialized_tag{});
     parallel.parse_threads(5);
     REQUIRE(parallel.initialize(path.string()));
+    // The same file through the paged reader, serially and with 5 workers,
+    // each with its own page cache.
+    ifcopenshell::file paged(ifcopenshell::uninitialized_tag{});
+    paged.paged_reading(true);
+    paged.parse_threads(1);
+    REQUIRE(paged.initialize(path.string()));
+    ifcopenshell::file paged_parallel(ifcopenshell::uninitialized_tag{});
+    paged_parallel.paged_reading(true);
+    paged_parallel.parse_threads(5);
+    REQUIRE(paged_parallel.initialize(path.string()));
     std::filesystem::remove(path);
 
     size_t count = 0;
@@ -573,6 +583,14 @@ TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and
         a.to_string(sa);
         b.to_string(sb);
         REQUIRE(sb.str() == sa.str());
+        for (ifcopenshell::file* other : {&paged, &paged_parallel}) {
+            const express::base c = other->instance_by_id((int)a.id());
+            REQUIRE(c);
+            std::ostringstream sc;
+            c.to_string(sc);
+            REQUIRE(sc.str() == sa.str());
+            REQUIRE(other->instances_by_reference((int)a.id()).size() == serial.instances_by_reference((int)a.id()).size());
+        }
         ++count;
     }
     size_t parallel_count = 0;