From 43686bde853d39b6335d934ec1b8686177f4bf55 Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Mon, 14 Sep 2026 07:32:12 +1000 Subject: [PATCH] ifcparse: opt in to running the full parse through the paged reader file::paged_reading(true), set before initialize(), runs the full parse (serial or parallel) through the paged reader with 64 KB pages and a 4 MB cache instead of reading the whole file into memory; the whole file is then never held. Every stage already reads through the reader, so nothing else changes. The equality test now runs the same file paged, serially and with five workers each holding its own page cache. TXG 58 MB / 210_King 147 MB / OKgate22 231 MB: one thread 1.09 / 2.89 / 5.25 s against 1.07 / 2.75 / 5.12 s in memory, twelve threads 0.48 / 1.27 / 1.99 s against 0.44 / 1.24 / 2.01 s; peak memory 311 / 727 / 1119 MB against 365 / 871 / 1347 MB, that is, down by the size of the file. Whether this should become the default is a decision the numbers on the PR are meant to inform. This commit was written by an AI coding tool and has not been verified by a human. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL --- .../ifcopenshell/ifcopenshell_wrapper.pyi | 7 +++++++ src/ifcparse/file.h | 7 +++++++ src/ifcparse/parse.cpp | 10 ++++++++-- .../tests/test_ifcopenshell_parse.cpp | 20 ++++++++++++++++++- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi index 877ed50486..cbfec471a5 100644 --- a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi +++ b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi @@ -928,12 +928,19 @@ class file(file_mixin): def parse_threads(self, *args: int) -> int: """Get, or with an argument set, the number of threads ``initialize()`` parses instances with; 0 uses one per core (capped at 16) or honours ``IFCOPENSHELL_PARSE_THREADS``.""" ... + def effective_parse_threads(self) -> int: """The thread count ``initialize()`` will use given ``parse_threads()`` and the environment.""" ... + + def paged_reading(self, *args: bool) -> bool: + """Get, or with an argument set, whether ``initialize()`` reads the file through the paged reader instead of loading it whole. Set before ``initialize()``.""" + ... + def lazy_loading(self, *args: bool) -> bool: """Get, or with an argument set, whether ``initialize()`` indexes the file with one pass and parses each instance's attributes on first access. Set before ``initialize()``.""" ... + def get_inverse_indices_by_id(self, instance_id: int) -> tuple[int, ...]: ... def _get_inverse(self, e: entity_instance) -> tuple[entity_instance, ...]: ... def _get_inverse_indices(self, *args: Union[entity_instance, int]) -> tuple[int, ...]: diff --git a/src/ifcparse/file.h b/src/ifcparse/file.h index 92c5d021f8..f5f50b5da6 100644 --- a/src/ifcparse/file.h +++ b/src/ifcparse/file.h @@ -215,6 +215,7 @@ public: private: bool lazy_loading_ = false; unsigned parse_threads_ = 0; + bool paged_reading_ = false; file_open_status good_ = file_open_status::SUCCESS; std::reference_wrapper logger_; @@ -296,6 +297,12 @@ public: void parse_threads(unsigned value) { parse_threads_ = value; } unsigned parse_threads() const { return parse_threads_; } unsigned effective_parse_threads() const; + // Read the file through the paged reader (64 KB pages, 4 MB cache) + // instead of loading it into memory as a whole. Set before + // initialize(). Applies to the full parse; lazy loading always reads + // in pages. + void paged_reading(bool value) { paged_reading_ = value; } + bool paged_reading() const { return paged_reading_; } #ifdef USE_MMAP bool initialize(const std::string& path, bool use_mmap); #endif diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index 0c74b107d3..e27941d373 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -1948,9 +1948,14 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r } } if (!indexed) { - file_reader s(path); std::get(storage_).parse_threads = effective_parse_threads(); - std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); + if (paged_reading_) { + file_reader s(path, 64 << 10, 64); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); + } else { + file_reader s(path); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); + } } if ((good_ = std::get(storage_).good_)) { @@ -3380,6 +3385,7 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con } template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); +template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); #ifdef USE_MMAP template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); diff --git a/src/ifcparse/tests/test_ifcopenshell_parse.cpp b/src/ifcparse/tests/test_ifcopenshell_parse.cpp index 6663cb2b45..1ecfaa9a95 100644 --- a/src/ifcparse/tests/test_ifcopenshell_parse.cpp +++ b/src/ifcparse/tests/test_ifcopenshell_parse.cpp @@ -508,7 +508,7 @@ TEST_CASE("Lazy loading passes over a stray keyword like the full parser and fal CHECK(lazy.instance_by_id(8)); } -TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and GlobalIds as serial parsing, comments in DATA included", "[ifcparse]") { +TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inverses and GlobalIds as serial in-memory parsing, comments in DATA included", "[ifcparse]") { // The fixture is small, so the threshold would keep it serial; write a // file big enough to be chunked by repeating its DATA section under new // names, with a comment and a string holding '/*' between the copies. @@ -560,6 +560,16 @@ TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and ifcopenshell::file parallel(ifcopenshell::uninitialized_tag{}); parallel.parse_threads(5); REQUIRE(parallel.initialize(path.string())); + // The same file through the paged reader, serially and with 5 workers, + // each with its own page cache. + ifcopenshell::file paged(ifcopenshell::uninitialized_tag{}); + paged.paged_reading(true); + paged.parse_threads(1); + REQUIRE(paged.initialize(path.string())); + ifcopenshell::file paged_parallel(ifcopenshell::uninitialized_tag{}); + paged_parallel.paged_reading(true); + paged_parallel.parse_threads(5); + REQUIRE(paged_parallel.initialize(path.string())); std::filesystem::remove(path); size_t count = 0; @@ -573,6 +583,14 @@ TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and a.to_string(sa); b.to_string(sb); REQUIRE(sb.str() == sa.str()); + for (ifcopenshell::file* other : {&paged, &paged_parallel}) { + const express::base c = other->instance_by_id((int)a.id()); + REQUIRE(c); + std::ostringstream sc; + c.to_string(sc); + REQUIRE(sc.str() == sa.str()); + REQUIRE(other->instances_by_reference((int)a.id()).size() == serial.instances_by_reference((int)a.id()).size()); + } ++count; } size_t parallel_count = 0;