diff --git a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi index 877ed50486..cbfec471a5 100644 --- a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi +++ b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi @@ -928,12 +928,19 @@ class file(file_mixin): def parse_threads(self, *args: int) -> int: """Get, or with an argument set, the number of threads ``initialize()`` parses instances with; 0 uses one per core (capped at 16) or honours ``IFCOPENSHELL_PARSE_THREADS``.""" ... + def effective_parse_threads(self) -> int: """The thread count ``initialize()`` will use given ``parse_threads()`` and the environment.""" ... + + def paged_reading(self, *args: bool) -> bool: + """Get, or with an argument set, whether ``initialize()`` reads the file through the paged reader instead of loading it whole. Set before ``initialize()``.""" + ... + def lazy_loading(self, *args: bool) -> bool: """Get, or with an argument set, whether ``initialize()`` indexes the file with one pass and parses each instance's attributes on first access. Set before ``initialize()``.""" ... + def get_inverse_indices_by_id(self, instance_id: int) -> tuple[int, ...]: ... def _get_inverse(self, e: entity_instance) -> tuple[entity_instance, ...]: ... def _get_inverse_indices(self, *args: Union[entity_instance, int]) -> tuple[int, ...]: diff --git a/src/ifcparse/file.h b/src/ifcparse/file.h index 92c5d021f8..f5f50b5da6 100644 --- a/src/ifcparse/file.h +++ b/src/ifcparse/file.h @@ -215,6 +215,7 @@ public: private: bool lazy_loading_ = false; unsigned parse_threads_ = 0; + bool paged_reading_ = false; file_open_status good_ = file_open_status::SUCCESS; std::reference_wrapper logger_; @@ -296,6 +297,12 @@ public: void parse_threads(unsigned value) { parse_threads_ = value; } unsigned parse_threads() const { return parse_threads_; } unsigned effective_parse_threads() const; + // Read the file through the paged reader (64 KB pages, 4 MB cache) + // instead of loading it into memory as a whole. Set before + // initialize(). Applies to the full parse; lazy loading always reads + // in pages. + void paged_reading(bool value) { paged_reading_ = value; } + bool paged_reading() const { return paged_reading_; } #ifdef USE_MMAP bool initialize(const std::string& path, bool use_mmap); #endif diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index 0c74b107d3..e27941d373 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -1948,9 +1948,14 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r } } if (!indexed) { - file_reader s(path); std::get(storage_).parse_threads = effective_parse_threads(); - std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); + if (paged_reading_) { + file_reader s(path, 64 << 10, 64); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); + } else { + file_reader s(path); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); + } } if ((good_ = std::get(storage_).good_)) { @@ -3380,6 +3385,7 @@ void ifcopenshell::impl::in_memory_file_storage::read_from_stream(Reader* s, con } template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); +template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); #ifdef USE_MMAP template void ifcopenshell::impl::in_memory_file_storage::read_from_stream(file_reader* s, const ifcopenshell::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); diff --git a/src/ifcparse/tests/test_ifcopenshell_parse.cpp b/src/ifcparse/tests/test_ifcopenshell_parse.cpp index 6663cb2b45..1ecfaa9a95 100644 --- a/src/ifcparse/tests/test_ifcopenshell_parse.cpp +++ b/src/ifcparse/tests/test_ifcopenshell_parse.cpp @@ -508,7 +508,7 @@ TEST_CASE("Lazy loading passes over a stray keyword like the full parser and fal CHECK(lazy.instance_by_id(8)); } -TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and GlobalIds as serial parsing, comments in DATA included", "[ifcparse]") { +TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inverses and GlobalIds as serial in-memory parsing, comments in DATA included", "[ifcparse]") { // The fixture is small, so the threshold would keep it serial; write a // file big enough to be chunked by repeating its DATA section under new // names, with a comment and a string holding '/*' between the copies. @@ -560,6 +560,16 @@ TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and ifcopenshell::file parallel(ifcopenshell::uninitialized_tag{}); parallel.parse_threads(5); REQUIRE(parallel.initialize(path.string())); + // The same file through the paged reader, serially and with 5 workers, + // each with its own page cache. + ifcopenshell::file paged(ifcopenshell::uninitialized_tag{}); + paged.paged_reading(true); + paged.parse_threads(1); + REQUIRE(paged.initialize(path.string())); + ifcopenshell::file paged_parallel(ifcopenshell::uninitialized_tag{}); + paged_parallel.paged_reading(true); + paged_parallel.parse_threads(5); + REQUIRE(paged_parallel.initialize(path.string())); std::filesystem::remove(path); size_t count = 0; @@ -573,6 +583,14 @@ TEST_CASE("Parallel parsing yields the same instances, attributes, inverses and a.to_string(sa); b.to_string(sb); REQUIRE(sb.str() == sa.str()); + for (ifcopenshell::file* other : {&paged, &paged_parallel}) { + const express::base c = other->instance_by_id((int)a.id()); + REQUIRE(c); + std::ostringstream sc; + c.to_string(sc); + REQUIRE(sc.str() == sa.str()); + REQUIRE(other->instances_by_reference((int)a.id()).size() == serial.instances_by_reference((int)a.id()).size()); + } ++count; } size_t parallel_count = 0;