From f24e9a6bca96c2b032d734102c00e24c77e5021e Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Mon, 14 Sep 2026 18:47:06 +1000 Subject: [PATCH] ifcparse: build the lazy index in parallel On a file large enough the DATA section is split at the same boundaries the parallel parse uses (chunk_bounds(), now shared) and each chunk is indexed by its own worker with its own paged reader, lexer, shells, offsets, GlobalIds and inverse records; the results are merged in file order, so instance order, GlobalId precedence and inverse records are identical to the serial index. The serial index is the same code run on one chunk. The name table is reserved before the merge, which also helps the serial case. The default thread count is the one the full parse uses. TXG 58 MB / 210_King 147 MB / OKgate22 231 MB, lazy open: 12 threads 0.37 / 1.01 / 1.34 s against 0.55 / 1.59 / 2.11 s on one thread and 0.44 / 1.17 / 1.91 s for the default (parallel full) open; memory after the open +15 to +50 MB at 12 threads for the workers' page caches. The equality test now opens the 12 MB replicated fixture lazily with five workers and compares it instance by instance with the serial parse. This commit was written by an AI coding tool and has not been verified by a human. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL --- src/ifcparse/parse.cpp | 403 +++++++++++------- .../tests/test_ifcopenshell_parse.cpp | 9 +- 2 files changed, 250 insertions(+), 162 deletions(-) diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index ff362ad53e..8e63abeb1b 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -1946,6 +1946,7 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r if (ty == FT_IFCSPF) { storage_.emplace<1>(this, logger_.get()); header_.reset(new spf_header(this, &logger_.get())); + std::get(storage_).parse_threads = effective_parse_threads(); bool indexed = false; if (lazy_loading_) { indexed = std::get(storage_).index_lazily(path, schema_, max_id_, types_to_bypass_loading_); @@ -1959,7 +1960,6 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r } } if (!indexed) { - std::get(storage_).parse_threads = effective_parse_threads(); if (paged_reading_) { file_reader s(path, 64 << 10, 64); std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); @@ -2693,6 +2693,104 @@ void for_each_instance_header(Reader& reader, spf_lexer& lexer, size_t e } +namespace { +// Matches a literal byte by byte, across spans. +class literal_matcher { + const char* literal_; + size_t length_; + size_t matched_ = 0; + public: + explicit literal_matcher(const char* literal) + : literal_(literal), length_(std::strlen(literal)) {} + // True on the byte that completes the literal. + bool feed(char c) { + if (c == literal_[matched_]) { + if (++matched_ == length_) { + matched_ = 0; + return true; + } + } else { + matched_ = c == literal_[0] ? 1 : 0; + } + return false; + } +}; + +// The split points for a parallel pass over the DATA section: the start +// of the section, then nearest each nominal split point an instance +// boundary, then the end of the section. Fewer than three entries means +// the file cannot be split. Shared by the parallel parse and the +// parallel lazy index. +template +std::vector chunk_bounds(const Reader& source, unsigned threads) { + const size_t n = source.size(); + // One pass over the bytes finds the DATA section and, nearest each + // nominal split point, an instance boundary: a '#' that starts a + // line outside any string and any comment. This is the one place + // that looks at raw bytes instead of tokens, because tokenizing the + // file serially to find the split points would leave nothing to + // parallelise. It applies three rules only: a string starts and + // ends at a quote (a doubled quote closes and reopens, which comes + // to the same thing) and cannot span a line; a comment runs from + // /* to */. Getting a string's end wrong can only lose a candidate + // boundary, never accept a wrong one, since no string contains a + // newline. + literal_matcher data_matcher("\nDATA;"), endsec_matcher("\nENDSEC"); + size_t data_begin = 0, data_end = 0; + bool in_string = false, in_comment = false, newline = false; + char previous = 0; + std::vector bounds; + size_t next_split = 0; + source.for_each_span(0, n, [&](const char* data, size_t length, size_t offset) { + for (size_t i = 0; i < length; ++i) { + const char c = data[i]; + const size_t at = offset + i; + if (data_begin == 0) { + if (data_matcher.feed(c)) { + data_begin = at + 1; + bounds.push_back(data_begin); + next_split = data_begin + (n - data_begin) / threads; + } + continue; + } + if (data_end != 0) { + return; + } + if (in_comment) { + if (previous == '*' && c == '/') { + in_comment = false; + } + } else if (in_string) { + if (c == '\'' || c == '\n') { + in_string = false; + } + } else if (c == '\'') { + in_string = true; + } else if (previous == '/' && c == '*') { + in_comment = true; + } else if (endsec_matcher.feed(c)) { + data_end = at + 1 - 7; + return; + } else if (newline && c == '#' && at >= next_split && bounds.size() < threads) { + bounds.push_back(at); + next_split = data_begin + (n - data_begin) * bounds.size() / threads; + } + newline = c == '\n'; + previous = c; + } + }); + if (data_begin == 0) { + return {}; + } + if (data_end == 0) { + data_end = n; + } + bounds.push_back(data_end); + return bounds; +} + +} + struct ifcopenshell::impl::in_memory_file_storage::lazy_source { file_reader reader; spf_lexer> lexer; @@ -2781,96 +2879,161 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& this->schema = schema; resolve_references_in_place = true; lazy_ = true; - byref_excl_.reserve(reader.size() / 32); - // One pass over the DATA section with the tokenizer's index policy: - // the instance headers through the shared loop, then the attribute list - // as tokens with only the parentheses, commas and names looked at. - // Nothing is decoded. A token the tokenizer rejects, or a structure the - // loop below does not expect, stops the index and the caller parses in - // full. - const char* failure = nullptr; - size_t failure_offset = 0; - try { - for_each_instance_header(reader, lexer, reader.size(), schema, bypassed_types, lazy_bypassed_, logger_.get(), [&](uint32_t name, const ifcopenshell::declaration* declaration, size_t) { - const uint64_t attributes_offset = reader.tell(); - const uint16_t type_index = (uint16_t)declaration->index_in_schema(); - int depth = 1; - int attribute = 0; - bool first_value = true; - size_t guid_begin = 0, guid_end = 0; - while (depth > 0) { - token t = lexer.next(); - if (!t) { - failure = "file ends inside an instance"; - failure_offset = attributes_offset; - return false; - } - if (t.is_operator()) { - if (t.value_char == '(') { - ++depth; - } else if (t.value_char == ')') { - --depth; - } else if (t.value_char == ',' && depth == 1) { - ++attribute; - } else if (t.value_char == ';') { - failure = "; inside an instance"; - failure_offset = t.start_pos; + // One pass over the DATA section with the tokenizer's index policy: the + // instance headers through the shared loop, then the attribute list as + // tokens with only the parentheses, commas and names looked at. Nothing + // is decoded. On a file large enough the section is split at the same + // boundaries the parallel parse uses and each chunk is indexed by its + // own worker with its own reader, lexer, shells and inverse records, + // merged in file order; the serial case is the same code on one chunk. + // A token the tokenizer rejects, or a structure the loop does not + // expect, stops the index and the caller parses in full. + struct index_output { + std::vector shells; + std::vector> offsets; + std::vector> guids; // shell index, raw text + std::vector bypassed; + entities_by_ref inverses; + const char* failure = nullptr; + size_t failure_offset = 0; + std::exception_ptr error; + }; + const auto index_chunk = [&](file_reader& chunk_reader, spf_lexer>& chunk_lexer, size_t end, index_output& out) { + try { + for_each_instance_header(chunk_reader, chunk_lexer, end, schema, bypassed_types, out.bypassed, logger_.get(), [&](uint32_t name, const ifcopenshell::declaration* declaration, size_t) { + const uint64_t attributes_offset = chunk_reader.tell(); + const uint16_t type_index = (uint16_t)declaration->index_in_schema(); + int depth = 1; + int attribute = 0; + bool first_value = true; + size_t guid_begin = 0, guid_end = 0; + while (depth > 0) { + token t = chunk_lexer.next(); + if (!t) { + out.failure = "file ends inside an instance"; + out.failure_offset = attributes_offset; return false; } - } else if (t.is_identifier()) { - byref_excl_.add((uint32_t)t.as_identifier(), name, type_index, attribute); - } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) { - guid_begin = t.start_pos + 1; - guid_end = reader.tell() - 1; + if (t.is_operator()) { + if (t.value_char == '(') { + ++depth; + } else if (t.value_char == ')') { + --depth; + } else if (t.value_char == ',' && depth == 1) { + ++attribute; + } else if (t.value_char == ';') { + out.failure = "; inside an instance"; + out.failure_offset = t.start_pos; + return false; + } + } else if (t.is_identifier()) { + out.inverses.add((uint32_t)t.as_identifier(), name, type_index, attribute); + } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) { + guid_begin = t.start_pos + 1; + guid_end = chunk_reader.tell() - 1; + } + if (depth == 1) { + first_value = false; + } + chunk_lexer.reset_pool(); } - if (depth == 1) { - first_value = false; + if (!chunk_lexer.next().is_operator(';')) { + out.failure = "expected ; after )"; + out.failure_offset = chunk_reader.tell(); + return false; } - lexer.reset_pool(); - } - if (!lexer.next().is_operator(';')) { - failure = "expected ; after )"; - failure_offset = reader.tell(); - return false; - } - auto data = ifcopenshell::make_pointer_type(file, declaration, name, instance_data::lazy_tag{}); + out.shells.push_back(ifcopenshell::make_pointer_type(file, declaration, name, instance_data::lazy_tag{})); + out.offsets.push_back({name, attributes_offset}); + if (guid_end > guid_begin && declaration->is(*ifcroot)) { + std::string guid; + guid.reserve(guid_end - guid_begin); + for (size_t at = guid_begin; at < guid_end; ++at) { + guid.push_back(chunk_reader.get(at)); + } + out.guids.push_back({out.shells.size() - 1, std::move(guid)}); + } + return true; + }); + } catch (const invalid_token_exception&) { + out.failure = "invalid token"; + out.failure_offset = chunk_reader.tell(); + } catch (...) { + out.error = std::current_exception(); + } + }; + + std::vector outputs; + constexpr size_t min_bytes_per_thread = 2u << 20; + const unsigned threads = (unsigned)std::min(parse_threads, std::max(1, reader.size() / min_bytes_per_thread)); + const std::vector bounds = threads > 1 ? chunk_bounds(reader, threads) : std::vector(); + if (bounds.size() >= 3) { + outputs.resize(bounds.size() - 1); + std::vector workers; + for (size_t k = 0; k + 1 < bounds.size(); ++k) { + outputs[k].inverses.reserve((bounds[k + 1] - bounds[k]) / 32); + workers.emplace_back([&, k]() { + file_reader chunk_reader = reader.reopen(); + chunk_reader.seek(bounds[k]); + spf_lexer> chunk_lexer(&chunk_reader, logger_.get()); + index_chunk(chunk_reader, chunk_lexer, bounds[k + 1], outputs[k]); + }); + } + for (auto& worker : workers) { + worker.join(); + } + } else { + outputs.resize(1); + outputs[0].inverses.reserve(reader.size() / 32); + index_chunk(reader, lexer, reader.size(), outputs[0]); + } + for (const auto& out : outputs) { + if (out.error) { + std::rethrow_exception(out.error); + } + if (out.failure != nullptr) { + logger_.get().message(ifcopenshell::logger::LOG_NOTICE, std::string("Lazy loading not possible (") + out.failure + " at offset " + std::to_string(out.failure_offset) + "), parsing the file in full"); + return false; + } + } + + // Merge in file order: what the serial index did per instance. + size_t shell_count = 0; + for (const auto& out : outputs) { + shell_count += out.shells.size(); + } + lazy_offsets_.reserve(shell_count); + byid_.reserve(byid_.size() + shell_count); + for (auto& out : outputs) { + for (const auto& data : out.shells) { + const uint32_t name = data->id(); if (!byid_.insert({name, data}).second) { logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Overwriting instance with name #" + std::to_string(name)); byid_.erase(name); byid_.insert({name, data}); } - lazy_offsets_.push_back({name, attributes_offset}); - express::base instance(data); - bytype_excl_[declaration].push_back(instance); + bytype_excl_[data->declaration()].push_back(express::base(data)); max_id = (std::max)(max_id, (unsigned int)name); - if (guid_end > guid_begin && declaration->is(*ifcroot)) { - std::string guid; - guid.reserve(guid_end - guid_begin); - for (size_t at = guid_begin; at < guid_end; ++at) { - guid.push_back(reader.get(at)); - } - if (guid.find('\\') != std::string::npos || guid.find("''") != std::string::npos) { - guid = ifcopenshell::decode_spf_string(guid); - } - std::array key; - if (guid_key(guid, key)) { - if (byguid_.count(key) != 0) { - logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Instance encountered with non-unique GlobalId " + guid); - } - byguid_[key] = instance; - } + } + lazy_offsets_.insert(lazy_offsets_.end(), out.offsets.begin(), out.offsets.end()); + for (auto& entry : out.guids) { + std::string& guid = entry.second; + if (guid.find('\\') != std::string::npos || guid.find("''") != std::string::npos) { + guid = ifcopenshell::decode_spf_string(guid); } - return true; - }); - } catch (const invalid_token_exception&) { - failure = "invalid token"; - failure_offset = reader.tell(); - } - if (failure != nullptr) { - logger_.get().message(ifcopenshell::logger::LOG_NOTICE, std::string("Lazy loading not possible (") + failure + " at offset " + std::to_string(failure_offset) + "), parsing the file in full"); - return false; + std::array key; + if (guid_key(guid, key)) { + if (byguid_.count(key) != 0) { + logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Instance encountered with non-unique GlobalId " + guid); + } + byguid_[key] = express::base(out.shells[entry.first]); + } + } + lazy_bypassed_.insert(lazy_bypassed_.end(), out.bypassed.begin(), out.bypassed.end()); + byref_excl_.append(std::move(out.inverses)); + std::vector().swap(out.shells); } + outputs.clear(); std::sort(lazy_bypassed_.begin(), lazy_bypassed_.end()); std::sort(lazy_offsets_.begin(), lazy_offsets_.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); @@ -2919,27 +3082,6 @@ void parse_chunk(const Reader& source, size_t begin, size_t end, const ifcopensh } } -// Matches a literal byte by byte, across spans. -class literal_matcher { - const char* literal_; - size_t length_; - size_t matched_ = 0; - public: - explicit literal_matcher(const char* literal) - : literal_(literal), length_(std::strlen(literal)) {} - // True on the byte that completes the literal. - bool feed(char c) { - if (c == literal_[matched_]) { - if (++matched_ == length_) { - matched_ = 0; - return true; - } - } else { - matched_ = c == literal_[0] ? 1 : 0; - } - return false; - } -}; } @@ -2955,68 +3097,7 @@ bool ifcopenshell::impl::in_memory_file_storage::read_instances_parallel(Reader* return false; } - // One pass over the bytes finds the DATA section and, nearest each - // nominal split point, an instance boundary: a '#' that starts a - // line outside any string and any comment. This is the one place - // that looks at raw bytes instead of tokens, because tokenizing the - // file serially to find the split points would leave nothing to - // parallelise. It applies three rules only: a string starts and - // ends at a quote (a doubled quote closes and reopens, which comes - // to the same thing) and cannot span a line; a comment runs from - // /* to */. Getting a string's end wrong can only lose a candidate - // boundary, never accept a wrong one, since no string contains a - // newline. - literal_matcher data_matcher("\nDATA;"), endsec_matcher("\nENDSEC"); - size_t data_begin = 0, data_end = 0; - bool in_string = false, in_comment = false, newline = false; - char previous = 0; - std::vector bounds; - size_t next_split = 0; - s->for_each_span(0, n, [&](const char* data, size_t length, size_t offset) { - for (size_t i = 0; i < length; ++i) { - const char c = data[i]; - const size_t at = offset + i; - if (data_begin == 0) { - if (data_matcher.feed(c)) { - data_begin = at + 1; - bounds.push_back(data_begin); - next_split = data_begin + (n - data_begin) / threads; - } - continue; - } - if (data_end != 0) { - return; - } - if (in_comment) { - if (previous == '*' && c == '/') { - in_comment = false; - } - } else if (in_string) { - if (c == '\'' || c == '\n') { - in_string = false; - } - } else if (c == '\'') { - in_string = true; - } else if (previous == '/' && c == '*') { - in_comment = true; - } else if (endsec_matcher.feed(c)) { - data_end = at + 1 - 7; - return; - } else if (newline && c == '#' && at >= next_split && bounds.size() < threads) { - bounds.push_back(at); - next_split = data_begin + (n - data_begin) * bounds.size() / threads; - } - newline = c == '\n'; - previous = c; - } - }); - if (data_begin == 0) { - return false; - } - if (data_end == 0) { - data_end = n; - } - bounds.push_back(data_end); + const std::vector bounds = chunk_bounds(*s, threads); if (bounds.size() < 3) { return false; } diff --git a/src/ifcparse/tests/test_ifcopenshell_parse.cpp b/src/ifcparse/tests/test_ifcopenshell_parse.cpp index 1ecfaa9a95..a290478417 100644 --- a/src/ifcparse/tests/test_ifcopenshell_parse.cpp +++ b/src/ifcparse/tests/test_ifcopenshell_parse.cpp @@ -570,6 +570,12 @@ TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inve paged_parallel.paged_reading(true); paged_parallel.parse_threads(5); REQUIRE(paged_parallel.initialize(path.string())); + // And the lazy index built by 5 workers. + ifcopenshell::file lazy_parallel(ifcopenshell::uninitialized_tag{}); + lazy_parallel.lazy_loading(true); + lazy_parallel.parse_threads(5); + REQUIRE(lazy_parallel.initialize(path.string())); + REQUIRE(lazy_parallel.lazy_loading()); std::filesystem::remove(path); size_t count = 0; @@ -583,7 +589,7 @@ TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inve a.to_string(sa); b.to_string(sb); REQUIRE(sb.str() == sa.str()); - for (ifcopenshell::file* other : {&paged, &paged_parallel}) { + for (ifcopenshell::file* other : {&paged, &paged_parallel, &lazy_parallel}) { const express::base c = other->instance_by_id((int)a.id()); REQUIRE(c); std::ostringstream sc; @@ -603,5 +609,6 @@ TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inve for (const auto& rooted : serial.instances_by_type("IfcRoot")) { const std::string guid = rooted.get_attribute_value(0); REQUIRE(parallel.instance_by_guid(guid).id() == serial.instance_by_guid(guid).id()); + REQUIRE(lazy_parallel.instance_by_guid(guid).id() == serial.instance_by_guid(guid).id()); } }