diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index ff362ad53e..8e63abeb1b 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -1946,6 +1946,7 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r if (ty == FT_IFCSPF) { storage_.emplace<1>(this, logger_.get()); header_.reset(new spf_header(this, &logger_.get())); + std::get(storage_).parse_threads = effective_parse_threads(); bool indexed = false; if (lazy_loading_) { indexed = std::get(storage_).index_lazily(path, schema_, max_id_, types_to_bypass_loading_); @@ -1959,7 +1960,6 @@ bool ifcopenshell::file::initialize(const std::string& path, filetype ty, bool r } } if (!indexed) { - std::get(storage_).parse_threads = effective_parse_threads(); if (paged_reading_) { file_reader s(path, 64 << 10, 64); std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); @@ -2693,6 +2693,104 @@ void for_each_instance_header(Reader& reader, spf_lexer& lexer, size_t e } +namespace { +// Matches a literal byte by byte, across spans. +class literal_matcher { + const char* literal_; + size_t length_; + size_t matched_ = 0; + public: + explicit literal_matcher(const char* literal) + : literal_(literal), length_(std::strlen(literal)) {} + // True on the byte that completes the literal. + bool feed(char c) { + if (c == literal_[matched_]) { + if (++matched_ == length_) { + matched_ = 0; + return true; + } + } else { + matched_ = c == literal_[0] ? 1 : 0; + } + return false; + } +}; + +// The split points for a parallel pass over the DATA section: the start +// of the section, then nearest each nominal split point an instance +// boundary, then the end of the section. Fewer than three entries means +// the file cannot be split. Shared by the parallel parse and the +// parallel lazy index. +template +std::vector chunk_bounds(const Reader& source, unsigned threads) { + const size_t n = source.size(); + // One pass over the bytes finds the DATA section and, nearest each + // nominal split point, an instance boundary: a '#' that starts a + // line outside any string and any comment. This is the one place + // that looks at raw bytes instead of tokens, because tokenizing the + // file serially to find the split points would leave nothing to + // parallelise. It applies three rules only: a string starts and + // ends at a quote (a doubled quote closes and reopens, which comes + // to the same thing) and cannot span a line; a comment runs from + // /* to */. Getting a string's end wrong can only lose a candidate + // boundary, never accept a wrong one, since no string contains a + // newline. + literal_matcher data_matcher("\nDATA;"), endsec_matcher("\nENDSEC"); + size_t data_begin = 0, data_end = 0; + bool in_string = false, in_comment = false, newline = false; + char previous = 0; + std::vector bounds; + size_t next_split = 0; + source.for_each_span(0, n, [&](const char* data, size_t length, size_t offset) { + for (size_t i = 0; i < length; ++i) { + const char c = data[i]; + const size_t at = offset + i; + if (data_begin == 0) { + if (data_matcher.feed(c)) { + data_begin = at + 1; + bounds.push_back(data_begin); + next_split = data_begin + (n - data_begin) / threads; + } + continue; + } + if (data_end != 0) { + return; + } + if (in_comment) { + if (previous == '*' && c == '/') { + in_comment = false; + } + } else if (in_string) { + if (c == '\'' || c == '\n') { + in_string = false; + } + } else if (c == '\'') { + in_string = true; + } else if (previous == '/' && c == '*') { + in_comment = true; + } else if (endsec_matcher.feed(c)) { + data_end = at + 1 - 7; + return; + } else if (newline && c == '#' && at >= next_split && bounds.size() < threads) { + bounds.push_back(at); + next_split = data_begin + (n - data_begin) * bounds.size() / threads; + } + newline = c == '\n'; + previous = c; + } + }); + if (data_begin == 0) { + return {}; + } + if (data_end == 0) { + data_end = n; + } + bounds.push_back(data_end); + return bounds; +} + +} + struct ifcopenshell::impl::in_memory_file_storage::lazy_source { file_reader reader; spf_lexer> lexer; @@ -2781,96 +2879,161 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& this->schema = schema; resolve_references_in_place = true; lazy_ = true; - byref_excl_.reserve(reader.size() / 32); - // One pass over the DATA section with the tokenizer's index policy: - // the instance headers through the shared loop, then the attribute list - // as tokens with only the parentheses, commas and names looked at. - // Nothing is decoded. A token the tokenizer rejects, or a structure the - // loop below does not expect, stops the index and the caller parses in - // full. - const char* failure = nullptr; - size_t failure_offset = 0; - try { - for_each_instance_header(reader, lexer, reader.size(), schema, bypassed_types, lazy_bypassed_, logger_.get(), [&](uint32_t name, const ifcopenshell::declaration* declaration, size_t) { - const uint64_t attributes_offset = reader.tell(); - const uint16_t type_index = (uint16_t)declaration->index_in_schema(); - int depth = 1; - int attribute = 0; - bool first_value = true; - size_t guid_begin = 0, guid_end = 0; - while (depth > 0) { - token t = lexer.next(); - if (!t) { - failure = "file ends inside an instance"; - failure_offset = attributes_offset; - return false; - } - if (t.is_operator()) { - if (t.value_char == '(') { - ++depth; - } else if (t.value_char == ')') { - --depth; - } else if (t.value_char == ',' && depth == 1) { - ++attribute; - } else if (t.value_char == ';') { - failure = "; inside an instance"; - failure_offset = t.start_pos; + // One pass over the DATA section with the tokenizer's index policy: the + // instance headers through the shared loop, then the attribute list as + // tokens with only the parentheses, commas and names looked at. Nothing + // is decoded. On a file large enough the section is split at the same + // boundaries the parallel parse uses and each chunk is indexed by its + // own worker with its own reader, lexer, shells and inverse records, + // merged in file order; the serial case is the same code on one chunk. + // A token the tokenizer rejects, or a structure the loop does not + // expect, stops the index and the caller parses in full. + struct index_output { + std::vector shells; + std::vector> offsets; + std::vector> guids; // shell index, raw text + std::vector bypassed; + entities_by_ref inverses; + const char* failure = nullptr; + size_t failure_offset = 0; + std::exception_ptr error; + }; + const auto index_chunk = [&](file_reader& chunk_reader, spf_lexer>& chunk_lexer, size_t end, index_output& out) { + try { + for_each_instance_header(chunk_reader, chunk_lexer, end, schema, bypassed_types, out.bypassed, logger_.get(), [&](uint32_t name, const ifcopenshell::declaration* declaration, size_t) { + const uint64_t attributes_offset = chunk_reader.tell(); + const uint16_t type_index = (uint16_t)declaration->index_in_schema(); + int depth = 1; + int attribute = 0; + bool first_value = true; + size_t guid_begin = 0, guid_end = 0; + while (depth > 0) { + token t = chunk_lexer.next(); + if (!t) { + out.failure = "file ends inside an instance"; + out.failure_offset = attributes_offset; return false; } - } else if (t.is_identifier()) { - byref_excl_.add((uint32_t)t.as_identifier(), name, type_index, attribute); - } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) { - guid_begin = t.start_pos + 1; - guid_end = reader.tell() - 1; + if (t.is_operator()) { + if (t.value_char == '(') { + ++depth; + } else if (t.value_char == ')') { + --depth; + } else if (t.value_char == ',' && depth == 1) { + ++attribute; + } else if (t.value_char == ';') { + out.failure = "; inside an instance"; + out.failure_offset = t.start_pos; + return false; + } + } else if (t.is_identifier()) { + out.inverses.add((uint32_t)t.as_identifier(), name, type_index, attribute); + } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) { + guid_begin = t.start_pos + 1; + guid_end = chunk_reader.tell() - 1; + } + if (depth == 1) { + first_value = false; + } + chunk_lexer.reset_pool(); } - if (depth == 1) { - first_value = false; + if (!chunk_lexer.next().is_operator(';')) { + out.failure = "expected ; after )"; + out.failure_offset = chunk_reader.tell(); + return false; } - lexer.reset_pool(); - } - if (!lexer.next().is_operator(';')) { - failure = "expected ; after )"; - failure_offset = reader.tell(); - return false; - } - auto data = ifcopenshell::make_pointer_type(file, declaration, name, instance_data::lazy_tag{}); + out.shells.push_back(ifcopenshell::make_pointer_type(file, declaration, name, instance_data::lazy_tag{})); + out.offsets.push_back({name, attributes_offset}); + if (guid_end > guid_begin && declaration->is(*ifcroot)) { + std::string guid; + guid.reserve(guid_end - guid_begin); + for (size_t at = guid_begin; at < guid_end; ++at) { + guid.push_back(chunk_reader.get(at)); + } + out.guids.push_back({out.shells.size() - 1, std::move(guid)}); + } + return true; + }); + } catch (const invalid_token_exception&) { + out.failure = "invalid token"; + out.failure_offset = chunk_reader.tell(); + } catch (...) { + out.error = std::current_exception(); + } + }; + + std::vector outputs; + constexpr size_t min_bytes_per_thread = 2u << 20; + const unsigned threads = (unsigned)std::min(parse_threads, std::max(1, reader.size() / min_bytes_per_thread)); + const std::vector bounds = threads > 1 ? chunk_bounds(reader, threads) : std::vector(); + if (bounds.size() >= 3) { + outputs.resize(bounds.size() - 1); + std::vector workers; + for (size_t k = 0; k + 1 < bounds.size(); ++k) { + outputs[k].inverses.reserve((bounds[k + 1] - bounds[k]) / 32); + workers.emplace_back([&, k]() { + file_reader chunk_reader = reader.reopen(); + chunk_reader.seek(bounds[k]); + spf_lexer> chunk_lexer(&chunk_reader, logger_.get()); + index_chunk(chunk_reader, chunk_lexer, bounds[k + 1], outputs[k]); + }); + } + for (auto& worker : workers) { + worker.join(); + } + } else { + outputs.resize(1); + outputs[0].inverses.reserve(reader.size() / 32); + index_chunk(reader, lexer, reader.size(), outputs[0]); + } + for (const auto& out : outputs) { + if (out.error) { + std::rethrow_exception(out.error); + } + if (out.failure != nullptr) { + logger_.get().message(ifcopenshell::logger::LOG_NOTICE, std::string("Lazy loading not possible (") + out.failure + " at offset " + std::to_string(out.failure_offset) + "), parsing the file in full"); + return false; + } + } + + // Merge in file order: what the serial index did per instance. + size_t shell_count = 0; + for (const auto& out : outputs) { + shell_count += out.shells.size(); + } + lazy_offsets_.reserve(shell_count); + byid_.reserve(byid_.size() + shell_count); + for (auto& out : outputs) { + for (const auto& data : out.shells) { + const uint32_t name = data->id(); if (!byid_.insert({name, data}).second) { logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Overwriting instance with name #" + std::to_string(name)); byid_.erase(name); byid_.insert({name, data}); } - lazy_offsets_.push_back({name, attributes_offset}); - express::base instance(data); - bytype_excl_[declaration].push_back(instance); + bytype_excl_[data->declaration()].push_back(express::base(data)); max_id = (std::max)(max_id, (unsigned int)name); - if (guid_end > guid_begin && declaration->is(*ifcroot)) { - std::string guid; - guid.reserve(guid_end - guid_begin); - for (size_t at = guid_begin; at < guid_end; ++at) { - guid.push_back(reader.get(at)); - } - if (guid.find('\\') != std::string::npos || guid.find("''") != std::string::npos) { - guid = ifcopenshell::decode_spf_string(guid); - } - std::array key; - if (guid_key(guid, key)) { - if (byguid_.count(key) != 0) { - logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Instance encountered with non-unique GlobalId " + guid); - } - byguid_[key] = instance; - } + } + lazy_offsets_.insert(lazy_offsets_.end(), out.offsets.begin(), out.offsets.end()); + for (auto& entry : out.guids) { + std::string& guid = entry.second; + if (guid.find('\\') != std::string::npos || guid.find("''") != std::string::npos) { + guid = ifcopenshell::decode_spf_string(guid); } - return true; - }); - } catch (const invalid_token_exception&) { - failure = "invalid token"; - failure_offset = reader.tell(); - } - if (failure != nullptr) { - logger_.get().message(ifcopenshell::logger::LOG_NOTICE, std::string("Lazy loading not possible (") + failure + " at offset " + std::to_string(failure_offset) + "), parsing the file in full"); - return false; + std::array key; + if (guid_key(guid, key)) { + if (byguid_.count(key) != 0) { + logger_.get().message(ifcopenshell::logger::LOG_WARNING, "Instance encountered with non-unique GlobalId " + guid); + } + byguid_[key] = express::base(out.shells[entry.first]); + } + } + lazy_bypassed_.insert(lazy_bypassed_.end(), out.bypassed.begin(), out.bypassed.end()); + byref_excl_.append(std::move(out.inverses)); + std::vector().swap(out.shells); } + outputs.clear(); std::sort(lazy_bypassed_.begin(), lazy_bypassed_.end()); std::sort(lazy_offsets_.begin(), lazy_offsets_.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); @@ -2919,27 +3082,6 @@ void parse_chunk(const Reader& source, size_t begin, size_t end, const ifcopensh } } -// Matches a literal byte by byte, across spans. -class literal_matcher { - const char* literal_; - size_t length_; - size_t matched_ = 0; - public: - explicit literal_matcher(const char* literal) - : literal_(literal), length_(std::strlen(literal)) {} - // True on the byte that completes the literal. - bool feed(char c) { - if (c == literal_[matched_]) { - if (++matched_ == length_) { - matched_ = 0; - return true; - } - } else { - matched_ = c == literal_[0] ? 1 : 0; - } - return false; - } -}; } @@ -2955,68 +3097,7 @@ bool ifcopenshell::impl::in_memory_file_storage::read_instances_parallel(Reader* return false; } - // One pass over the bytes finds the DATA section and, nearest each - // nominal split point, an instance boundary: a '#' that starts a - // line outside any string and any comment. This is the one place - // that looks at raw bytes instead of tokens, because tokenizing the - // file serially to find the split points would leave nothing to - // parallelise. It applies three rules only: a string starts and - // ends at a quote (a doubled quote closes and reopens, which comes - // to the same thing) and cannot span a line; a comment runs from - // /* to */. Getting a string's end wrong can only lose a candidate - // boundary, never accept a wrong one, since no string contains a - // newline. - literal_matcher data_matcher("\nDATA;"), endsec_matcher("\nENDSEC"); - size_t data_begin = 0, data_end = 0; - bool in_string = false, in_comment = false, newline = false; - char previous = 0; - std::vector bounds; - size_t next_split = 0; - s->for_each_span(0, n, [&](const char* data, size_t length, size_t offset) { - for (size_t i = 0; i < length; ++i) { - const char c = data[i]; - const size_t at = offset + i; - if (data_begin == 0) { - if (data_matcher.feed(c)) { - data_begin = at + 1; - bounds.push_back(data_begin); - next_split = data_begin + (n - data_begin) / threads; - } - continue; - } - if (data_end != 0) { - return; - } - if (in_comment) { - if (previous == '*' && c == '/') { - in_comment = false; - } - } else if (in_string) { - if (c == '\'' || c == '\n') { - in_string = false; - } - } else if (c == '\'') { - in_string = true; - } else if (previous == '/' && c == '*') { - in_comment = true; - } else if (endsec_matcher.feed(c)) { - data_end = at + 1 - 7; - return; - } else if (newline && c == '#' && at >= next_split && bounds.size() < threads) { - bounds.push_back(at); - next_split = data_begin + (n - data_begin) * bounds.size() / threads; - } - newline = c == '\n'; - previous = c; - } - }); - if (data_begin == 0) { - return false; - } - if (data_end == 0) { - data_end = n; - } - bounds.push_back(data_end); + const std::vector bounds = chunk_bounds(*s, threads); if (bounds.size() < 3) { return false; } diff --git a/src/ifcparse/tests/test_ifcopenshell_parse.cpp b/src/ifcparse/tests/test_ifcopenshell_parse.cpp index 1ecfaa9a95..a290478417 100644 --- a/src/ifcparse/tests/test_ifcopenshell_parse.cpp +++ b/src/ifcparse/tests/test_ifcopenshell_parse.cpp @@ -570,6 +570,12 @@ TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inve paged_parallel.paged_reading(true); paged_parallel.parse_threads(5); REQUIRE(paged_parallel.initialize(path.string())); + // And the lazy index built by 5 workers. + ifcopenshell::file lazy_parallel(ifcopenshell::uninitialized_tag{}); + lazy_parallel.lazy_loading(true); + lazy_parallel.parse_threads(5); + REQUIRE(lazy_parallel.initialize(path.string())); + REQUIRE(lazy_parallel.lazy_loading()); std::filesystem::remove(path); size_t count = 0; @@ -583,7 +589,7 @@ TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inve a.to_string(sa); b.to_string(sb); REQUIRE(sb.str() == sa.str()); - for (ifcopenshell::file* other : {&paged, &paged_parallel}) { + for (ifcopenshell::file* other : {&paged, &paged_parallel, &lazy_parallel}) { const express::base c = other->instance_by_id((int)a.id()); REQUIRE(c); std::ostringstream sc; @@ -603,5 +609,6 @@ TEST_CASE("Parallel and paged parsing yield the same instances, attributes, inve for (const auto& rooted : serial.instances_by_type("IfcRoot")) { const std::string guid = rooted.get_attribute_value(0); REQUIRE(parallel.instance_by_guid(guid).id() == serial.instance_by_guid(guid).id()); + REQUIRE(lazy_parallel.instance_by_guid(guid).id() == serial.instance_by_guid(guid).id()); } }