From 97c83f02922bab559c613c48e3bc131aed896713 Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Mon, 14 Sep 2026 18:47:06 +1000 Subject: [PATCH] ifcparse: pass over attribute-list text in the lazy index without copying it Inside an attribute list the index looks only at operators and names, so a third tokenizer policy, attribute_tokens, returns a keyword (an inline typed value such as IFCLABEL), an enumeration or a binary as Token_LITERAL without copying its text; only a name's digits are kept. The instance headers still go through index_tokens, which keeps the keyword. Same next(), one more compile-time branch. Found by callgrind on the lazy open (see the PR): string-pool access and keyword text copying for tokens the index never read. Whole-file tokenizing of TXG (58 MB) with the index policy 265 MB/s -> 293 MB/s; lazy open TXG / 210_King / OKgate22 0.58 / 1.73 / 2.86 s -> 0.57 / 1.65 / 2.78 s. Small: the per-token call is the larger cost, which the next commit addresses by splitting the work over threads. This commit was written by an AI coding tool and has not been verified by a human. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL --- src/ifcparse/parse.cpp | 23 +++++++++++++++++------ src/ifcparse/parse.h | 10 ++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index e27941d373..ff362ad53e 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -342,7 +342,9 @@ token spf_lexer::next() { if (remaining >= 8) { uint64_t x = stream->peek_u64(); if ((ttype == token::Token_NONE ? SWAR::has_special_char(x) : SWAR::has_special_char(x)) == 0) { - str.append(reinterpret_cast(&x), 8); + if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) { + str.append(reinterpret_cast(&x), 8); + } stream->increment(8); remaining -= 8; continue; @@ -351,7 +353,9 @@ token spf_lexer::next() { if (remaining >= 4) { uint32_t x = stream->peek_u32(); if ((ttype == token::Token_NONE ? SWAR::has_special_char(x) : SWAR::has_special_char(x)) == 0) { - str.append(reinterpret_cast(&x), 4); + if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) { + str.append(reinterpret_cast(&x), 4); + } stream->increment(4); remaining -= 4; continue; @@ -372,7 +376,7 @@ token spf_lexer::next() { if ((ttype == token::Token_BINARY && character == '"') || (ttype == token::Token_ENUMERATION && character == '.')) { // Skip - } else { + } else if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) { str.push_back(character); } } @@ -383,6 +387,12 @@ token spf_lexer::next() { if constexpr (!Policy::decode_values) { // Only names and keywords are read; everything else is a literal // whose position is all the caller wants. + if constexpr (!Policy::keep_keywords) { + if (ttype != token::Token_IDENTIFIER) { + pop_pool_entry(); + return token(pos, token::Token_LITERAL); + } + } if (ttype == token::Token_IDENTIFIER) { int int_val; if (!parse_num_(str.c_str(), str.size(), int_val)) { @@ -443,7 +453,8 @@ template class IFC_PARSE_API ifcopenshell::spf_lexer>; #define IFC_INSTANTIATE_LEXER_NEXT(Reader) \ template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); \ - template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); + template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); \ + template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); IFC_INSTANTIATE_LEXER_NEXT(file_reader) IFC_INSTANTIATE_LEXER_NEXT(file_reader) IFC_INSTANTIATE_LEXER_NEXT(file_reader) @@ -2789,7 +2800,7 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& bool first_value = true; size_t guid_begin = 0, guid_end = 0; while (depth > 0) { - token t = lexer.next(); + token t = lexer.next(); if (!t) { failure = "file ends inside an instance"; failure_offset = attributes_offset; @@ -2818,7 +2829,7 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& } lexer.reset_pool(); } - if (!lexer.next().is_operator(';')) { + if (!lexer.next().is_operator(';')) { failure = "expected ; after )"; failure_offset = reader.tell(); return false; diff --git a/src/ifcparse/parse.h b/src/ifcparse/parse.h index 348dac5fde..ae2524057a 100644 --- a/src/ifcparse/parse.h +++ b/src/ifcparse/parse.h @@ -59,10 +59,20 @@ IFC_PARSE_API std::string decode_spf_string(const std::string& value); struct full_tokens { static constexpr bool decode_strings = true; static constexpr bool decode_values = true; + static constexpr bool keep_keywords = true; }; struct index_tokens { static constexpr bool decode_strings = false; static constexpr bool decode_values = false; + static constexpr bool keep_keywords = true; +}; +/// Inside an attribute list the index only looks at operators and names, so +/// a keyword (an inline typed value such as IFCLABEL), an enumeration or a +/// binary comes back as Token_LITERAL without its text being copied. +struct attribute_tokens { + static constexpr bool decode_strings = false; + static constexpr bool decode_values = false; + static constexpr bool keep_keywords = false; }; /// A stream of tokens to be read from a file_reader.