ifcparse: pass over attribute-list text in the lazy index without copying it

Inside an attribute list the index looks only at operators and names,
so a third tokenizer policy, attribute_tokens, returns a keyword (an
inline typed value such as IFCLABEL), an enumeration or a binary as
Token_LITERAL without copying its text; only a name's digits are kept.
The instance headers still go through index_tokens, which keeps the
keyword. Same next(), one more compile-time branch.

Found by callgrind on the lazy open (see the PR): string-pool access and
keyword text copying for tokens the index never read. Whole-file
tokenizing of TXG (58 MB) with the index policy 265 MB/s -> 293 MB/s;
lazy open TXG / 210_King / OKgate22 0.58 / 1.73 / 2.86 s -> 0.57 / 1.65 /
2.78 s. Small: the per-token call is the larger cost, which the next
commit addresses by splitting the work over threads.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL

src/ifcparse/parse.cpp

@@ -342,7 +342,9 @@ token spf_lexer<Reader>::next() {
             if (remaining >= 8) {
                 uint64_t x = stream->peek_u64();
                 if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
-                    str.append(reinterpret_cast<const char*>(&x), 8);
+                    if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                        str.append(reinterpret_cast<const char*>(&x), 8);
+                    }
                     stream->increment(8);
                     remaining -= 8;
                     continue;
@@ -351,7 +353,9 @@ token spf_lexer<Reader>::next() {
             if (remaining >= 4) {
                 uint32_t x = stream->peek_u32();
                 if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
-                    str.append(reinterpret_cast<const char*>(&x), 4);
+                    if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                        str.append(reinterpret_cast<const char*>(&x), 4);
+                    }
                     stream->increment(4);
                     remaining -= 4;
                     continue;
@@ -372,7 +376,7 @@ token spf_lexer<Reader>::next() {
                 if ((ttype == token::Token_BINARY && character == '"') ||
                     (ttype == token::Token_ENUMERATION && character == '.')) {
                     // Skip
-                } else {
+                } else if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
                     str.push_back(character);
                 }
             }
@@ -383,6 +387,12 @@ token spf_lexer<Reader>::next() {
         if constexpr (!Policy::decode_values) {
             // Only names and keywords are read; everything else is a literal
             // whose position is all the caller wants.
+            if constexpr (!Policy::keep_keywords) {
+                if (ttype != token::Token_IDENTIFIER) {
+                    pop_pool_entry();
+                    return token(pos, token::Token_LITERAL);
+                }
+            }
             if (ttype == token::Token_IDENTIFIER) {
                 int int_val;
                 if (!parse_num_(str.c_str(), str.size(), int_val)) {
@@ -443,7 +453,8 @@ template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<mmap_impl>>;
 
 #define IFC_INSTANTIATE_LEXER_NEXT(Reader) \
     template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::full_tokens>(); \
-    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>();
+    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>(); \
+    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::attribute_tokens>();
 IFC_INSTANTIATE_LEXER_NEXT(file_reader<full_buffer_impl>)
 IFC_INSTANTIATE_LEXER_NEXT(file_reader<paged_file_impl>)
 IFC_INSTANTIATE_LEXER_NEXT(file_reader<pushed_sequential_impl>)
@@ -2789,7 +2800,7 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string&
             bool first_value = true;
             size_t guid_begin = 0, guid_end = 0;
             while (depth > 0) {
-                token t = lexer.next<index_tokens>();
+                token t = lexer.next<attribute_tokens>();
                 if (!t) {
                     failure = "file ends inside an instance";
                     failure_offset = attributes_offset;
@@ -2818,7 +2829,7 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string&
                 }
                 lexer.reset_pool();
             }
-            if (!lexer.next<index_tokens>().is_operator(';')) {
+            if (!lexer.next<attribute_tokens>().is_operator(';')) {
                 failure = "expected ; after )";
                 failure_offset = reader.tell();
                 return false;

src/ifcparse/parse.h

@@ -59,10 +59,20 @@ IFC_PARSE_API std::string decode_spf_string(const std::string& value);
 struct full_tokens {
     static constexpr bool decode_strings = true;
     static constexpr bool decode_values = true;
+    static constexpr bool keep_keywords = true;
 };
 struct index_tokens {
     static constexpr bool decode_strings = false;
     static constexpr bool decode_values = false;
+    static constexpr bool keep_keywords = true;
+};
+/// Inside an attribute list the index only looks at operators and names, so
+/// a keyword (an inline typed value such as IFCLABEL), an enumeration or a
+/// binary comes back as Token_LITERAL without its text being copied.
+struct attribute_tokens {
+    static constexpr bool decode_strings = false;
+    static constexpr bool decode_values = false;
+    static constexpr bool keep_keywords = false;
 };
 
 /// A stream of tokens to be read from a file_reader.