ifcparse: give the tokenizer a compile-time policy for what it decodes

spf_lexer::next() becomes next<Policy>(). full_tokens, the default, is
what the parser has always had. index_tokens is what the lazy index
needs: a string is ended but not decoded, and a number, enumeration or
binary comes back as Token_LITERAL with only its position; names,
keywords and operators are read as before. Each policy compiles to its
own loop from the one implementation, so there is no second tokenizer.

character_decoder gains skip(): the same state machine as the
conversion with the collection compiled out, so an escape such as \S\'
(an apostrophe as the page character) ends the string at the same byte
under both policies. A byte-level scan would have ended it early.

Also fixes a comment that follows a token without whitespace, ",/* x */",
which skip_comment() never saw because the slash had been consumed.

TXG (58 MB), single thread: tokenizing the whole file 194 MB/s with
full_tokens, 249 MB/s with index_tokens; through 64 KB pages 196 and
205 MB/s. The parse itself is unchanged.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL

src/ifcparse/character_decoder.cpp

@@ -151,7 +151,11 @@ character_decoder<Reader>::~character_decoder() {
 }
 
 namespace {
-    template <typename Reader>
+    // Reads the string at the stream's read pointer, up to and including
+    // its closing quote. With Decode the characters are collected in
+    // builder_ and converted; without it the same state machine runs so the
+    // string ends at the same byte, and nothing is collected.
+    template <typename Reader, bool Decode = true>
     std::string read_string(std::u32string& builder_, Reader& stream_, logger& logger_, typename ifcopenshell::character_decoder<Reader>::ConversionMode mode, char substitution_character) {
         unsigned int parse_state = 0;
         builder_.clear();
@@ -166,7 +170,9 @@ namespace {
                 if (stream_.remaining() >= 8) {
                     uint64_t x = stream_.peek_u64();
                     if (SWAR::has_special_char(x) == 0) {
-                        SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 8);
+                        if constexpr (Decode) {
+                            SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 8);
+                        }
                         stream_.increment(8);
                         continue;
                     }
@@ -174,7 +180,9 @@ namespace {
                 if (stream_.remaining() >= 4) {
                     uint32_t x = stream_.peek_u32();
                     if (SWAR::has_special_char(x) == 0) {
-                        SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 4);
+                        if constexpr (Decode) {
+                            SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 4);
+                        }
                         stream_.increment(4);
                         continue;
                     }
@@ -187,7 +195,9 @@ namespace {
             }
 
             if (EXPECTS_CHARACTER(parse_state)) {
-                builder_.push_back(ifcopenshell::convert_codepage(codepage, current_char + 0x80));
+                if constexpr (Decode) {
+                    builder_.push_back(ifcopenshell::convert_codepage(codepage, current_char + 0x80));
+                }
                 parse_state = 0;
             } else if (current_char == '\'' && (parse_state == 0U)) {
                 parse_state = APOSTROPHE;
@@ -235,7 +245,9 @@ namespace {
                 if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) ||
                     (hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) ||
                     (hex_count == 8)) {
-                    builder_.push_back(hex);
+                    if constexpr (Decode) {
+                        builder_.push_back(hex);
+                    }
                     if (hex_count == 2) {
                         parse_state = 0;
                     } else {
@@ -253,10 +265,15 @@ namespace {
                 throw invalid_token_exception(stream_.tell(), current_char);
             } else {
                 parse_state = hex = hex_count = 0;
-                builder_.push_back(current_char);
+                if constexpr (Decode) {
+                    builder_.push_back(current_char);
+                }
             }
             stream_.increment();
         }
+        if constexpr (!Decode) {
+            return std::string();
+        }
         // builder_.push_back('\'');
 
         if (mode == ifcopenshell::character_decoder<Reader>::UTF8) {
@@ -303,6 +320,11 @@ character_decoder<Reader>::operator std::string() {
     return read_string(builder_, *stream_, logger_, mode, substitution_character);
 }
 
+template <typename Reader>
+void character_decoder<Reader>::skip() {
+    read_string<Reader, false>(builder_, *stream_, logger_, mode, substitution_character);
+}
+
 template <typename Reader>
 std::string character_decoder<Reader>::get(size_t& ptr) {
     auto local_stream = *stream_;

src/ifcparse/character_decoder.h

@@ -62,6 +62,10 @@ class IFC_PARSE_API character_decoder {
     // Gets a decoded string representation at the token stream
     // read pointer and advances the underlying token stream.
     operator std::string();
+    // Advances the token stream past the string at the read pointer
+    // without decoding it: the same state machine as the conversion, so
+    // escapes such as \S\' end the string at the same byte.
+    void skip();
     // Gets a decoded string representation at the offset provided,
     // does not mutate the underlying token stream read pointer.
     std::string get(size_t& offset);

src/ifcparse/parse.cpp

@@ -268,6 +268,7 @@ IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) {
 // Returns the offset of the current token and moves cursor to next
 //
 template <typename Reader>
+template <typename Policy>
 token spf_lexer<Reader>::next() {
 
     if (stream->eof()) {
@@ -278,6 +279,11 @@ token spf_lexer<Reader>::next() {
     char character = stream->read();
 
     if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
+        if (character == '/') {
+            // skip_comment() wants to see the slash itself, so a comment
+            // that follows the previous token without whitespace is skipped.
+            stream->seek(pos);
+        }
         while ((skip_whitespace() != 0U) || (skip_comment() != 0U)) {
         }
         if (stream->eof()) {
@@ -303,8 +309,14 @@ token spf_lexer<Reader>::next() {
 
     if (character == '\'') {
         // If a string is encountered defer processing to the character_decoder
-        str = *decoder_;
+        if constexpr (Policy::decode_strings) {
-        return token(pos, token::Token_STRING, str);
+            str = *decoder_;
+            return token(pos, token::Token_STRING, str);
+        } else {
+            decoder_->skip();
+            pop_pool_entry();
+            return token(pos, token::Token_STRING);
+        }
     } else {
         auto ttype = token::Token_NONE;
         if (character == '"' || character == '.') {
@@ -364,6 +376,26 @@ token spf_lexer<Reader>::next() {
             remaining -= 1;
         }
 
+        if constexpr (!Policy::decode_values) {
+            // Only names and keywords are read; everything else is a literal
+            // whose position is all the caller wants.
+            if (ttype == token::Token_IDENTIFIER) {
+                int int_val;
+                if (!parse_num_(str.c_str(), str.size(), int_val)) {
+                    throw invalid_token_exception(pos, str, "instance name");
+                }
+                pop_pool_entry();
+                return token(pos, ttype, (int64_t)int_val);
+            }
+            if (ttype == token::Token_NONE && !str.empty()) {
+                const char first = str.front();
+                if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
+                    return token(pos, token::Token_KEYWORD, str);
+                }
+            }
+            pop_pool_entry();
+            return token(pos, token::Token_LITERAL);
+        }
         if (ttype == token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) {
             pop_pool_entry();
             return token(pos, token::Token_BOOL, str[0]);
@@ -405,6 +437,17 @@ template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<pushed_sequenti
 template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<mmap_impl>>;
 #endif
 
+#define IFC_INSTANTIATE_LEXER_NEXT(Reader) \
+    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::full_tokens>(); \
+    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>();
+IFC_INSTANTIATE_LEXER_NEXT(file_reader<full_buffer_impl>)
+IFC_INSTANTIATE_LEXER_NEXT(file_reader<paged_file_impl>)
+IFC_INSTANTIATE_LEXER_NEXT(file_reader<pushed_sequential_impl>)
+#ifdef USE_MMAP
+IFC_INSTANTIATE_LEXER_NEXT(file_reader<mmap_impl>)
+#endif
+#undef IFC_INSTANTIATE_LEXER_NEXT
+
 bool token::is_operator() {
     return type == Token_OPERATOR;
 }

src/ifcparse/parse.h

@@ -51,6 +51,20 @@ IFC_PARSE_API std::string encode_spf_string(const std::string& value);
 
 IFC_PARSE_API std::string decode_spf_string(const std::string& value);
 
+/// What a pass over the tokens has to produce. The parser needs every
+/// value; the lazy index only needs to know where the tokens are and which
+/// of them are instance names, so it ends strings without decoding them and
+/// passes over numbers, enumerations and binaries. The choice is a template
+/// parameter of spf_lexer::next(), so each pass compiles to its own loop.
+struct full_tokens {
+    static constexpr bool decode_strings = true;
+    static constexpr bool decode_values = true;
+};
+struct index_tokens {
+    static constexpr bool decode_strings = false;
+    static constexpr bool decode_values = false;
+};
+
 /// A stream of tokens to be read from a file_reader.
 template <typename Reader>
 class IFC_PARSE_API spf_lexer {
@@ -82,6 +96,10 @@ class IFC_PARSE_API spf_lexer {
     Reader* stream;
     // file* file;
     spf_lexer(Reader* stream, ifcopenshell::logger& logger = ifcopenshell::logger::root());
+    // The next token. With index_tokens a string, number, enumeration or
+    // binary comes back as Token_LITERAL (Token_STRING for a string) with
+    // only its position; names, keywords and operators are always read.
+    template <typename Policy = full_tokens>
     token next();
     ~spf_lexer();
     // void TokenString(size_t offset, std::string& result);

src/ifcparse/storage.h

@@ -157,7 +157,10 @@ namespace ifcopenshell {
             Token_INT,
             Token_BOOL,
             Token_FLOAT,
-            Token_BINARY
+            Token_BINARY,
+            // A number, enumeration, binary or string the tokenizer policy
+            // passed over without decoding; only its position is known.
+            Token_LITERAL
         };
 
         size_t start_pos;
@@ -173,6 +176,9 @@ namespace ifcopenshell {
         token() : start_pos(0),
                   type(Token_NONE) {}
 
+        token(size_t start_position, token_type token_kind)
+            : start_pos(start_position), type(token_kind), value_int(0) {}
+
         token(size_t start_position, token_type token_kind, const std::string& string_value)
             : start_pos(start_position), type(token_kind), value_string(&string_value) {}
 

src/ifcparse/tests/test_ifcopenshell_parse.cpp

@@ -294,3 +294,51 @@ TEST_CASE("Only a 22-character GlobalId is indexed", "[ifcparse]") {
     wall.set_attribute_value(0, std::string("1F$7lN9$r5MOA_lpAoNM52"));
     CHECK(file.instance_by_guid("1F$7lN9$r5MOA_lpAoNM52").id() == 2);
 }
+TEST_CASE("The index token policy ends every token where the full policy does, without decoding", "[ifcparse]") {
+    // Doubled quotes, a \S\' escape (an apostrophe as the page character,
+    // which a byte scan would take for the end of the string), a \X2\
+    // escape, a comment, binaries, enumerations, numbers and names.
+    const std::string data =
+        "#1=IFCWALL('it''s','a\\S\\'b','\\X2\\00E9\\X0\\c',/* #9 */ #2, \"0A\", .T., -1.5E-3, 42, $, *, (IFCLABEL('x'), #3));\n";
+    ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> full_reader(data, ifcopenshell::caller_fed_tag{});
+    ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> index_reader(data, ifcopenshell::caller_fed_tag{});
+    ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> full(&full_reader), index(&index_reader);
+    size_t count = 0;
+    std::vector<unsigned> names;
+    while (true) {
+        ifcopenshell::token a = full.next(), b = index.next<ifcopenshell::index_tokens>();
+        REQUIRE((bool)a == (bool)b);
+        if (!a) {
+            break;
+        }
+        ++count;
+        CHECK(a.start_pos == b.start_pos);
+        CHECK(full_reader.tell() == index_reader.tell());
+        if (a.is_identifier()) {
+            REQUIRE(b.is_identifier());
+            CHECK(a.as_identifier() == b.as_identifier());
+            names.push_back(b.as_identifier());
+        } else if (a.is_keyword()) {
+            REQUIRE(b.is_keyword());
+            CHECK(a.as_string() == b.as_string());
+        } else if (a.is_operator()) {
+            REQUIRE(b.is_operator());
+            CHECK(a.value_char == b.value_char);
+        } else if (a.is_string()) {
+            CHECK(b.type == ifcopenshell::token::Token_STRING);
+        } else {
+            CHECK(b.type == ifcopenshell::token::Token_LITERAL);
+        }
+        full.reset_pool();
+        index.reset_pool();
+    }
+    CHECK(count == 34);
+    CHECK(names == std::vector<unsigned>{1, 2, 3});
+    // And the full policy decoded the escapes.
+    ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> again(data, ifcopenshell::caller_fed_tag{});
+    ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> lexer(&again);
+    lexer.next(); lexer.next(); lexer.next(); lexer.next();
+    CHECK(lexer.next().as_string() == "it's");
+    lexer.next();
+    CHECK(lexer.next().as_string() == "a\xc2\xa7" "b");
+}