ifcparse: the tokenizer as scan(Consumer&), next() as its one-token consumer

The body of next() becomes spf_lexer::scan(Consumer&), the same code
wrapped in a loop that hands each token to the consumer's callbacks
(operator_, identifier, string, keyword, enumeration, binary, boolean,
integer, real, literal) instead of building a token object; each
callback returns whether to go on. The consumer's constexpr flags say
what is decoded: decode_strings, decode_values, keep_keywords. It lives
in spf_scan.h, with the SWAR helpers and number parsing it needs, so a
consumer inlines into the loop. next<Policy>() is kept as the consumer
that stops after one token: the attribute reader, header parser and
streamer pull tokens recursively and stay as they are.

The lazy index is now attribute_consumer: depth and attribute index from
the operators, every name straight into the inverse index, the bounds
of the first attribute if it is a string, done at the closing semicolon.
The attribute_tokens policy it replaces is gone.

Tokenizing 50 MB files with nothing decoded, in memory: index policy
through next() 214–247 MB/s, scan() with the inlined consumer 314–403
MB/s (TXG 247 → 345); through 64 KB pages 245–284 MB/s. Lazy open on one
thread TXG / 210_King / OKgate22 0.55 / 1.59 / 2.11 s → 0.52 / 1.57 /
2.05 s. The full tokenizer through the adapter is unchanged (TXG 202–210
MB/s against 195–219 before), as is the strict parse.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL

src/ifcparse/parse.cpp

@@ -137,313 +137,14 @@ std::string& spf_lexer<Reader>::get_temp_string() const {
     return (*stringpool_[slice])[offset];
 }
 
-namespace {
 
 #if defined(__APPLE__) || defined(__EMSCRIPTEN__)
-double parse_double_c(const char* start, char** end) {
+double ifcopenshell::parse_double_c(const char* start, char** end) {
     static const locale_t loc = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0);
     return strtod_l(start, end, loc);
 }
 #endif
 
-template <typename T>
-bool parse_num_(const char* pStart, size_t size, T& val) {
-    if (size == 0) {
-        return false;
-    }
-    if (*pStart == '+') {
-        ++pStart;
-        --size;
-        if (size == 0) {
-            return false;
-        }
-    }
-    if constexpr (std::is_floating_point_v<T>) {
-#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
-        // pStart is NUL-terminated at pStart + size (callers pass c_str()), so
-        // strtod_l stops exactly at the end of a well-formed number. from_chars
-        // is not instantiated for double here — its float overload is =deleted
-        // in libc++ (Apple's and Emscripten's).
-        char* pEnd = nullptr;
-        const double result = parse_double_c(pStart, &pEnd);
-        if (pEnd != pStart + size) {
-            return false;
-        }
-        val = static_cast<T>(result);
-        return true;
-#else
-        auto re = std::from_chars(pStart, pStart + size, val);
-        return re.ec == std::errc() && re.ptr == pStart + size;
-#endif
-    } else {
-        auto re = std::from_chars(pStart, pStart + size, val);
-        return re.ec == std::errc() && re.ptr == pStart + size;
-    }
-}
-
-} // namespace
-
-// These helpers sit on the tokenizer's innermost loop; left to the
-// compiler's heuristics they end up as calls, one per eight bytes.
-#if defined(_MSC_VER)
-#define IFC_SWAR_INLINE __forceinline
-#else
-#define IFC_SWAR_INLINE inline __attribute__((always_inline))
-#endif
-
-namespace SWAR {
-constexpr uint32_t ONES32 = 0x01010101u;
-constexpr uint32_t HIGHS32 = 0x80808080u;
-constexpr uint64_t ONES = 0x0101010101010101ull;
-constexpr uint64_t HIGHS = 0x8080808080808080ull;
-
-constexpr uint64_t splat(unsigned char c) {
-    return ONES * c;
-}
-
-IFC_SWAR_INLINE uint32_t has_zero_byte(uint32_t x) {
-    return (x - ONES32) & ~x & HIGHS32;
-}
-
-IFC_SWAR_INLINE uint64_t has_zero_byte(uint64_t x) {
-    return (x - ONES) & ~x & HIGHS;
-}
-
-IFC_SWAR_INLINE uint32_t eq_mask(uint32_t x, uint32_t c) {
-    return has_zero_byte(x ^ c);
-}
-
-IFC_SWAR_INLINE uint64_t eq_mask(uint64_t x, uint64_t c) {
-    return has_zero_byte(x ^ c);
-}
-
-namespace chars {
-constexpr uint64_t lpar = splat('(');
-constexpr uint64_t rpar = splat(')');
-constexpr uint64_t eq = splat('=');
-constexpr uint64_t comma = splat(',');
-constexpr uint64_t semi = splat(';');
-constexpr uint64_t slash = splat('/');
-
-constexpr uint64_t space = splat(' ');
-constexpr uint64_t cr = splat('\r');
-constexpr uint64_t lf = splat('\n');
-constexpr uint64_t tab = splat('\t');
-
-constexpr uint64_t quote = splat('"');
-constexpr uint64_t dot = splat('.');
-} // namespace chars
-
-template <bool IncludeDot = true>
-IFC_SWAR_INLINE uint64_t has_special_char(uint64_t x) {
-    return eq_mask(x, chars::lpar) |
-           eq_mask(x, chars::rpar) |
-           eq_mask(x, chars::eq) |
-           eq_mask(x, chars::comma) |
-           eq_mask(x, chars::semi) |
-           eq_mask(x, chars::slash) |
-           eq_mask(x, chars::space) |
-           eq_mask(x, chars::cr) |
-           eq_mask(x, chars::lf) |
-           eq_mask(x, chars::tab) |
-           eq_mask(x, chars::quote) |
-           (IncludeDot ? eq_mask(x, chars::dot) : uint64_t{0});
-}
-
-template <bool IncludeDot = true>
-IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) {
-    return eq_mask(x, static_cast<uint32_t>(chars::lpar)) |
-           eq_mask(x, static_cast<uint32_t>(chars::rpar)) |
-           eq_mask(x, static_cast<uint32_t>(chars::eq)) |
-           eq_mask(x, static_cast<uint32_t>(chars::comma)) |
-           eq_mask(x, static_cast<uint32_t>(chars::semi)) |
-           eq_mask(x, static_cast<uint32_t>(chars::slash)) |
-           eq_mask(x, static_cast<uint32_t>(chars::space)) |
-           eq_mask(x, static_cast<uint32_t>(chars::cr)) |
-           eq_mask(x, static_cast<uint32_t>(chars::lf)) |
-           eq_mask(x, static_cast<uint32_t>(chars::tab)) |
-           eq_mask(x, static_cast<uint32_t>(chars::quote)) |
-           (IncludeDot ? eq_mask(x, static_cast<uint32_t>(chars::dot)) : uint32_t{0});
-}
-
-}
-
-//
-// Returns the offset of the current token and moves cursor to next
-//
-template <typename Reader>
-template <typename Policy>
-token spf_lexer<Reader>::next() {
-
-    if (stream->eof()) {
-        return token{};
-    }
-
-    auto pos = stream->tell();
-    char character = stream->read();
-
-    if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
-        if (character == '/') {
-            // skip_comment() wants to see the slash itself, so a comment
-            // that follows the previous token without whitespace is skipped.
-            stream->seek(pos);
-        }
-        while ((skip_whitespace() != 0U) || (skip_comment() != 0U)) {
-        }
-        if (stream->eof()) {
-            return token{};
-        }
-        pos = stream->tell();
-        character = stream->read();
-    }
-
-    // If the cursor is at [()=,;$*] we know token consists of single char
-    if (character == '(' ||
-        character == ')' ||
-        character == '=' ||
-        character == ',' ||
-        character == ';' ||
-        character == '$' ||
-        character == '*')
-    {
-        return token(pos, character);
-    }
-
-    auto& str = get_temp_string();
-
-    if (character == '\'') {
-        // If a string is encountered defer processing to the character_decoder
-        if constexpr (Policy::decode_strings) {
-            str = *decoder_;
-            return token(pos, token::Token_STRING, str);
-        } else {
-            decoder_->skip();
-            pop_pool_entry();
-            return token(pos, token::Token_STRING);
-        }
-    } else {
-        auto ttype = token::Token_NONE;
-        if (character == '"' || character == '.') {
-            if (character == '"') {
-                ttype = token::Token_BINARY;
-            } else {
-                ttype = token::Token_ENUMERATION;
-            }
-            str.clear();
-        } else if (character == '#') {
-            ttype = token::Token_IDENTIFIER;
-            str.clear();
-        } else {
-            str.assign(&character, 1);
-        }
-
-        auto remaining = stream->remaining();
-        while (remaining) {
-            if (remaining >= 8) {
-                uint64_t x = stream->peek_u64();
-                if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
-                    if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
-                        str.append(reinterpret_cast<const char*>(&x), 8);
-                    }
-                    stream->increment(8);
-                    remaining -= 8;
-                    continue;
-                }
-            }
-            if (remaining >= 4) {
-                uint32_t x = stream->peek_u32();
-                if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
-                    if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
-                        str.append(reinterpret_cast<const char*>(&x), 4);
-                    }
-                    stream->increment(4);
-                    remaining -= 4;
-                    continue;
-                }
-            }
-
-            // Read character and increment pointer if not starting a new token
-            char character = stream->peek();
-            if (character == '(' ||
-                character == ')' ||
-                character == '=' ||
-                character == ',' ||
-                character == ';' ||
-                character == '/') {
-                break;
-            }
-            if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
-                if ((ttype == token::Token_BINARY && character == '"') ||
-                    (ttype == token::Token_ENUMERATION && character == '.')) {
-                    // Skip
-                } else if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
-                    str.push_back(character);
-                }
-            }
-            stream->increment();
-            remaining -= 1;
-        }
-
-        if constexpr (!Policy::decode_values) {
-            // Only names and keywords are read; everything else is a literal
-            // whose position is all the caller wants.
-            if constexpr (!Policy::keep_keywords) {
-                if (ttype != token::Token_IDENTIFIER) {
-                    pop_pool_entry();
-                    return token(pos, token::Token_LITERAL);
-                }
-            }
-            if (ttype == token::Token_IDENTIFIER) {
-                int int_val;
-                if (!parse_num_(str.c_str(), str.size(), int_val)) {
-                    throw invalid_token_exception(pos, str, "instance name");
-                }
-                pop_pool_entry();
-                return token(pos, ttype, (int64_t)int_val);
-            }
-            if (ttype == token::Token_NONE && !str.empty()) {
-                const char first = str.front();
-                if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
-                    return token(pos, token::Token_KEYWORD, str);
-                }
-            }
-            pop_pool_entry();
-            return token(pos, token::Token_LITERAL);
-        }
-        if (ttype == token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) {
-            pop_pool_entry();
-            return token(pos, token::Token_BOOL, str[0]);
-        } else if (ttype == token::Token_IDENTIFIER) {
-            int int_val;
-            if (!parse_num_(str.c_str(), str.size(), int_val)) {
-                throw invalid_token_exception(pos, str, "instance name");
-            }
-            pop_pool_entry();
-            return token(pos, ttype, (int64_t)int_val);
-        } else if (ttype == token::Token_NONE && !str.empty()) {
-            int64_t int_val;
-            double float_val;
-            auto& first = str.front();
-            if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
-                ttype = token::Token_KEYWORD;
-                return token(pos, ttype, str);
-            } else if (parse_num_(str.c_str(), str.size(), int_val)) {
-                ttype = token::Token_INT;
-                pop_pool_entry();
-                return token(pos, ttype, int_val);
-            } else if (parse_num_(str.c_str(), str.size(), float_val)) {
-                ttype = token::Token_FLOAT;
-                pop_pool_entry();
-                return token(pos, float_val);
-            }
-        } else if (ttype == token::Token_BINARY || ttype == token::Token_ENUMERATION) {
-            return token(pos, ttype, str);
-        }
-
-        throw invalid_token_exception(pos, str, "valid token");
-    }
-}
-
 template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<full_buffer_impl>>;
 template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<paged_file_impl>>;
 template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<pushed_sequential_impl>>;
@@ -453,8 +154,7 @@ template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<mmap_impl>>;
 
 #define IFC_INSTANTIATE_LEXER_NEXT(Reader) \
     template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::full_tokens>(); \
-    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>(); \
+    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>();
-    template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::attribute_tokens>();
 IFC_INSTANTIATE_LEXER_NEXT(file_reader<full_buffer_impl>)
 IFC_INSTANTIATE_LEXER_NEXT(file_reader<paged_file_impl>)
 IFC_INSTANTIATE_LEXER_NEXT(file_reader<pushed_sequential_impl>)
@@ -2791,6 +2491,105 @@ std::vector<size_t> chunk_bounds(const Reader& source, unsigned threads) {
 
 }
 
+namespace {
+
+// The lazy index's consumer for one instance's attribute list, fed by
+// spf_lexer::scan() from just past the opening parenthesis: depth and the
+// attribute index from the operators, every name straight into the inverse
+// index, the bounds of the first attribute if it is a string (the GlobalId
+// candidate), and done at the semicolon that closes the instance. Nothing
+// is decoded or copied.
+struct attribute_consumer {
+    static constexpr bool decode_strings = false;
+    static constexpr bool decode_values = false;
+    static constexpr bool keep_keywords = false;
+
+    ifcopenshell::impl::in_memory_file_storage::entities_by_ref& inverses;
+    uint32_t name;
+    uint16_t type;
+    int depth = 1;
+    int attribute = 0;
+    bool first_value = true;
+    bool closed = false;
+    bool done = false;
+    size_t guid_begin = 0, guid_end = 0;
+    const char* failure = nullptr;
+    size_t failure_offset = 0;
+
+    bool after_close(size_t pos) {
+        failure = "expected ; after )";
+        failure_offset = pos;
+        return false;
+    }
+    bool operator_(size_t pos, char c) {
+        if (closed) {
+            if (c == ';') {
+                done = true;
+                return false;
+            }
+            return after_close(pos);
+        }
+        switch (c) {
+        case '(':
+            ++depth;
+            return true;
+        case ')':
+            if (--depth == 0) {
+                closed = true;
+            }
+            return true;
+        case ',':
+            if (depth == 1) {
+                ++attribute;
+            }
+            return true;
+        case ';':
+            failure = "; inside an instance";
+            failure_offset = pos;
+            return false;
+        default:
+            if (depth == 1) {
+                first_value = false;
+            }
+            return true;
+        }
+    }
+    bool identifier(size_t pos, uint32_t referenced) {
+        if (closed) {
+            return after_close(pos);
+        }
+        inverses.add(referenced, name, type, attribute);
+        if (depth == 1) {
+            first_value = false;
+        }
+        return true;
+    }
+    bool string(size_t begin, size_t end) {
+        if (closed) {
+            return after_close(begin);
+        }
+        if (depth == 1 && attribute == 0 && first_value) {
+            guid_begin = begin + 1;
+            guid_end = end - 1;
+        }
+        if (depth == 1) {
+            first_value = false;
+        }
+        return true;
+    }
+    bool literal(size_t pos) {
+        if (closed) {
+            return after_close(pos);
+        }
+        if (depth == 1) {
+            first_value = false;
+        }
+        return true;
+    }
+};
+
+}
+
 struct ifcopenshell::impl::in_memory_file_storage::lazy_source {
     file_reader<paged_file_impl> reader;
     spf_lexer<file_reader<paged_file_impl>> lexer;
@@ -2903,46 +2702,20 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string&
         try {
             for_each_instance_header<index_tokens>(chunk_reader, chunk_lexer, end, schema, bypassed_types, out.bypassed, logger_.get(), [&](uint32_t name, const ifcopenshell::declaration* declaration, size_t) {
                 const uint64_t attributes_offset = chunk_reader.tell();
-                const uint16_t type_index = (uint16_t)declaration->index_in_schema();
+                attribute_consumer consumer{out.inverses, name, (uint16_t)declaration->index_in_schema()};
-                int depth = 1;
+                chunk_lexer.scan(consumer);
-                int attribute = 0;
+                chunk_lexer.reset_pool();
-                bool first_value = true;
+                if (consumer.failure != nullptr) {
-                size_t guid_begin = 0, guid_end = 0;
+                    out.failure = consumer.failure;
-                while (depth > 0) {
+                    out.failure_offset = consumer.failure_offset;
-                    token t = chunk_lexer.next<attribute_tokens>();
+                    return false;
-                    if (!t) {
+                }
+                if (!consumer.done) {
                     out.failure = "file ends inside an instance";
                     out.failure_offset = attributes_offset;
                     return false;
                 }
-                    if (t.is_operator()) {
+                const size_t guid_begin = consumer.guid_begin, guid_end = consumer.guid_end;
-                        if (t.value_char == '(') {
-                            ++depth;
-                        } else if (t.value_char == ')') {
-                            --depth;
-                        } else if (t.value_char == ',' && depth == 1) {
-                            ++attribute;
-                        } else if (t.value_char == ';') {
-                            out.failure = "; inside an instance";
-                            out.failure_offset = t.start_pos;
-                            return false;
-                        }
-                    } else if (t.is_identifier()) {
-                        out.inverses.add((uint32_t)t.as_identifier(), name, type_index, attribute);
-                    } else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) {
-                        guid_begin = t.start_pos + 1;
-                        guid_end = chunk_reader.tell() - 1;
-                    }
-                    if (depth == 1) {
-                        first_value = false;
-                    }
-                    chunk_lexer.reset_pool();
-                }
-                if (!chunk_lexer.next<attribute_tokens>().is_operator(';')) {
-                    out.failure = "expected ; after )";
-                    out.failure_offset = chunk_reader.tell();
-                    return false;
-                }
                 out.shells.push_back(ifcopenshell::make_pointer_type<instance_data>(file, declaration, name, instance_data::lazy_tag{}));
                 out.offsets.push_back({name, attributes_offset});
                 if (guid_end > guid_begin && declaration->is(*ifcroot)) {

src/ifcparse/parse.h

@@ -66,14 +66,7 @@ struct index_tokens {
     static constexpr bool decode_values = false;
     static constexpr bool keep_keywords = true;
 };
-/// Inside an attribute list the index only looks at operators and names, so
+
-/// a keyword (an inline typed value such as IFCLABEL), an enumeration or a
-/// binary comes back as Token_LITERAL without its text being copied.
-struct attribute_tokens {
-    static constexpr bool decode_strings = false;
-    static constexpr bool decode_values = false;
-    static constexpr bool keep_keywords = false;
-};
 
 /// A stream of tokens to be read from a file_reader.
 template <typename Reader>
@@ -106,9 +99,14 @@ class IFC_PARSE_API spf_lexer {
     Reader* stream;
     // file* file;
     spf_lexer(Reader* stream, ifcopenshell::logger& logger = ifcopenshell::logger::root());
-    // The next token. With index_tokens a string, number, enumeration or
+    // The tokenizer: every token from the cursor on is handed to the
-    // binary comes back as Token_LITERAL (Token_STRING for a string) with
+    // consumer's callbacks, which inline into the loop; see spf_scan.h.
-    // only its position; names, keywords and operators are always read.
+    template <typename Consumer>
+    void scan(Consumer& consumer);
+    // The next token, through scan() with a consumer that stops after one.
+    // With index_tokens a string, number, enumeration or binary comes back
+    // as Token_LITERAL (Token_STRING for a string) with only its position;
+    // names, keywords and operators are always read.
     template <typename Policy = full_tokens>
     token next();
     ~spf_lexer();
@@ -122,4 +120,6 @@ IFC_PARSE_API std::vector<express::base> traverse_breadth_first(const express::b
 
 IFC_PARSE_API std::ostream& operator<<(std::ostream& stream, const ifcopenshell::file& file);
 
+#include "spf_scan.h"
+
 #endif

src/ifcparse/spf_scan.h

@@ -0,0 +1,396 @@
+/********************************************************************************
+ *                                                                              *
+ * This file is part of IfcOpenShell.                                           *
+ *                                                                              *
+ * IfcOpenShell is free software: you can redistribute it and/or modify         *
+ * it under the terms of the Lesser GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3.0 of the License, or          *
+ * (at your option) any later version.                                          *
+ *                                                                              *
+ * IfcOpenShell is distributed in the hope that it will be useful,              *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of               *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
+ * Lesser GNU General Public License for more details.                          *
+ *                                                                              *
+ * You should have received a copy of the Lesser GNU General Public License     *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.         *
+ *                                                                              *
+ ********************************************************************************/
+
+// The tokenizer's body, spf_lexer::scan(), and what it needs. It lives in a
+// header so a consumer's callbacks inline into the loop; next() is the
+// consumer that stops after one token.
+
+#ifndef IFCPARSE_SPF_SCAN_H
+#define IFCPARSE_SPF_SCAN_H
+
+#include "parse.h"
+
+#include <charconv>
+#include <cstring>
+#include <string>
+
+namespace ifcopenshell {
+
+#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
+IFC_PARSE_API double parse_double_c(const char* start, char** end);
+#endif
+
+template <typename T>
+inline bool parse_num_(const char* pStart, size_t size, T& val) {
+    if (size == 0) {
+        return false;
+    }
+    if (*pStart == '+') {
+        ++pStart;
+        --size;
+        if (size == 0) {
+            return false;
+        }
+    }
+    if constexpr (std::is_floating_point_v<T>) {
+#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
+        // pStart is NUL-terminated at pStart + size (callers pass c_str()), so
+        // strtod_l stops exactly at the end of a well-formed number. from_chars
+        // is not instantiated for double here — its float overload is =deleted
+        // in libc++ (Apple's and Emscripten's).
+        char* pEnd = nullptr;
+        const double result = parse_double_c(pStart, &pEnd);
+        if (pEnd != pStart + size) {
+            return false;
+        }
+        val = static_cast<T>(result);
+        return true;
+#else
+        auto re = std::from_chars(pStart, pStart + size, val);
+        return re.ec == std::errc() && re.ptr == pStart + size;
+#endif
+    } else {
+        auto re = std::from_chars(pStart, pStart + size, val);
+        return re.ec == std::errc() && re.ptr == pStart + size;
+    }
+}
+
+
+// These helpers sit on the tokenizer's innermost loop; left to the
+// compiler's heuristics they end up as calls, one per eight bytes.
+#if defined(_MSC_VER)
+#define IFC_SWAR_INLINE __forceinline
+#else
+#define IFC_SWAR_INLINE inline __attribute__((always_inline))
+#endif
+
+namespace SWAR {
+constexpr uint32_t ONES32 = 0x01010101u;
+constexpr uint32_t HIGHS32 = 0x80808080u;
+constexpr uint64_t ONES = 0x0101010101010101ull;
+constexpr uint64_t HIGHS = 0x8080808080808080ull;
+
+constexpr uint64_t splat(unsigned char c) {
+    return ONES * c;
+}
+
+IFC_SWAR_INLINE uint32_t has_zero_byte(uint32_t x) {
+    return (x - ONES32) & ~x & HIGHS32;
+}
+
+IFC_SWAR_INLINE uint64_t has_zero_byte(uint64_t x) {
+    return (x - ONES) & ~x & HIGHS;
+}
+
+IFC_SWAR_INLINE uint32_t eq_mask(uint32_t x, uint32_t c) {
+    return has_zero_byte(x ^ c);
+}
+
+IFC_SWAR_INLINE uint64_t eq_mask(uint64_t x, uint64_t c) {
+    return has_zero_byte(x ^ c);
+}
+
+namespace chars {
+constexpr uint64_t lpar = splat('(');
+constexpr uint64_t rpar = splat(')');
+constexpr uint64_t eq = splat('=');
+constexpr uint64_t comma = splat(',');
+constexpr uint64_t semi = splat(';');
+constexpr uint64_t slash = splat('/');
+
+constexpr uint64_t space = splat(' ');
+constexpr uint64_t cr = splat('\r');
+constexpr uint64_t lf = splat('\n');
+constexpr uint64_t tab = splat('\t');
+
+constexpr uint64_t quote = splat('"');
+constexpr uint64_t dot = splat('.');
+} // namespace chars
+
+template <bool IncludeDot = true>
+IFC_SWAR_INLINE uint64_t has_special_char(uint64_t x) {
+    return eq_mask(x, chars::lpar) |
+           eq_mask(x, chars::rpar) |
+           eq_mask(x, chars::eq) |
+           eq_mask(x, chars::comma) |
+           eq_mask(x, chars::semi) |
+           eq_mask(x, chars::slash) |
+           eq_mask(x, chars::space) |
+           eq_mask(x, chars::cr) |
+           eq_mask(x, chars::lf) |
+           eq_mask(x, chars::tab) |
+           eq_mask(x, chars::quote) |
+           (IncludeDot ? eq_mask(x, chars::dot) : uint64_t{0});
+}
+
+template <bool IncludeDot = true>
+IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) {
+    return eq_mask(x, static_cast<uint32_t>(chars::lpar)) |
+           eq_mask(x, static_cast<uint32_t>(chars::rpar)) |
+           eq_mask(x, static_cast<uint32_t>(chars::eq)) |
+           eq_mask(x, static_cast<uint32_t>(chars::comma)) |
+           eq_mask(x, static_cast<uint32_t>(chars::semi)) |
+           eq_mask(x, static_cast<uint32_t>(chars::slash)) |
+           eq_mask(x, static_cast<uint32_t>(chars::space)) |
+           eq_mask(x, static_cast<uint32_t>(chars::cr)) |
+           eq_mask(x, static_cast<uint32_t>(chars::lf)) |
+           eq_mask(x, static_cast<uint32_t>(chars::tab)) |
+           eq_mask(x, static_cast<uint32_t>(chars::quote)) |
+           (IncludeDot ? eq_mask(x, static_cast<uint32_t>(chars::dot)) : uint32_t{0});
+}
+
+}
+
+
+// One pass over the tokens from the cursor, handing each to the consumer
+// without building a token; each callback returns whether to go on. The
+// consumer's constexpr flags decide what is decoded: decode_strings (else a
+// string is ended, not decoded, and reported by its bounds), decode_values
+// (else numbers, enumerations and binaries are reported as literals by
+// position only) and keep_keywords (else a keyword is a literal too, so
+// nothing is copied but a name's digits). Returns at the end of the input.
+template <typename Reader>
+template <typename Consumer>
+void spf_lexer<Reader>::scan(Consumer& consumer) {
+    while (true) {
+        if (stream->eof()) {
+            return;
+        }
+
+        auto pos = stream->tell();
+        char character = stream->read();
+
+        if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
+            if (character == '/') {
+                // skip_comment() wants to see the slash itself, so a comment
+                // that follows the previous token without whitespace is skipped.
+                stream->seek(pos);
+            }
+            while ((skip_whitespace() != 0U) || (skip_comment() != 0U)) {
+            }
+            if (stream->eof()) {
+                return;
+            }
+            pos = stream->tell();
+            character = stream->read();
+        }
+
+        // If the cursor is at [()=,;$*] we know token consists of single char
+        if (character == '(' ||
+            character == ')' ||
+            character == '=' ||
+            character == ',' ||
+            character == ';' ||
+            character == '$' ||
+            character == '*')
+        {
+            if (!consumer.operator_(pos, character)) {
+                return;
+            }
+            continue;
+        }
+
+        if (character == '\'') {
+            // If a string is encountered defer processing to the character_decoder
+            if constexpr (Consumer::decode_strings) {
+                auto& str = get_temp_string();
+                str = *decoder_;
+                if (!consumer.string(pos, str)) {
+                    return;
+                }
+            } else {
+                decoder_->skip();
+                if (!consumer.string(pos, stream->tell())) {
+                    return;
+                }
+            }
+            continue;
+        }
+
+        auto& str = get_temp_string();
+        auto ttype = token::Token_NONE;
+        if (character == '"' || character == '.') {
+            if (character == '"') {
+                ttype = token::Token_BINARY;
+            } else {
+                ttype = token::Token_ENUMERATION;
+            }
+            str.clear();
+        } else if (character == '#') {
+            ttype = token::Token_IDENTIFIER;
+            str.clear();
+        } else {
+            str.assign(&character, 1);
+        }
+
+        auto remaining = stream->remaining();
+        while (remaining) {
+            if (remaining >= 8) {
+                uint64_t x = stream->peek_u64();
+                if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
+                    if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                        str.append(reinterpret_cast<const char*>(&x), 8);
+                    }
+                    stream->increment(8);
+                    remaining -= 8;
+                    continue;
+                }
+            }
+            if (remaining >= 4) {
+                uint32_t x = stream->peek_u32();
+                if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
+                    if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                        str.append(reinterpret_cast<const char*>(&x), 4);
+                    }
+                    stream->increment(4);
+                    remaining -= 4;
+                    continue;
+                }
+            }
+
+            // Read character and increment pointer if not starting a new token
+            char c = stream->peek();
+            if (c == '(' ||
+                c == ')' ||
+                c == '=' ||
+                c == ',' ||
+                c == ';' ||
+                c == '/') {
+                break;
+            }
+            if (!(c == ' ' || c == '\r' || c == '\n' || c == '\t')) {
+                if ((ttype == token::Token_BINARY && c == '"') ||
+                    (ttype == token::Token_ENUMERATION && c == '.')) {
+                    // Skip
+                } else if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                    str.push_back(c);
+                }
+            }
+            stream->increment();
+            remaining -= 1;
+        }
+
+        if (ttype == token::Token_IDENTIFIER) {
+            int int_val;
+            if (!parse_num_(str.c_str(), str.size(), int_val)) {
+                throw invalid_token_exception(pos, str, "instance name");
+            }
+            pop_pool_entry();
+            if (!consumer.identifier(pos, (uint32_t)int_val)) {
+                return;
+            }
+            continue;
+        }
+
+        if constexpr (!Consumer::decode_values) {
+            // Only names and keywords are read; everything else is a literal
+            // whose position is all the consumer wants.
+            if constexpr (Consumer::keep_keywords) {
+                if (ttype == token::Token_NONE && !str.empty()) {
+                    const char first = str.front();
+                    if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
+                        if (!consumer.keyword(pos, str)) {
+                            return;
+                        }
+                        continue;
+                    }
+                }
+            }
+            pop_pool_entry();
+            if (!consumer.literal(pos)) {
+                return;
+            }
+            continue;
+        } else {
+            if (ttype == token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) {
+                pop_pool_entry();
+                if (!consumer.boolean(pos, str[0])) {
+                    return;
+                }
+                continue;
+            } else if (ttype == token::Token_NONE && !str.empty()) {
+                int64_t int_val;
+                double float_val;
+                auto& first = str.front();
+                if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
+                    if (!consumer.keyword(pos, str)) {
+                        return;
+                    }
+                    continue;
+                } else if (parse_num_(str.c_str(), str.size(), int_val)) {
+                    pop_pool_entry();
+                    if (!consumer.integer(pos, int_val)) {
+                        return;
+                    }
+                    continue;
+                } else if (parse_num_(str.c_str(), str.size(), float_val)) {
+                    pop_pool_entry();
+                    if (!consumer.real(pos, float_val)) {
+                        return;
+                    }
+                    continue;
+                }
+            } else if (ttype == token::Token_BINARY) {
+                if (!consumer.binary(pos, str)) {
+                    return;
+                }
+                continue;
+            } else if (ttype == token::Token_ENUMERATION) {
+                if (!consumer.enumeration(pos, str)) {
+                    return;
+                }
+                continue;
+            }
+
+            throw invalid_token_exception(pos, str, "valid token");
+        }
+    }
+}
+
+// The consumer behind next(): builds one token and stops.
+template <typename Policy>
+struct token_consumer {
+    static constexpr bool decode_strings = Policy::decode_strings;
+    static constexpr bool decode_values = Policy::decode_values;
+    static constexpr bool keep_keywords = Policy::keep_keywords;
+    token result;
+    bool operator_(size_t pos, char c) { result = token(pos, c); return false; }
+    bool identifier(size_t pos, uint32_t name) { result = token(pos, token::Token_IDENTIFIER, (int64_t)name); return false; }
+    bool string(size_t pos, const std::string& text) { result = token(pos, token::Token_STRING, text); return false; }
+    bool string(size_t pos, size_t /*end*/) { result = token(pos, token::Token_STRING); return false; }
+    bool keyword(size_t pos, const std::string& text) { result = token(pos, token::Token_KEYWORD, text); return false; }
+    bool enumeration(size_t pos, const std::string& text) { result = token(pos, token::Token_ENUMERATION, text); return false; }
+    bool binary(size_t pos, const std::string& text) { result = token(pos, token::Token_BINARY, text); return false; }
+    bool boolean(size_t pos, char c) { result = token(pos, token::Token_BOOL, c); return false; }
+    bool integer(size_t pos, int64_t value) { result = token(pos, token::Token_INT, value); return false; }
+    bool real(size_t pos, double value) { result = token(pos, value); return false; }
+    bool literal(size_t pos) { result = token(pos, token::Token_LITERAL); return false; }
+};
+
+template <typename Reader>
+template <typename Policy>
+token spf_lexer<Reader>::next() {
+    token_consumer<Policy> consumer;
+    scan(consumer);
+    return consumer.result;
+}
+
+} // namespace ifcopenshell
+
+#endif

src/ifcparse/tests/test_ifcopenshell_parse.cpp

@@ -296,6 +296,19 @@ TEST_CASE("Only a 22-character GlobalId is indexed", "[ifcparse]") {
     wall.set_attribute_value(0, std::string("1F$7lN9$r5MOA_lpAoNM52"));
     CHECK(file.instance_by_guid("1F$7lN9$r5MOA_lpAoNM52").id() == 2);
 }
+namespace {
+struct recording_consumer {
+    static constexpr bool decode_strings = false;
+    static constexpr bool decode_values = false;
+    static constexpr bool keep_keywords = false;
+    std::vector<std::pair<size_t, char>> seen;
+    bool operator_(size_t pos, char c) { seen.push_back({pos, c}); return true; }
+    bool identifier(size_t pos, uint32_t) { seen.push_back({pos, '#'}); return true; }
+    bool string(size_t pos, size_t) { seen.push_back({pos, '\''}); return true; }
+    bool literal(size_t pos) { seen.push_back({pos, 'L'}); return true; }
+};
+}
+
 TEST_CASE("The index token policy ends every token where the full policy does, without decoding", "[ifcparse]") {
     // Doubled quotes, a \S\' escape (an apostrophe as the page character,
     // which a byte scan would take for the end of the string), a \X2\
@@ -336,6 +349,30 @@ TEST_CASE("The index token policy ends every token where the full policy does, w
     }
     CHECK(count == 34);
     CHECK(names == std::vector<unsigned>{1, 2, 3});
+    // A scan() consumer that decodes nothing sees the same tokens at the same
+    // positions as next() under the index policy, in one pass.
+    ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> scan_reader(data, ifcopenshell::caller_fed_tag{});
+    ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> scanner(&scan_reader);
+    recording_consumer recorded;
+    scanner.scan(recorded);
+    ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> index_again(data, ifcopenshell::caller_fed_tag{});
+    ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> index2(&index_again);
+    std::vector<std::pair<size_t, char>> expected;
+    while (true) {
+        ifcopenshell::token tk = index2.next<ifcopenshell::index_tokens>();
+        if (!tk) {
+            break;
+        }
+        expected.push_back({tk.start_pos, tk.is_operator() ? tk.value_char : tk.is_identifier() ? '#' : tk.is_string() ? '\'' : (tk.is_keyword() ? 'K' : 'L')});
+        index2.reset_pool();
+    }
+    // Keywords inside the attribute list are literals to a consumer that keeps no keyword text.
+    for (auto& e : expected) {
+        if (e.second == 'K') {
+            e.second = 'L';
+        }
+    }
+    CHECK(recorded.seen == expected);
     // And the full policy decoded the escapes.
     ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> again(data, ifcopenshell::caller_fed_tag{});
     ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> lexer(&again);