From 188319996d859d4b1eec8721a4f6ca3a51b538cd Mon Sep 17 00:00:00 2001
From: Thomas Krijnen <t.krijnen@gmail.com>
Date: Tue, 15 Sep 2026 14:17:14 +0200
Subject: [PATCH] Use scalar scanning for discarded token text

Keep the existing delimiter logic and compile out repeated SWAR probes for consumers that do not retain text. On Clinic MEP the index scan rises from about 200 to 285 MB/s in memory before the identifier optimization.

Generated with the assistance of an AI coding tool.
---
 src/ifcparse/spf_scan.h | 38 +++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/src/ifcparse/spf_scan.h b/src/ifcparse/spf_scan.h
index 6d4615e1c8..61956deff7 100644
--- a/src/ifcparse/spf_scan.h
+++ b/src/ifcparse/spf_scan.h
@@ -243,26 +243,30 @@ void spf_lexer<Reader>::scan(Consumer& consumer) {
 
         auto remaining = stream->remaining();
         while (remaining) {
-            if (remaining >= 8) {
-                uint64_t x = stream->peek_u64();
-                if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
-                    if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
-                        text->append(reinterpret_cast<const char*>(&x), 8);
+            // Most index tokens are shorter than a word. Testing both word
+            // widths at every byte costs more than the scalar delimiter check.
+            if constexpr (Consumer::keep_keywords) {
+                if (remaining >= 8) {
+                    uint64_t x = stream->peek_u64();
+                    if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
+                        if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                            text->append(reinterpret_cast<const char*>(&x), 8);
+                        }
+                        stream->increment(8);
+                        remaining -= 8;
+                        continue;
                     }
-                    stream->increment(8);
-                    remaining -= 8;
-                    continue;
                 }
-            }
-            if (remaining >= 4) {
-                uint32_t x = stream->peek_u32();
-                if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
-                    if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
-                        text->append(reinterpret_cast<const char*>(&x), 4);
+                if (remaining >= 4) {
+                    uint32_t x = stream->peek_u32();
+                    if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
+                        if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
+                            text->append(reinterpret_cast<const char*>(&x), 4);
+                        }
+                        stream->increment(4);
+                        remaining -= 4;
+                        continue;
                     }
-                    stream->increment(4);
-                    remaining -= 4;
-                    continue;
                 }
             }