Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-21 12:13:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use scalar scanning for discarded token text

Browse Source 
Keep the existing delimiter logic and compile out repeated SWAR probes for consumers that do not retain text. On Clinic MEP the index scan rises from about 200 to 285 MB/s in memory before the identifier optimization.

Generated with the assistance of an AI coding tool.
This commit is contained in:
Thomas Krijnen
2026-09-15 14:17:14 +02:00
parent cb66a2ddae
commit 188319996d
1 changed files with 21 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/spf_scan.h
+21 -17
View File
@@ -243,26 +243,30 @@ void spf_lexer<Reader>::scan(Consumer& consumer) {
auto remaining = stream->remaining();
while (remaining) {
if (remaining >= 8) {
uint64_t x = stream->peek_u64();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
text->append(reinterpret_cast<const char*>(&x), 8);
// Most index tokens are shorter than a word. Testing both word
// widths at every byte costs more than the scalar delimiter check.
if constexpr (Consumer::keep_keywords) {
if (remaining >= 8) {
uint64_t x = stream->peek_u64();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
text->append(reinterpret_cast<const char*>(&x), 8);
}
stream->increment(8);
remaining -= 8;
continue;
}
stream->increment(8);
remaining -= 8;
continue;
}
}
if (remaining >= 4) {
uint32_t x = stream->peek_u32();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
text->append(reinterpret_cast<const char*>(&x), 4);
if (remaining >= 4) {
uint32_t x = stream->peek_u32();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
text->append(reinterpret_cast<const char*>(&x), 4);
}
stream->increment(4);
remaining -= 4;
continue;
}
stream->increment(4);
remaining -= 4;
continue;
}
}