Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-21 11:33:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcparse: the tokenizer as scan(Consumer&), next() as its one-token consumer

Browse Source 
The body of next() becomes spf_lexer::scan(Consumer&), the same code
wrapped in a loop that hands each token to the consumer's callbacks
(operator_, identifier, string, keyword, enumeration, binary, boolean,
integer, real, literal) instead of building a token object; each
callback returns whether to go on. The consumer's constexpr flags say
what is decoded: decode_strings, decode_values, keep_keywords. It lives
in spf_scan.h, with the SWAR helpers and number parsing it needs, so a
consumer inlines into the loop. next<Policy>() is kept as the consumer
that stops after one token: the attribute reader, header parser and
streamer pull tokens recursively and stay as they are.

The lazy index is now attribute_consumer: depth and attribute index from
the operators, every name straight into the inverse index, the bounds
of the first attribute if it is a string, done at the closing semicolon.
The attribute_tokens policy it replaces is gone.

Tokenizing 50 MB files with nothing decoded, in memory: index policy
through next() 214–247 MB/s, scan() with the inlined consumer 314–403
MB/s (TXG 247 → 345); through 64 KB pages 245–284 MB/s. Lazy open on one
thread TXG / 210_King / OKgate22 0.55 / 1.59 / 2.11 s → 0.52 / 1.57 /
2.05 s. The full tokenizer through the adapter is unchanged (TXG 202–210
MB/s against 195–219 before), as is the strict parse.

This commit was written by an AI coding tool and has not been verified by
a human.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL
This commit is contained in:
Dion Moult
2026-09-15 06:13:09 +10:00
parent f24e9a6bca
commit 17ec5d4504
4 changed files with 557 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/parse.cpp
+113 -340
View File
@@ -137,313 +137,14 @@ std::string& spf_lexer<Reader>::get_temp_string() const {
return (*stringpool_[slice])[offset];
}
namespace {
#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
double parse_double_c(const char* start, char** end) {
double ifcopenshell::parse_double_c(const char* start, char** end) {
static const locale_t loc = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0);
return strtod_l(start, end, loc);
}
#endif
template <typename T>
bool parse_num_(const char* pStart, size_t size, T& val) {
if (size == 0) {
return false;
}
if (*pStart == '+') {
++pStart;
--size;
if (size == 0) {
return false;
}
}
if constexpr (std::is_floating_point_v<T>) {
#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
// pStart is NUL-terminated at pStart + size (callers pass c_str()), so
// strtod_l stops exactly at the end of a well-formed number. from_chars
// is not instantiated for double here — its float overload is =deleted
// in libc++ (Apple's and Emscripten's).
char* pEnd = nullptr;
const double result = parse_double_c(pStart, &pEnd);
if (pEnd != pStart + size) {
return false;
}
val = static_cast<T>(result);
return true;
#else
auto re = std::from_chars(pStart, pStart + size, val);
return re.ec == std::errc() && re.ptr == pStart + size;
#endif
} else {
auto re = std::from_chars(pStart, pStart + size, val);
return re.ec == std::errc() && re.ptr == pStart + size;
}
}
} // namespace
// These helpers sit on the tokenizer's innermost loop; left to the
// compiler's heuristics they end up as calls, one per eight bytes.
#if defined(_MSC_VER)
#define IFC_SWAR_INLINE __forceinline
#else
#define IFC_SWAR_INLINE inline __attribute__((always_inline))
#endif
namespace SWAR {
constexpr uint32_t ONES32 = 0x01010101u;
constexpr uint32_t HIGHS32 = 0x80808080u;
constexpr uint64_t ONES = 0x0101010101010101ull;
constexpr uint64_t HIGHS = 0x8080808080808080ull;
constexpr uint64_t splat(unsigned char c) {
return ONES * c;
}
IFC_SWAR_INLINE uint32_t has_zero_byte(uint32_t x) {
return (x - ONES32) & ~x & HIGHS32;
}
IFC_SWAR_INLINE uint64_t has_zero_byte(uint64_t x) {
return (x - ONES) & ~x & HIGHS;
}
IFC_SWAR_INLINE uint32_t eq_mask(uint32_t x, uint32_t c) {
return has_zero_byte(x ^ c);
}
IFC_SWAR_INLINE uint64_t eq_mask(uint64_t x, uint64_t c) {
return has_zero_byte(x ^ c);
}
namespace chars {
constexpr uint64_t lpar = splat('(');
constexpr uint64_t rpar = splat(')');
constexpr uint64_t eq = splat('=');
constexpr uint64_t comma = splat(',');
constexpr uint64_t semi = splat(';');
constexpr uint64_t slash = splat('/');
constexpr uint64_t space = splat(' ');
constexpr uint64_t cr = splat('\r');
constexpr uint64_t lf = splat('\n');
constexpr uint64_t tab = splat('\t');
constexpr uint64_t quote = splat('"');
constexpr uint64_t dot = splat('.');
} // namespace chars
template <bool IncludeDot = true>
IFC_SWAR_INLINE uint64_t has_special_char(uint64_t x) {
return eq_mask(x, chars::lpar) |
eq_mask(x, chars::rpar) |
eq_mask(x, chars::eq) |
eq_mask(x, chars::comma) |
eq_mask(x, chars::semi) |
eq_mask(x, chars::slash) |
eq_mask(x, chars::space) |
eq_mask(x, chars::cr) |
eq_mask(x, chars::lf) |
eq_mask(x, chars::tab) |
eq_mask(x, chars::quote) |
(IncludeDot ? eq_mask(x, chars::dot) : uint64_t{0});
}
template <bool IncludeDot = true>
IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) {
return eq_mask(x, static_cast<uint32_t>(chars::lpar)) |
eq_mask(x, static_cast<uint32_t>(chars::rpar)) |
eq_mask(x, static_cast<uint32_t>(chars::eq)) |
eq_mask(x, static_cast<uint32_t>(chars::comma)) |
eq_mask(x, static_cast<uint32_t>(chars::semi)) |
eq_mask(x, static_cast<uint32_t>(chars::slash)) |
eq_mask(x, static_cast<uint32_t>(chars::space)) |
eq_mask(x, static_cast<uint32_t>(chars::cr)) |
eq_mask(x, static_cast<uint32_t>(chars::lf)) |
eq_mask(x, static_cast<uint32_t>(chars::tab)) |
eq_mask(x, static_cast<uint32_t>(chars::quote)) |
(IncludeDot ? eq_mask(x, static_cast<uint32_t>(chars::dot)) : uint32_t{0});
}
}
//
// Returns the offset of the current token and moves cursor to next
//
template <typename Reader>
template <typename Policy>
token spf_lexer<Reader>::next() {
if (stream->eof()) {
return token{};
}
auto pos = stream->tell();
char character = stream->read();
if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
if (character == '/') {
// skip_comment() wants to see the slash itself, so a comment
// that follows the previous token without whitespace is skipped.
stream->seek(pos);
}
while ((skip_whitespace() != 0U) || (skip_comment() != 0U)) {
}
if (stream->eof()) {
return token{};
}
pos = stream->tell();
character = stream->read();
}
// If the cursor is at [()=,;$*] we know token consists of single char
if (character == '(' ||
character == ')' ||
character == '=' ||
character == ',' ||
character == ';' ||
character == '$' ||
character == '*')
{
return token(pos, character);
}
auto& str = get_temp_string();
if (character == '\'') {
// If a string is encountered defer processing to the character_decoder
if constexpr (Policy::decode_strings) {
str = *decoder_;
return token(pos, token::Token_STRING, str);
} else {
decoder_->skip();
pop_pool_entry();
return token(pos, token::Token_STRING);
}
} else {
auto ttype = token::Token_NONE;
if (character == '"' || character == '.') {
if (character == '"') {
ttype = token::Token_BINARY;
} else {
ttype = token::Token_ENUMERATION;
}
str.clear();
} else if (character == '#') {
ttype = token::Token_IDENTIFIER;
str.clear();
} else {
str.assign(&character, 1);
}
auto remaining = stream->remaining();
while (remaining) {
if (remaining >= 8) {
uint64_t x = stream->peek_u64();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
str.append(reinterpret_cast<const char*>(&x), 8);
}
stream->increment(8);
remaining -= 8;
continue;
}
}
if (remaining >= 4) {
uint32_t x = stream->peek_u32();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
str.append(reinterpret_cast<const char*>(&x), 4);
}
stream->increment(4);
remaining -= 4;
continue;
}
}
// Read character and increment pointer if not starting a new token
char character = stream->peek();
if (character == '(' ||
character == ')' ||
character == '=' ||
character == ',' ||
character == ';' ||
character == '/') {
break;
}
if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
if ((ttype == token::Token_BINARY && character == '"') ||
(ttype == token::Token_ENUMERATION && character == '.')) {
// Skip
} else if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) {
str.push_back(character);
}
}
stream->increment();
remaining -= 1;
}
if constexpr (!Policy::decode_values) {
// Only names and keywords are read; everything else is a literal
// whose position is all the caller wants.
if constexpr (!Policy::keep_keywords) {
if (ttype != token::Token_IDENTIFIER) {
pop_pool_entry();
return token(pos, token::Token_LITERAL);
}
}
if (ttype == token::Token_IDENTIFIER) {
int int_val;
if (!parse_num_(str.c_str(), str.size(), int_val)) {
throw invalid_token_exception(pos, str, "instance name");
}
pop_pool_entry();
return token(pos, ttype, (int64_t)int_val);
}
if (ttype == token::Token_NONE && !str.empty()) {
const char first = str.front();
if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
return token(pos, token::Token_KEYWORD, str);
}
}
pop_pool_entry();
return token(pos, token::Token_LITERAL);
}
if (ttype == token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) {
pop_pool_entry();
return token(pos, token::Token_BOOL, str[0]);
} else if (ttype == token::Token_IDENTIFIER) {
int int_val;
if (!parse_num_(str.c_str(), str.size(), int_val)) {
throw invalid_token_exception(pos, str, "instance name");
}
pop_pool_entry();
return token(pos, ttype, (int64_t)int_val);
} else if (ttype == token::Token_NONE && !str.empty()) {
int64_t int_val;
double float_val;
auto& first = str.front();
if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
ttype = token::Token_KEYWORD;
return token(pos, ttype, str);
} else if (parse_num_(str.c_str(), str.size(), int_val)) {
ttype = token::Token_INT;
pop_pool_entry();
return token(pos, ttype, int_val);
} else if (parse_num_(str.c_str(), str.size(), float_val)) {
ttype = token::Token_FLOAT;
pop_pool_entry();
return token(pos, float_val);
}
} else if (ttype == token::Token_BINARY || ttype == token::Token_ENUMERATION) {
return token(pos, ttype, str);
}
throw invalid_token_exception(pos, str, "valid token");
}
}
template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<full_buffer_impl>>;
template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<paged_file_impl>>;
template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<pushed_sequential_impl>>;
@@ -453,8 +154,7 @@ template class IFC_PARSE_API ifcopenshell::spf_lexer<file_reader<mmap_impl>>;
#define IFC_INSTANTIATE_LEXER_NEXT(Reader) \
template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::full_tokens>(); \
template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>(); \
template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::attribute_tokens>();
template IFC_PARSE_API token ifcopenshell::spf_lexer<Reader>::next<ifcopenshell::index_tokens>();
IFC_INSTANTIATE_LEXER_NEXT(file_reader<full_buffer_impl>)
IFC_INSTANTIATE_LEXER_NEXT(file_reader<paged_file_impl>)
IFC_INSTANTIATE_LEXER_NEXT(file_reader<pushed_sequential_impl>)
@@ -2791,6 +2491,105 @@ std::vector<size_t> chunk_bounds(const Reader& source, unsigned threads) {
}
namespace {
// The lazy index's consumer for one instance's attribute list, fed by
// spf_lexer::scan() from just past the opening parenthesis: depth and the
// attribute index from the operators, every name straight into the inverse
// index, the bounds of the first attribute if it is a string (the GlobalId
// candidate), and done at the semicolon that closes the instance. Nothing
// is decoded or copied.
struct attribute_consumer {
static constexpr bool decode_strings = false;
static constexpr bool decode_values = false;
static constexpr bool keep_keywords = false;
ifcopenshell::impl::in_memory_file_storage::entities_by_ref& inverses;
uint32_t name;
uint16_t type;
int depth = 1;
int attribute = 0;
bool first_value = true;
bool closed = false;
bool done = false;
size_t guid_begin = 0, guid_end = 0;
const char* failure = nullptr;
size_t failure_offset = 0;
bool after_close(size_t pos) {
failure = "expected ; after )";
failure_offset = pos;
return false;
}
bool operator_(size_t pos, char c) {
if (closed) {
if (c == ';') {
done = true;
return false;
}
return after_close(pos);
}
switch (c) {
case '(':
++depth;
return true;
case ')':
if (--depth == 0) {
closed = true;
}
return true;
case ',':
if (depth == 1) {
++attribute;
}
return true;
case ';':
failure = "; inside an instance";
failure_offset = pos;
return false;
default:
if (depth == 1) {
first_value = false;
}
return true;
}
}
bool identifier(size_t pos, uint32_t referenced) {
if (closed) {
return after_close(pos);
}
inverses.add(referenced, name, type, attribute);
if (depth == 1) {
first_value = false;
}
return true;
}
bool string(size_t begin, size_t end) {
if (closed) {
return after_close(begin);
}
if (depth == 1 && attribute == 0 && first_value) {
guid_begin = begin + 1;
guid_end = end - 1;
}
if (depth == 1) {
first_value = false;
}
return true;
}
bool literal(size_t pos) {
if (closed) {
return after_close(pos);
}
if (depth == 1) {
first_value = false;
}
return true;
}
};
}
struct ifcopenshell::impl::in_memory_file_storage::lazy_source {
file_reader<paged_file_impl> reader;
spf_lexer<file_reader<paged_file_impl>> lexer;
@@ -2903,46 +2702,20 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string&
try {
for_each_instance_header<index_tokens>(chunk_reader, chunk_lexer, end, schema, bypassed_types, out.bypassed, logger_.get(), [&](uint32_t name, const ifcopenshell::declaration* declaration, size_t) {
const uint64_t attributes_offset = chunk_reader.tell();
const uint16_t type_index = (uint16_t)declaration->index_in_schema();
int depth = 1;
int attribute = 0;
bool first_value = true;
size_t guid_begin = 0, guid_end = 0;
while (depth > 0) {
token t = chunk_lexer.next<attribute_tokens>();
if (!t) {
out.failure = "file ends inside an instance";
out.failure_offset = attributes_offset;
return false;
}
if (t.is_operator()) {
if (t.value_char == '(') {
++depth;
} else if (t.value_char == ')') {
--depth;
} else if (t.value_char == ',' && depth == 1) {
++attribute;
} else if (t.value_char == ';') {
out.failure = "; inside an instance";
out.failure_offset = t.start_pos;
return false;
}
} else if (t.is_identifier()) {
out.inverses.add((uint32_t)t.as_identifier(), name, type_index, attribute);
} else if (t.type == token::Token_STRING && depth == 1 && attribute == 0 && first_value) {
guid_begin = t.start_pos + 1;
guid_end = chunk_reader.tell() - 1;
}
if (depth == 1) {
first_value = false;
}
chunk_lexer.reset_pool();
}
if (!chunk_lexer.next<attribute_tokens>().is_operator(';')) {
out.failure = "expected ; after )";
out.failure_offset = chunk_reader.tell();
attribute_consumer consumer{out.inverses, name, (uint16_t)declaration->index_in_schema()};
chunk_lexer.scan(consumer);
chunk_lexer.reset_pool();
if (consumer.failure != nullptr) {
out.failure = consumer.failure;
out.failure_offset = consumer.failure_offset;
return false;
}
if (!consumer.done) {
out.failure = "file ends inside an instance";
out.failure_offset = attributes_offset;
return false;
}
const size_t guid_begin = consumer.guid_begin, guid_end = consumer.guid_end;
out.shells.push_back(ifcopenshell::make_pointer_type<instance_data>(file, declaration, name, instance_data::lazy_tag{}));
out.offsets.push_back({name, attributes_offset});
if (guid_end > guid_begin && declaration->is(*ifcroot)) {
src/ifcparse/parse.h
+11 -11
View File
@@ -66,14 +66,7 @@ struct index_tokens {
static constexpr bool decode_values = false;
static constexpr bool keep_keywords = true;
};
/// Inside an attribute list the index only looks at operators and names, so
/// a keyword (an inline typed value such as IFCLABEL), an enumeration or a
/// binary comes back as Token_LITERAL without its text being copied.
struct attribute_tokens {
static constexpr bool decode_strings = false;
static constexpr bool decode_values = false;
static constexpr bool keep_keywords = false;
};
/// A stream of tokens to be read from a file_reader.
template <typename Reader>
@@ -106,9 +99,14 @@ class IFC_PARSE_API spf_lexer {
Reader* stream;
// file* file;
spf_lexer(Reader* stream, ifcopenshell::logger& logger = ifcopenshell::logger::root());
// The next token. With index_tokens a string, number, enumeration or
// binary comes back as Token_LITERAL (Token_STRING for a string) with
// only its position; names, keywords and operators are always read.
// The tokenizer: every token from the cursor on is handed to the
// consumer's callbacks, which inline into the loop; see spf_scan.h.
template <typename Consumer>
void scan(Consumer& consumer);
// The next token, through scan() with a consumer that stops after one.
// With index_tokens a string, number, enumeration or binary comes back
// as Token_LITERAL (Token_STRING for a string) with only its position;
// names, keywords and operators are always read.
template <typename Policy = full_tokens>
token next();
~spf_lexer();
@@ -122,4 +120,6 @@ IFC_PARSE_API std::vector<express::base> traverse_breadth_first(const express::b
IFC_PARSE_API std::ostream& operator<<(std::ostream& stream, const ifcopenshell::file& file);
#include "spf_scan.h"
#endif
src/ifcparse/spf_scan.h
+396
View File
@@ -0,0 +1,396 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
// The tokenizer's body, spf_lexer::scan(), and what it needs. It lives in a
// header so a consumer's callbacks inline into the loop; next() is the
// consumer that stops after one token.
#ifndef IFCPARSE_SPF_SCAN_H
#define IFCPARSE_SPF_SCAN_H
#include "parse.h"
#include <charconv>
#include <cstring>
#include <string>
namespace ifcopenshell {
#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
IFC_PARSE_API double parse_double_c(const char* start, char** end);
#endif
template <typename T>
inline bool parse_num_(const char* pStart, size_t size, T& val) {
if (size == 0) {
return false;
}
if (*pStart == '+') {
++pStart;
--size;
if (size == 0) {
return false;
}
}
if constexpr (std::is_floating_point_v<T>) {
#if defined(__APPLE__) || defined(__EMSCRIPTEN__)
// pStart is NUL-terminated at pStart + size (callers pass c_str()), so
// strtod_l stops exactly at the end of a well-formed number. from_chars
// is not instantiated for double here — its float overload is =deleted
// in libc++ (Apple's and Emscripten's).
char* pEnd = nullptr;
const double result = parse_double_c(pStart, &pEnd);
if (pEnd != pStart + size) {
return false;
}
val = static_cast<T>(result);
return true;
#else
auto re = std::from_chars(pStart, pStart + size, val);
return re.ec == std::errc() && re.ptr == pStart + size;
#endif
} else {
auto re = std::from_chars(pStart, pStart + size, val);
return re.ec == std::errc() && re.ptr == pStart + size;
}
}
// These helpers sit on the tokenizer's innermost loop; left to the
// compiler's heuristics they end up as calls, one per eight bytes.
#if defined(_MSC_VER)
#define IFC_SWAR_INLINE __forceinline
#else
#define IFC_SWAR_INLINE inline __attribute__((always_inline))
#endif
namespace SWAR {
constexpr uint32_t ONES32 = 0x01010101u;
constexpr uint32_t HIGHS32 = 0x80808080u;
constexpr uint64_t ONES = 0x0101010101010101ull;
constexpr uint64_t HIGHS = 0x8080808080808080ull;
constexpr uint64_t splat(unsigned char c) {
return ONES * c;
}
IFC_SWAR_INLINE uint32_t has_zero_byte(uint32_t x) {
return (x - ONES32) & ~x & HIGHS32;
}
IFC_SWAR_INLINE uint64_t has_zero_byte(uint64_t x) {
return (x - ONES) & ~x & HIGHS;
}
IFC_SWAR_INLINE uint32_t eq_mask(uint32_t x, uint32_t c) {
return has_zero_byte(x ^ c);
}
IFC_SWAR_INLINE uint64_t eq_mask(uint64_t x, uint64_t c) {
return has_zero_byte(x ^ c);
}
namespace chars {
constexpr uint64_t lpar = splat('(');
constexpr uint64_t rpar = splat(')');
constexpr uint64_t eq = splat('=');
constexpr uint64_t comma = splat(',');
constexpr uint64_t semi = splat(';');
constexpr uint64_t slash = splat('/');
constexpr uint64_t space = splat(' ');
constexpr uint64_t cr = splat('\r');
constexpr uint64_t lf = splat('\n');
constexpr uint64_t tab = splat('\t');
constexpr uint64_t quote = splat('"');
constexpr uint64_t dot = splat('.');
} // namespace chars
template <bool IncludeDot = true>
IFC_SWAR_INLINE uint64_t has_special_char(uint64_t x) {
return eq_mask(x, chars::lpar) |
eq_mask(x, chars::rpar) |
eq_mask(x, chars::eq) |
eq_mask(x, chars::comma) |
eq_mask(x, chars::semi) |
eq_mask(x, chars::slash) |
eq_mask(x, chars::space) |
eq_mask(x, chars::cr) |
eq_mask(x, chars::lf) |
eq_mask(x, chars::tab) |
eq_mask(x, chars::quote) |
(IncludeDot ? eq_mask(x, chars::dot) : uint64_t{0});
}
template <bool IncludeDot = true>
IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) {
return eq_mask(x, static_cast<uint32_t>(chars::lpar)) |
eq_mask(x, static_cast<uint32_t>(chars::rpar)) |
eq_mask(x, static_cast<uint32_t>(chars::eq)) |
eq_mask(x, static_cast<uint32_t>(chars::comma)) |
eq_mask(x, static_cast<uint32_t>(chars::semi)) |
eq_mask(x, static_cast<uint32_t>(chars::slash)) |
eq_mask(x, static_cast<uint32_t>(chars::space)) |
eq_mask(x, static_cast<uint32_t>(chars::cr)) |
eq_mask(x, static_cast<uint32_t>(chars::lf)) |
eq_mask(x, static_cast<uint32_t>(chars::tab)) |
eq_mask(x, static_cast<uint32_t>(chars::quote)) |
(IncludeDot ? eq_mask(x, static_cast<uint32_t>(chars::dot)) : uint32_t{0});
}
}
// One pass over the tokens from the cursor, handing each to the consumer
// without building a token; each callback returns whether to go on. The
// consumer's constexpr flags decide what is decoded: decode_strings (else a
// string is ended, not decoded, and reported by its bounds), decode_values
// (else numbers, enumerations and binaries are reported as literals by
// position only) and keep_keywords (else a keyword is a literal too, so
// nothing is copied but a name's digits). Returns at the end of the input.
template <typename Reader>
template <typename Consumer>
void spf_lexer<Reader>::scan(Consumer& consumer) {
while (true) {
if (stream->eof()) {
return;
}
auto pos = stream->tell();
char character = stream->read();
if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
if (character == '/') {
// skip_comment() wants to see the slash itself, so a comment
// that follows the previous token without whitespace is skipped.
stream->seek(pos);
}
while ((skip_whitespace() != 0U) || (skip_comment() != 0U)) {
}
if (stream->eof()) {
return;
}
pos = stream->tell();
character = stream->read();
}
// If the cursor is at [()=,;$*] we know token consists of single char
if (character == '(' ||
character == ')' ||
character == '=' ||
character == ',' ||
character == ';' ||
character == '$' ||
character == '*')
{
if (!consumer.operator_(pos, character)) {
return;
}
continue;
}
if (character == '\'') {
// If a string is encountered defer processing to the character_decoder
if constexpr (Consumer::decode_strings) {
auto& str = get_temp_string();
str = *decoder_;
if (!consumer.string(pos, str)) {
return;
}
} else {
decoder_->skip();
if (!consumer.string(pos, stream->tell())) {
return;
}
}
continue;
}
auto& str = get_temp_string();
auto ttype = token::Token_NONE;
if (character == '"' || character == '.') {
if (character == '"') {
ttype = token::Token_BINARY;
} else {
ttype = token::Token_ENUMERATION;
}
str.clear();
} else if (character == '#') {
ttype = token::Token_IDENTIFIER;
str.clear();
} else {
str.assign(&character, 1);
}
auto remaining = stream->remaining();
while (remaining) {
if (remaining >= 8) {
uint64_t x = stream->peek_u64();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
str.append(reinterpret_cast<const char*>(&x), 8);
}
stream->increment(8);
remaining -= 8;
continue;
}
}
if (remaining >= 4) {
uint32_t x = stream->peek_u32();
if ((ttype == token::Token_NONE ? SWAR::has_special_char<false>(x) : SWAR::has_special_char<true>(x)) == 0) {
if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
str.append(reinterpret_cast<const char*>(&x), 4);
}
stream->increment(4);
remaining -= 4;
continue;
}
}
// Read character and increment pointer if not starting a new token
char c = stream->peek();
if (c == '(' ||
c == ')' ||
c == '=' ||
c == ',' ||
c == ';' ||
c == '/') {
break;
}
if (!(c == ' ' || c == '\r' || c == '\n' || c == '\t')) {
if ((ttype == token::Token_BINARY && c == '"') ||
(ttype == token::Token_ENUMERATION && c == '.')) {
// Skip
} else if (Consumer::keep_keywords || ttype == token::Token_IDENTIFIER) {
str.push_back(c);
}
}
stream->increment();
remaining -= 1;
}
if (ttype == token::Token_IDENTIFIER) {
int int_val;
if (!parse_num_(str.c_str(), str.size(), int_val)) {
throw invalid_token_exception(pos, str, "instance name");
}
pop_pool_entry();
if (!consumer.identifier(pos, (uint32_t)int_val)) {
return;
}
continue;
}
if constexpr (!Consumer::decode_values) {
// Only names and keywords are read; everything else is a literal
// whose position is all the consumer wants.
if constexpr (Consumer::keep_keywords) {
if (ttype == token::Token_NONE && !str.empty()) {
const char first = str.front();
if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
if (!consumer.keyword(pos, str)) {
return;
}
continue;
}
}
}
pop_pool_entry();
if (!consumer.literal(pos)) {
return;
}
continue;
} else {
if (ttype == token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) {
pop_pool_entry();
if (!consumer.boolean(pos, str[0])) {
return;
}
continue;
} else if (ttype == token::Token_NONE && !str.empty()) {
int64_t int_val;
double float_val;
auto& first = str.front();
if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
if (!consumer.keyword(pos, str)) {
return;
}
continue;
} else if (parse_num_(str.c_str(), str.size(), int_val)) {
pop_pool_entry();
if (!consumer.integer(pos, int_val)) {
return;
}
continue;
} else if (parse_num_(str.c_str(), str.size(), float_val)) {
pop_pool_entry();
if (!consumer.real(pos, float_val)) {
return;
}
continue;
}
} else if (ttype == token::Token_BINARY) {
if (!consumer.binary(pos, str)) {
return;
}
continue;
} else if (ttype == token::Token_ENUMERATION) {
if (!consumer.enumeration(pos, str)) {
return;
}
continue;
}
throw invalid_token_exception(pos, str, "valid token");
}
}
}
// The consumer behind next(): builds one token and stops.
template <typename Policy>
struct token_consumer {
static constexpr bool decode_strings = Policy::decode_strings;
static constexpr bool decode_values = Policy::decode_values;
static constexpr bool keep_keywords = Policy::keep_keywords;
token result;
bool operator_(size_t pos, char c) { result = token(pos, c); return false; }
bool identifier(size_t pos, uint32_t name) { result = token(pos, token::Token_IDENTIFIER, (int64_t)name); return false; }
bool string(size_t pos, const std::string& text) { result = token(pos, token::Token_STRING, text); return false; }
bool string(size_t pos, size_t /*end*/) { result = token(pos, token::Token_STRING); return false; }
bool keyword(size_t pos, const std::string& text) { result = token(pos, token::Token_KEYWORD, text); return false; }
bool enumeration(size_t pos, const std::string& text) { result = token(pos, token::Token_ENUMERATION, text); return false; }
bool binary(size_t pos, const std::string& text) { result = token(pos, token::Token_BINARY, text); return false; }
bool boolean(size_t pos, char c) { result = token(pos, token::Token_BOOL, c); return false; }
bool integer(size_t pos, int64_t value) { result = token(pos, token::Token_INT, value); return false; }
bool real(size_t pos, double value) { result = token(pos, value); return false; }
bool literal(size_t pos) { result = token(pos, token::Token_LITERAL); return false; }
};
template <typename Reader>
template <typename Policy>
token spf_lexer<Reader>::next() {
token_consumer<Policy> consumer;
scan(consumer);
return consumer.result;
}
} // namespace ifcopenshell
#endif
src/ifcparse/tests/test_ifcopenshell_parse.cpp
+37
View File
@@ -296,6 +296,19 @@ TEST_CASE("Only a 22-character GlobalId is indexed", "[ifcparse]") {
wall.set_attribute_value(0, std::string("1F$7lN9$r5MOA_lpAoNM52"));
CHECK(file.instance_by_guid("1F$7lN9$r5MOA_lpAoNM52").id() == 2);
}
namespace {
struct recording_consumer {
static constexpr bool decode_strings = false;
static constexpr bool decode_values = false;
static constexpr bool keep_keywords = false;
std::vector<std::pair<size_t, char>> seen;
bool operator_(size_t pos, char c) { seen.push_back({pos, c}); return true; }
bool identifier(size_t pos, uint32_t) { seen.push_back({pos, '#'}); return true; }
bool string(size_t pos, size_t) { seen.push_back({pos, '\''}); return true; }
bool literal(size_t pos) { seen.push_back({pos, 'L'}); return true; }
};
}
TEST_CASE("The index token policy ends every token where the full policy does, without decoding", "[ifcparse]") {
// Doubled quotes, a \S\' escape (an apostrophe as the page character,
// which a byte scan would take for the end of the string), a \X2\
@@ -336,6 +349,30 @@ TEST_CASE("The index token policy ends every token where the full policy does, w
}
CHECK(count == 34);
CHECK(names == std::vector<unsigned>{1, 2, 3});
// A scan() consumer that decodes nothing sees the same tokens at the same
// positions as next() under the index policy, in one pass.
ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> scan_reader(data, ifcopenshell::caller_fed_tag{});
ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> scanner(&scan_reader);
recording_consumer recorded;
scanner.scan(recorded);
ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> index_again(data, ifcopenshell::caller_fed_tag{});
ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> index2(&index_again);
std::vector<std::pair<size_t, char>> expected;
while (true) {
ifcopenshell::token tk = index2.next<ifcopenshell::index_tokens>();
if (!tk) {
break;
}
expected.push_back({tk.start_pos, tk.is_operator() ? tk.value_char : tk.is_identifier() ? '#' : tk.is_string() ? '\'' : (tk.is_keyword() ? 'K' : 'L')});
index2.reset_pool();
}
// Keywords inside the attribute list are literals to a consumer that keeps no keyword text.
for (auto& e : expected) {
if (e.second == 'K') {
e.second = 'L';
}
}
CHECK(recorded.seen == expected);
// And the full policy decoded the escapes.
ifcopenshell::file_reader<ifcopenshell::full_buffer_impl> again(data, ifcopenshell::caller_fed_tag{});
ifcopenshell::spf_lexer<ifcopenshell::file_reader<ifcopenshell::full_buffer_impl>> lexer(&again);