diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index e27941d373..ff362ad53e 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -342,7 +342,9 @@ token spf_lexer::next() { if (remaining >= 8) { uint64_t x = stream->peek_u64(); if ((ttype == token::Token_NONE ? SWAR::has_special_char(x) : SWAR::has_special_char(x)) == 0) { - str.append(reinterpret_cast(&x), 8); + if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) { + str.append(reinterpret_cast(&x), 8); + } stream->increment(8); remaining -= 8; continue; @@ -351,7 +353,9 @@ token spf_lexer::next() { if (remaining >= 4) { uint32_t x = stream->peek_u32(); if ((ttype == token::Token_NONE ? SWAR::has_special_char(x) : SWAR::has_special_char(x)) == 0) { - str.append(reinterpret_cast(&x), 4); + if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) { + str.append(reinterpret_cast(&x), 4); + } stream->increment(4); remaining -= 4; continue; @@ -372,7 +376,7 @@ token spf_lexer::next() { if ((ttype == token::Token_BINARY && character == '"') || (ttype == token::Token_ENUMERATION && character == '.')) { // Skip - } else { + } else if (Policy::keep_keywords || ttype == token::Token_IDENTIFIER) { str.push_back(character); } } @@ -383,6 +387,12 @@ token spf_lexer::next() { if constexpr (!Policy::decode_values) { // Only names and keywords are read; everything else is a literal // whose position is all the caller wants. + if constexpr (!Policy::keep_keywords) { + if (ttype != token::Token_IDENTIFIER) { + pop_pool_entry(); + return token(pos, token::Token_LITERAL); + } + } if (ttype == token::Token_IDENTIFIER) { int int_val; if (!parse_num_(str.c_str(), str.size(), int_val)) { @@ -443,7 +453,8 @@ template class IFC_PARSE_API ifcopenshell::spf_lexer>; #define IFC_INSTANTIATE_LEXER_NEXT(Reader) \ template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); \ - template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); + template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); \ + template IFC_PARSE_API token ifcopenshell::spf_lexer::next(); IFC_INSTANTIATE_LEXER_NEXT(file_reader) IFC_INSTANTIATE_LEXER_NEXT(file_reader) IFC_INSTANTIATE_LEXER_NEXT(file_reader) @@ -2789,7 +2800,7 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& bool first_value = true; size_t guid_begin = 0, guid_end = 0; while (depth > 0) { - token t = lexer.next(); + token t = lexer.next(); if (!t) { failure = "file ends inside an instance"; failure_offset = attributes_offset; @@ -2818,7 +2829,7 @@ bool ifcopenshell::impl::in_memory_file_storage::index_lazily(const std::string& } lexer.reset_pool(); } - if (!lexer.next().is_operator(';')) { + if (!lexer.next().is_operator(';')) { failure = "expected ; after )"; failure_offset = reader.tell(); return false; diff --git a/src/ifcparse/parse.h b/src/ifcparse/parse.h index 348dac5fde..ae2524057a 100644 --- a/src/ifcparse/parse.h +++ b/src/ifcparse/parse.h @@ -59,10 +59,20 @@ IFC_PARSE_API std::string decode_spf_string(const std::string& value); struct full_tokens { static constexpr bool decode_strings = true; static constexpr bool decode_values = true; + static constexpr bool keep_keywords = true; }; struct index_tokens { static constexpr bool decode_strings = false; static constexpr bool decode_values = false; + static constexpr bool keep_keywords = true; +}; +/// Inside an attribute list the index only looks at operators and names, so +/// a keyword (an inline typed value such as IFCLABEL), an enumeration or a +/// binary comes back as Token_LITERAL without its text being copied. +struct attribute_tokens { + static constexpr bool decode_strings = false; + static constexpr bool decode_values = false; + static constexpr bool keep_keywords = false; }; /// A stream of tokens to be read from a file_reader.