From 8e42f35db3be9e738a009af8de5c72bc67201de2 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 26 Mar 2026 15:49:28 +0100 Subject: [PATCH] Rework variable length token storage to use string pool; eliminate need for rereads --- src/ifcparse/IfcCharacterDecoder.cpp | 71 +---- src/ifcparse/IfcCharacterDecoder.h | 3 - src/ifcparse/IfcFile.cpp | 56 ++-- src/ifcparse/IfcParse.cpp | 423 +++++++++++---------------- src/ifcparse/IfcParse.h | 75 +---- src/ifcparse/IfcSpfHeader.cpp | 4 +- src/ifcparse/storage.h | 82 ++++-- 7 files changed, 272 insertions(+), 442 deletions(-) diff --git a/src/ifcparse/IfcCharacterDecoder.cpp b/src/ifcparse/IfcCharacterDecoder.cpp index 71685dcedc..1e9ce9f6b8 100644 --- a/src/ifcparse/IfcCharacterDecoder.cpp +++ b/src/ifcparse/IfcCharacterDecoder.cpp @@ -91,7 +91,7 @@ namespace { unsigned int parse_state = 0; builder_.clear(); - builder_.push_back('\''); + // builder_.push_back('\''); char current_char; int codepage = 1; unsigned int hex = 0; @@ -169,7 +169,7 @@ namespace { } stream_.increment(); } - builder_.push_back('\''); + // builder_.push_back('\''); if (mode == IfcParse::IfcCharacterDecoder::UTF8) { if (builder_.empty()) { @@ -222,73 +222,6 @@ std::string IfcCharacterDecoder::get(size_t& ptr) { return s; } -void IfcCharacterDecoder::skip() { - unsigned int parse_state = 0; - char current_char; - unsigned int hex_count = 0; - while ((current_char = stream_->peek()) != 0) { - if (EXPECTS_CHARACTER(parse_state)) { - parse_state = 0; - } else if (current_char == '\'' && (parse_state == 0U)) { - parse_state = APOSTROPHE; - } else if (current_char == '\\' && (parse_state == 0U)) { - parse_state = FIRST_SOLIDUS; - } else if (current_char == '\\' && EXPECTS_SOLIDUS(parse_state)) { - if (((parse_state & ALPHABET_DEFINITION) != 0U) || - ((parse_state & IGNORED_DIRECTIVE) != 0U) || - ((parse_state & ENDEXTENDED_0) != 0U)) { - parse_state = hex_count = 0; - } else if ((parse_state & ENCOUNTERED_HEX) != 0U) { - parse_state += THIRD_SOLIDUS; - parse_state -= ENCOUNTERED_HEX; - } else { - parse_state += SECOND_SOLIDUS; - } - } else if (current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state)) { - parse_state += ENDEXTENDED_X; - } else if (current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state)) { - parse_state += ENDEXTENDED_0; - } else if (current_char == 'X' && EXPECTS_ARBITRARY(parse_state)) { - parse_state += ARBITRARY; - } else if (current_char == '2' && EXPECTS_ARBITRARY2(parse_state)) { - parse_state += EXTENDED2; - } else if (current_char == '4' && EXPECTS_ARBITRARY2(parse_state)) { - parse_state += EXTENDED2 + EXTENDED4; - } else if (current_char == 'P' && EXPECTS_ALPHABET(parse_state)) { - parse_state += ALPHABET; - } else if ((current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state)) { - parse_state += IGNORED_DIRECTIVE; - } else if (IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state)) { - parse_state += ALPHABET_DEFINITION; - } else if (current_char == 'S' && EXPECTS_PAGE(parse_state)) { - parse_state += PAGE; - } else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) { - parse_state += HEX((++hex_count)); - if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) || - (hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) || - (hex_count == 8)) { - if (hex_count == 2) { - parse_state = 0; - } else { - CLEAR_HEX(parse_state); - parse_state |= ENCOUNTERED_HEX; - } - hex_count = 0; - } - } else if ((parse_state != 0U) && !( - (current_char == '\\' && parse_state == FIRST_SOLIDUS) || - (current_char == '\'' && parse_state == APOSTROPHE))) { - if (parse_state == APOSTROPHE && current_char != '\'') { - break; - } - throw IfcInvalidTokenException(stream_->tell(), current_char); - } else { - parse_state = hex_count = 0; - } - stream_->increment(); - } -} - IfcCharacterDecoder::ConversionMode IfcCharacterDecoder::mode = IfcCharacterDecoder::UTF8; char IfcCharacterDecoder::substitution_character = '_'; diff --git a/src/ifcparse/IfcCharacterDecoder.h b/src/ifcparse/IfcCharacterDecoder.h index e978a5b4c2..2a3fb0410b 100644 --- a/src/ifcparse/IfcCharacterDecoder.h +++ b/src/ifcparse/IfcCharacterDecoder.h @@ -54,9 +54,6 @@ class IFC_PARSE_API IfcCharacterDecoder { static char substitution_character; IfcCharacterDecoder(IfcParse::FileReader* stream); ~IfcCharacterDecoder(); - // Only advances the underlying token stream read pointer - // to the next token. - void skip(); // Gets a decoded string representation at the token stream // read pointer and advances the underlying token stream. operator std::string(); diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index a7da2d7eea..5bf27b4997 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -57,32 +57,33 @@ namespace { template void dispatch_token(std::optional instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) { - if (t.type == IfcParse::Token_BINARY) { - fn(IfcParse::TokenFunc::asBinary(t)); - } else if (IfcParse::TokenFunc::isBool(t)) { - fn(IfcParse::TokenFunc::asBool(t)); - } else if (IfcParse::TokenFunc::isLogical(t)) { - fn(IfcParse::TokenFunc::asLogical(t)); - } else if (t.type == IfcParse::Token_ENUMERATION) { - auto& s = IfcParse::TokenFunc::asStringRef(t); + if (t.is_binary()) { + fn(t.as_binary()); + } else if (t.is_bool()) { + fn(t.as_bool()); + } else if (t.is_logical()) { + fn(t.as_logical()); + } else if (t.is_enumeration()) { + const auto& s = t.as_string(); if (decl && decl->as_enumeration_type()) { try { fn(EnumerationReference(decl->as_enumeration_type(), decl->as_enumeration_type()->lookup_enum_offset(s))); } catch (IfcParse::IfcException& e) { - Logger::Error("An enumeration literal '" + s + "' is not valid for type '" + decl->name() + "' at offset " + std::to_string(t.startPos)); + Logger::Error("An enumeration literal '" + s + "' is not valid for type '" + decl->name() + "' at offset " + std::to_string(t.start_pos)); } } else { - Logger::Error("An enumeration literal '" + s + "' is not expected at attribute index '" + std::to_string(attribute_id) + "' at offset " + std::to_string(t.startPos)); + Logger::Error("An enumeration literal '" + s + "' is not expected at attribute index '" + std::to_string(attribute_id) + "' at offset " + std::to_string(t.start_pos)); } - } else if (t.type == IfcParse::Token_FLOAT) { - fn(IfcParse::TokenFunc::asFloat(t)); - } else if (t.type == IfcParse::Token_IDENTIFIER) { - fn(IfcParse::reference_or_simple_type{ IfcParse::InstanceReference{ IfcParse::TokenFunc::asIdentifier(t), t.startPos } }); - } else if (t.type == IfcParse::Token_INT) { - fn(IfcParse::TokenFunc::asInt(t)); - } else if (t.type == IfcParse::Token_STRING) { - fn(IfcParse::TokenFunc::asStringRef(t)); - } else if (t.type == IfcParse::Token_OPERATOR && t.value_char == '*') { + } else if (t.is_int()) { + // @nb make sure is_int() comes before is_float() + fn(t.as_int()); + } else if (t.is_float()) { + fn(t.as_float()); + } else if (t.is_identifier()) { + fn(IfcParse::reference_or_simple_type{IfcParse::InstanceReference{(int) t.as_identifier(), t.start_pos}}); + } else if (t.is_string()) { + fn(t.as_string()); + } else if (t.is_operator('*')) { // This is only in place for the validator fn(Derived{}); } @@ -681,22 +682,22 @@ std::optionalstream->eof() && !current_id) { - if (token_stream_[0].type == IfcParse::Token_IDENTIFIER && - token_stream_[1].type == IfcParse::Token_OPERATOR && + if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER && + token_stream_[1].type == IfcParse::Token::Token_OPERATOR && token_stream_[1].value_char == '=' && - token_stream_[2].type == IfcParse::Token_KEYWORD) { - current_id = (unsigned)TokenFunc::asIdentifier(token_stream_[0]); + token_stream_[2].type == IfcParse::Token::Token_KEYWORD) { + current_id = token_stream_[0].as_identifier(); const IfcParse::declaration* entity_type; try { - entity_type = schema_->declaration_by_name(TokenFunc::asStringRef(token_stream_[2])); + entity_type = schema_->declaration_by_name(token_stream_[2].as_string()); } catch (const IfcException& ex) { - Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].startPos)); + Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos)); current_id = 0; goto advance; } if (entity_type->as_entity() == nullptr) { - Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].startPos)); + Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos)); goto advance; } @@ -745,7 +746,7 @@ std::optionalstream->eof() && next_token.type == Token_NONE) { + if (!lexer_->stream->eof() && !next_token) { good_ = file_open_status::INVALID_SYNTAX; break; } @@ -755,6 +756,7 @@ std::optionalstream)->dropPages(); + lexer_->resetPool(); return return_value; } diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index d5d702c824..594127d7e9 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -153,6 +153,45 @@ size_t IfcSpfLexer::skipComment() const { return index; } +std::string& IfcSpfLexer::getTempString() const { + const size_t idx = pool_index++; + const size_t slice = idx >> 4; + const size_t offset = idx & 0xF; + + while (stringpool_.size() <= slice) { + stringpool_.push_back(std::make_unique>()); + } + return (*stringpool_[slice])[offset]; +} + +namespace { + +bool parse_int_(const char* pStart, int& val) { + char* pEnd; + long result = strtol(pStart, &pEnd, 10); + if (*pEnd != 0) { + return false; + } + val = (int)result; + return true; +} + +bool parse_float_(const char* pStart, double& val) { + char* pEnd; +#ifdef _MSC_VER + double result = _strtod_l(pStart, &pEnd, locale); +#else + double result = strtod_l(pStart, &pEnd, locale); +#endif + if (*pEnd != 0) { + return false; + } + val = result; + return true; +} + +} // namespace + // // Returns the offset of the current Token and moves cursor to next // @@ -169,7 +208,6 @@ Token IfcSpfLexer::Next() { return Token{}; } - auto& str = GetTempString(); auto pos = stream->tell(); char character = stream->read(); @@ -182,14 +220,30 @@ Token IfcSpfLexer::Next() { character == '$' || character == '*') { - return OperatorTokenPtr(this, pos, character); + return Token(pos, character); } + auto& str = getTempString(); + if (character == '\'') { // If a string is encountered defer processing to the IfcCharacterDecoder str = *decoder_; + return Token(pos, Token::Token_STRING, str); } else { - str.assign(&character, 1); + auto ttype = Token::Token_NONE; + if (character == '"' || character == '.') { + if (character == '"') { + ttype = Token::Token_BINARY; + } else { + ttype = Token::Token_ENUMERATION; + } + str.clear(); + } else if (character == '#') { + ttype = Token::Token_IDENTIFIER; + str.clear(); + } else { + str.assign(&character, 1); + } while (!stream->eof()) { // Read character and increment pointer if not starting a new token @@ -203,267 +257,157 @@ Token IfcSpfLexer::Next() { break; } if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) { - str.push_back(character); + if ((ttype == Token::Token_BINARY && character == '"') || (ttype == Token::Token_ENUMERATION && character == '.')) { + // Skip + } else { + str.push_back(character); + } } stream->increment(); } - } - return GeneralTokenPtr(this, pos, str); -} -// -// Reads a std::string from the file at specified offset -// Omits whitespace and comments -// -void IfcSpfLexer::TokenString(size_t offset, std::string& buffer) { - buffer.clear(); - auto local_stream = *this->stream; - local_stream.seek(offset); - while (!local_stream.eof()) { - char character = local_stream.peek(); - if (!buffer.empty() && (character == '(' || - character == ')' || - character == '=' || - character == ',' || - character == ';' || - character == '/')) { - break; + if (ttype == Token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) { + popPoolEntry(); + return Token(pos, Token::Token_BOOL, str[0]); + } else if (ttype == Token::Token_IDENTIFIER) { + int int_val; + if (!parse_int_(str.c_str(), int_val)) { + throw IfcInvalidTokenException(pos, str, "instance name"); + } + popPoolEntry(); + return Token(pos, ttype, int_val); + } else if (ttype == Token::Token_NONE && !str.empty()) { + int int_val; + double float_val; + auto& first = str.front(); + if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) { + ttype = Token::Token_KEYWORD; + return Token(pos, ttype, str); + } else if (parse_int_(str.c_str(), int_val)) { + ttype = Token::Token_INT; + popPoolEntry(); + return Token(pos, ttype, int_val); + } else if (parse_float_(str.c_str(), float_val)) { + ttype = Token::Token_FLOAT; + popPoolEntry(); + return Token(pos, float_val); + } + } else if (ttype == Token::Token_BINARY || ttype == Token::Token_ENUMERATION) { + return Token(pos, ttype, str); } - local_stream.increment(); - if (character == ' ' || - character == '\r' || - character == '\n' || - character == '\t') { - continue; - } - if (character == '\'') { - // todo, make decoder use local offset ptr - auto local_offset = local_stream.tell(); - buffer = decoder_->get(local_offset); - break; - } - buffer.push_back(character); + + throw IfcInvalidTokenException(pos, str, "valid token"); } } -//Note: according to STEP standard, there may be newlines in tokens -/* -inline void RemoveTokenSeparators(FileReader* stream, size_t start, size_t end, std::string& oDestination) { - oDestination.clear(); - for (unsigned i = start; i < end; i++) { - char character = stream->get(i); - if (character == ' ' || - character == '\r' || - character == '\n' || - character == '\t') { - continue; - } - oDestination += character; - } -} -*/ - -bool ParseInt(const char* pStart, int& val) { - char* pEnd; - long result = strtol(pStart, &pEnd, 10); - if (*pEnd != 0) { - return false; - } - val = (int)result; - return true; +bool Token::is_operator() { + return type == Token_OPERATOR; } -bool ParseFloat(const char* pStart, double& val) { - char* pEnd; -#ifdef _MSC_VER - double result = _strtod_l(pStart, &pEnd, locale); -#else - double result = strtod_l(pStart, &pEnd, locale); -#endif - if (*pEnd != 0) { - return false; - } - val = result; - return true; +bool Token::is_operator(char character) { + return type == Token_OPERATOR && value_char == character; } -bool ParseBool(const char* pStart, int& val) { - if (strlen(pStart) != 3 || pStart[0] != '.' || pStart[2] != '.') { - return false; - } - char mid = pStart[1]; - - if (mid == 'T') { - val = 1; - } else if (mid == 'F') { - val = 0; - } else if (mid == 'U') { - val = 2; - } else { - return false; - } - - return true; +bool Token::is_identifier() { + return type == Token_IDENTIFIER; } -Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, size_t start, char data) { - Token token(lexer, start, Token_OPERATOR); - token.value_char = data; - return token; +bool Token::is_string() { + return type == Token_STRING; } -Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, size_t start, const std::string& tokenStr) { - Token token(lexer, start, Token_NONE); - - //determine type of the token - const char& first = tokenStr.front(); - if (first == '#') { - token.type = Token_IDENTIFIER; - if (!ParseInt(tokenStr.c_str() + 1, token.value_int)) { - Logger::Message(Logger::LOG_ERROR, "Token '" + tokenStr + "' at offset " + std::to_string(token.startPos) + " is not valid"); - token.type = Token_OPERATOR; - token.value_char = '$'; - } - } else if (first == '\'') { - token.type = Token_STRING; - } else if (first == '.') { - token.type = Token_ENUMERATION; - if (ParseBool(tokenStr.c_str(), token.value_int)) { //bool is also enumeration - token.type = Token_BOOL; - } - } else if (first == '"') { - token.type = Token_BINARY; - } else if (ParseInt(tokenStr.c_str(), token.value_int)) { - token.type = Token_INT; - } else if (ParseFloat(tokenStr.c_str(), token.value_double)) { - token.type = Token_FLOAT; - } else { - token.type = Token_KEYWORD; - } - - return token; +bool Token::is_enumeration() { + // @nb this is a bit confusing? + return type == Token_ENUMERATION || type == Token_BOOL; } -bool TokenFunc::isOperator(const Token& token) { - return token.type == Token_OPERATOR; +bool Token::is_binary() { + return type == Token_BINARY; } -bool TokenFunc::isOperator(const Token& token, char character) { - return token.type == Token_OPERATOR && token.value_char == character; +bool Token::is_keyword() { + return type == Token_KEYWORD; } -bool TokenFunc::isIdentifier(const Token& token) { - return token.type == Token_IDENTIFIER; +bool Token::is_int() { + return type == Token_INT; } -bool TokenFunc::isString(const Token& token) { - return token.type == Token_STRING; +bool Token::is_bool() { + // Bool and logical share the same storage type, just logical unknown is stored as 'U'. + return type == Token_BOOL && value_char != 'U'; } -bool TokenFunc::isEnumeration(const Token& token) { - return token.type == Token_ENUMERATION || token.type == Token_BOOL; +bool Token::is_logical() { + return type == Token_BOOL; } -bool TokenFunc::isBinary(const Token& token) { - return token.type == Token_BINARY; -} - -bool TokenFunc::isKeyword(const Token& token) { - return token.type == Token_KEYWORD; -} - -bool TokenFunc::isInt(const Token& token) { - return token.type == Token_INT; -} - -bool TokenFunc::isBool(const Token& token) { - // Bool and logical share the same storage type, just logical unknown is stored as 2. - return token.type == Token_BOOL && token.value_int != 2; -} - -bool TokenFunc::isLogical(const Token& token) { - return token.type == Token_BOOL; -} - -bool TokenFunc::isFloat(const Token& token) { +bool Token::is_float() { #ifdef PERMISSIVE_FLOAT /// NB: We are being more permissive here then allowed by the standard - return token.type == Token_FLOAT || token.type == Token_INT; + return type == Token_FLOAT || type == Token_INT; #else - return token.type == Token_FLOAT; + return type == Token_FLOAT; #endif } -int TokenFunc::asInt(const Token& token) { - if (token.type != Token_INT) { - throw IfcInvalidTokenException(token.startPos, toString(token), "integer"); +int Token::as_int() { + if (type != Token_INT) { + throw IfcInvalidTokenException(start_pos, to_string(), "integer"); } - return token.value_int; + return value_int; } -int TokenFunc::asIdentifier(const Token& token) { - if (token.type != Token_IDENTIFIER) { - throw IfcInvalidTokenException(token.startPos, toString(token), "instance name"); +unsigned Token::as_identifier() { + if (type != Token_IDENTIFIER) { + throw IfcInvalidTokenException(start_pos, to_string(), "instance name"); } - return token.value_int; + return (unsigned) value_int; } -bool TokenFunc::asBool(const Token& token) { - if (token.type != Token_BOOL) { - throw IfcInvalidTokenException(token.startPos, toString(token), "boolean"); +bool Token::as_bool() { + if (type != Token_BOOL) { + throw IfcInvalidTokenException(start_pos, to_string(), "boolean"); } - return token.value_int == 1; + return value_char == 'T'; } -boost::logic::tribool TokenFunc::asLogical(const Token& token) { - if (token.type != Token_BOOL) { - throw IfcInvalidTokenException(token.startPos, toString(token), "boolean"); +boost::logic::tribool Token::as_logical() { + if (type != Token_BOOL) { + throw IfcInvalidTokenException(start_pos, to_string(), "logical"); } - if (token.value_int == 0) { + if (value_int == 'F') { return false; } - if (token.value_int == 1) { + if (value_int == 'T') { return true; } return boost::logic::indeterminate; } -double TokenFunc::asFloat(const Token& token) { +double Token::as_float() { #ifdef PERMISSIVE_FLOAT - if (token.type == Token_INT) { + if (type == Token_INT) { /// NB: We are being more permissive here then allowed by the standard - return token.value_int; + return value_int; } // ----> continues beyond preprocessor directive #endif - if (token.type == Token_FLOAT) { - return token.value_double; + if (type == Token_FLOAT) { + return value_double; } - throw IfcInvalidTokenException(token.startPos, toString(token), "real"); + throw IfcInvalidTokenException(start_pos, to_string(), "real"); } -const std::string& TokenFunc::asStringRef(const Token& token) { - if (token.type == Token_NONE) { - throw IfcParse::IfcException("Null token encountered, premature end of file?"); +const std::string& Token::as_string() { + if (is_string() || is_enumeration() || is_binary() || is_keyword()) { + // @todo quotes + return *value_string; } - std::string& str = token.lexer->GetTempString(); - token.lexer->TokenString(token.startPos, str); - if ((isString(token) || isEnumeration(token) || isBinary(token)) && !str.empty()) { - //remove start+end characters in-place - str.erase(str.end() - 1); - str.erase(str.begin()); - } - return str; + throw IfcInvalidTokenException(start_pos, to_string(), "string"); } -std::string TokenFunc::asString(const Token& token) { - if (isString(token) || isEnumeration(token) || isBinary(token)) { - return asStringRef(token); - } - throw IfcInvalidTokenException(token.startPos, toString(token), "string"); -} - -boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) { - const std::string& str = asStringRef(token); +boost::dynamic_bitset<> Token::as_binary() { + const std::string& str = as_string(); if (str.empty()) { throw IfcException("Token is not a valid binary sequence"); } @@ -475,7 +419,7 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) { } ++it; - unsigned i = ((unsigned)str.size() - 1) * 4 - n; + unsigned i = (str.size() - 1) * 4 - n; boost::dynamic_bitset<> bitset(i); for (; it != str.end(); ++it) { @@ -494,26 +438,18 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) { return bitset; } -std::string TokenFunc::toString(const Token& token) { +std::string Token::to_string() { std::string result; - if (token.type == Token_OPERATOR) { - result.push_back(token.value_char); - } else if (token.type == Token_INT) { - result = std::to_string(token.value_int); - } else if (token.type == Token_BOOL) { - if (token.value_int == 1) { - result = ".T."; - } else if (token.value_int == 0) { - result = ".F."; - } else { - result = ".U."; - } - } else if (token.type == Token_FLOAT) { + if (type == Token_OPERATOR || type == Token_BOOL) { + result.push_back(value_char); + } else if (type == Token_INT) { + result = std::to_string(value_int); + } else if (type == Token_FLOAT) { std::ostringstream oss; - oss << std::setprecision(15) << token.value_double; + oss << std::setprecision(15) << value_double; result = oss.str(); } else { - token.lexer->TokenString(token.startPos, result); + return as_string(); } return result; } @@ -525,34 +461,28 @@ std::string TokenFunc::toString(const Token& token) { void IfcParse::impl::in_memory_file_storage::load(std::optional entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { Token next = tokens->Next(); - /* - if (TokenFunc::isOperator(next, '(')) { - next = tokens->Next(); - } - */ - size_t attribute_index_within_data = 0; size_t return_value = 0; - while ((next.startPos != 0U) || (next.lexer != nullptr)) { - if (TokenFunc::isOperator(next, ',')) { + while (next) { + if (next.is_operator(',')) { if (attribute_index == -1) { attribute_index_within_data += 1; } - } else if (TokenFunc::isOperator(next, ')')) { + } else if (next.is_operator(')')) { break; - } else if (TokenFunc::isOperator(next, '(')) { + } else if (next.is_operator('(')) { return_value++; load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); } else { return_value++; - if (TokenFunc::isIdentifier(next) && entity && entity_instance_name) { + if (next.is_identifier() && entity && entity_instance_name) { register_inverse(*entity_instance_name, entity, next.value_int, attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); } - if (TokenFunc::isKeyword(next)) { + if (next.is_keyword()) { try { - const auto* decl = (schema ? schema : file->schema())->declaration_by_name(TokenFunc::asStringRef(next)); + const auto* decl = (schema ? schema : file->schema())->declaration_by_name(next.as_string()); parse_context ps; tokens->Next(); // The only case we know where a defined type contains entity @@ -568,7 +498,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional entity_i // @todo do we need express::Base here? Or should we just push InstanceData? context.push(simple_type_instance); } catch (IfcException& e) { - Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(next.startPos)); + Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(next.start_pos)); // #4070 We didn't actually capture an aggregate entry, undo length increment. return_value--; } @@ -580,25 +510,10 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional entity_i } } -// -// Reads an Entity from the list of Tokens at the specified offset in the file -// -std::shared_ptr IfcParse::impl::in_memory_file_storage::read(unsigned int i) { - Token datatype = tokens->Next(); - if (!TokenFunc::isKeyword(datatype)) { - throw IfcException("Unexpected token while parsing entity"); - } - const IfcParse::declaration* ty = file->schema()->declaration_by_name(TokenFunc::asStringRef(datatype)); - parse_context pc; - tokens->Next(); - load(i, ty->as_entity(), pc, -1); - return pc.construct(file, i, *references_to_resolve, ty, std::nullopt, -1); -} - void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const { auto old_offset = tokens->stream->tell(); Token semilocon = tokens->Next(); - if (!TokenFunc::isOperator(semilocon, ';')) { + if (!semilocon.is_operator(';')) { tokens->stream->seek(old_offset); } } @@ -1354,8 +1269,8 @@ bool IfcParse::InstanceStreamer::hasSemicolon() const { } catch (const std::out_of_range&) { return false; } - while (t.type != Token_NONE) { - if (TokenFunc::isOperator(t, ';')) { + while (t.type != Token::Token_NONE) { + if (t.is_operator(';')) { return true; } try { @@ -1378,8 +1293,8 @@ size_t IfcParse::InstanceStreamer::semicolonCount() const { } catch (const std::out_of_range&) { return false; } - while (t.type != Token_NONE) { - if (TokenFunc::isOperator(t, ';')) { + while (t.type != Token::Token_NONE) { + if (t.is_operator(';')) { count++; } try { diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index a0f0569dd4..1fed5de76c 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -47,64 +47,6 @@ extern const char *IFCOPENSHELL_VERSION; namespace IfcParse { -/// Provides functions to convert Tokens to binary data -/// Tokens are merely offsets to where they can be read in the file -class IFC_PARSE_API TokenFunc { - private: - static bool startsWith(const Token& token, char character); - - public: - /// Returns the offset at which the token is read from the file - // static unsigned int Offset(const Token& t); - /// Returns whether the token can be interpreted as a string - static bool isString(const Token& token); - /// Returns whether the token can be interpreted as an identifier - static bool isIdentifier(const Token& token); - /// Returns whether the token can be interpreted as a syntactical operator - static bool isOperator(const Token& token); - /// Returns whether the token is a given operator - static bool isOperator(const Token& token, char character); - /// Returns whether the token can be interpreted as an enumerated value - static bool isEnumeration(const Token& token); - /// Returns whether the token can be interpreted as a datatype name - static bool isKeyword(const Token& token); - /// Returns whether the token can be interpreted as an integer - static bool isInt(const Token& token); - /// Returns whether the token can be interpreted as a boolean - static bool isBool(const Token& token); - /// Returns whether the token can be interpreted as a logical - static bool isLogical(const Token& token); - /// Returns whether the token can be interpreted as a floating point number - static bool isFloat(const Token& token); - /// Returns whether the token can be interpreted as a binary type - static bool isBinary(const Token& token); - /// Returns the token interpreted as an integer - static int asInt(const Token& token); - /// Returns the token interpreted as an identifier - static int asIdentifier(const Token& token); - /// Returns the token interpreted as an boolean (.T. or .F.) - static bool asBool(const Token& token); - /// Returns the token interpreted as an logical (.T. or .F. or .U.) - static boost::logic::tribool asLogical(const Token& token); - /// Returns the token as a floating point number - static double asFloat(const Token& token); - /// Returns the token as a string (without the dot or apostrophe) - static std::string asString(const Token& token); - /// Returns the token as a string in internal buffer (for optimization purposes) - static const std::string& asStringRef(const Token& token); - /// Returns the token as a string (without the dot or apostrophe) - static boost::dynamic_bitset<> asBinary(const Token& token); - /// Returns a string representation of the token (including the dot or apostrophe) - static std::string toString(const Token& token); -}; - -// -// Functions for creating Tokens from an arbitary file offset -// The first 4 bits are reserved for Tokens of type ()=,;$* -// -Token OperatorTokenPtr(IfcSpfLexer* tokens, size_t start, char data); -Token GeneralTokenPtr(IfcSpfLexer* tokens, size_t start, const std::string& data); - /// A stream of tokens to be read from a FileReader. class IFC_PARSE_API IfcSpfLexer { private: @@ -112,17 +54,26 @@ class IFC_PARSE_API IfcSpfLexer { size_t skipWhitespace() const; size_t skipComment() const; + mutable std::vector>> stringpool_; + mutable size_t pool_index = 0; + public: - std::string& GetTempString() const { - static thread_local std::string string; - return string; + std::string& getTempString() const; + void resetPool() const { + pool_index = 0; } + void popPoolEntry() { + if (pool_index > 0) { + --pool_index; + } + } + FileReader* stream; // IfcFile* file; IfcSpfLexer(FileReader* stream); Token Next(); ~IfcSpfLexer(); - void TokenString(size_t offset, std::string& result); + // void TokenString(size_t offset, std::string& result); }; IFC_PARSE_API std::vector traverse(const express::Base& instance, int max_level = -1); diff --git a/src/ifcparse/IfcSpfHeader.cpp b/src/ifcparse/IfcSpfHeader.cpp index c394e48689..cc04ffcabb 100644 --- a/src/ifcparse/IfcSpfHeader.cpp +++ b/src/ifcparse/IfcSpfHeader.cpp @@ -45,7 +45,7 @@ namespace { void IfcSpfHeader::readSemicolon() { if (storage_ != nullptr) { - if (!TokenFunc::isOperator(storage_->tokens->Next(), ';')) { + if (!storage_->tokens->Next().is_operator(';')) { throw IfcException(std::string("Expected ;")); } } else { @@ -55,7 +55,7 @@ void IfcSpfHeader::readSemicolon() { void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) { if (storage_ != nullptr) { - if (TokenFunc::asStringRef(storage_->tokens->Next()) != term) { + if (storage_->tokens->Next().as_string() != term) { throw IfcException(std::string("Expected " + term)); } if (trail == TRAILING_SEMICOLON) { diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h index 382106adcf..0a84a04004 100644 --- a/src/ifcparse/storage.h +++ b/src/ifcparse/storage.h @@ -134,37 +134,71 @@ namespace IfcParse { class IfcSpfLexer; class FileReader; - enum TokenType { - Token_NONE, - Token_STRING, - Token_IDENTIFIER, - Token_OPERATOR, - Token_ENUMERATION, - Token_KEYWORD, - Token_INT, - Token_BOOL, - Token_FLOAT, - Token_BINARY - }; - struct Token { - IfcSpfLexer* lexer; //TODO: remove it from here - size_t startPos; + enum TokenType { + Token_NONE, + Token_STRING, + Token_IDENTIFIER, + Token_OPERATOR, + Token_ENUMERATION, + Token_KEYWORD, + Token_INT, + Token_BOOL, + Token_FLOAT, + Token_BINARY + }; + + size_t start_pos; TokenType type; + union { char value_char; //types: OPERATOR int value_int; //types: INT, IDENTIFIER double value_double; //types: FLOAT + const std::string* value_string; //types: STR, ENUM, KEYWORD; lifetime managed by IfcSpfLexer::string_pool_ }; - Token() : lexer(0), - startPos(0), - type(Token_NONE) { - } - Token(IfcSpfLexer* _lexer, size_t _startPos, TokenType _type) - : lexer(_lexer), - startPos(_startPos), - type(_type) { + Token() : start_pos(0), + type(Token_NONE) {} + + Token(size_t start, TokenType ty, const std::string& str) + : start_pos(start), type(ty), value_string(&str) {} + + Token(size_t start, TokenType ty, int i) + : start_pos(start), type(ty), value_int(i) {} + + Token(size_t start, double d) + : start_pos(start), type(Token_FLOAT), value_double(d) {} + + Token(size_t start, char op) + : start_pos(start), type(Token_OPERATOR), value_char(op) {} + + Token(size_t start, TokenType ty, char c) + : start_pos(start), type(ty), value_char(c) {} + + bool is_string(); + bool is_identifier(); + bool is_operator(); + bool is_operator(char character); + bool is_enumeration(); + bool is_keyword(); + bool is_int(); + bool is_bool(); + bool is_logical(); + bool is_float(); + bool is_binary(); + + int as_int(); + unsigned as_identifier(); + bool as_bool(); + boost::logic::tribool as_logical(); + double as_float(); + const std::string& as_string(); + boost::dynamic_bitset<> as_binary(); + std::string to_string(); + + operator bool() const { + return type != Token_NONE; } }; @@ -275,8 +309,6 @@ namespace IfcParse { void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); void unregister_inverse(unsigned, const IfcParse::entity* from_entity, const express::Base&, int attribute_index); - // @todo is this still used - std::shared_ptr read(unsigned int index); void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); file_open_status good_ = file_open_status::SUCCESS;