Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-09 09:21:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework variable length token storage to use string pool; eliminate need for rereads

Browse Source
This commit is contained in:
Thomas Krijnen
2026-03-26 15:49:28 +01:00
parent 9d69a712ac
commit 8e42f35db3
7 changed files with 272 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/IfcCharacterDecoder.cpp
+2 -69
View File
@@ -91,7 +91,7 @@ namespace {
unsigned int parse_state = 0;
builder_.clear();
builder_.push_back('\'');
// builder_.push_back('\'');
char current_char;
int codepage = 1;
unsigned int hex = 0;
@@ -169,7 +169,7 @@ namespace {
}
stream_.increment();
}
builder_.push_back('\'');
// builder_.push_back('\'');
if (mode == IfcParse::IfcCharacterDecoder::UTF8) {
if (builder_.empty()) {
@@ -222,73 +222,6 @@ std::string IfcCharacterDecoder::get(size_t& ptr) {
return s;
}
void IfcCharacterDecoder::skip() {
unsigned int parse_state = 0;
char current_char;
unsigned int hex_count = 0;
while ((current_char = stream_->peek()) != 0) {
if (EXPECTS_CHARACTER(parse_state)) {
parse_state = 0;
} else if (current_char == '\'' && (parse_state == 0U)) {
parse_state = APOSTROPHE;
} else if (current_char == '\\' && (parse_state == 0U)) {
parse_state = FIRST_SOLIDUS;
} else if (current_char == '\\' && EXPECTS_SOLIDUS(parse_state)) {
if (((parse_state & ALPHABET_DEFINITION) != 0U) ||
((parse_state & IGNORED_DIRECTIVE) != 0U) ||
((parse_state & ENDEXTENDED_0) != 0U)) {
parse_state = hex_count = 0;
} else if ((parse_state & ENCOUNTERED_HEX) != 0U) {
parse_state += THIRD_SOLIDUS;
parse_state -= ENCOUNTERED_HEX;
} else {
parse_state += SECOND_SOLIDUS;
}
} else if (current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state)) {
parse_state += ENDEXTENDED_X;
} else if (current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state)) {
parse_state += ENDEXTENDED_0;
} else if (current_char == 'X' && EXPECTS_ARBITRARY(parse_state)) {
parse_state += ARBITRARY;
} else if (current_char == '2' && EXPECTS_ARBITRARY2(parse_state)) {
parse_state += EXTENDED2;
} else if (current_char == '4' && EXPECTS_ARBITRARY2(parse_state)) {
parse_state += EXTENDED2 + EXTENDED4;
} else if (current_char == 'P' && EXPECTS_ALPHABET(parse_state)) {
parse_state += ALPHABET;
} else if ((current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state)) {
parse_state += IGNORED_DIRECTIVE;
} else if (IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state)) {
parse_state += ALPHABET_DEFINITION;
} else if (current_char == 'S' && EXPECTS_PAGE(parse_state)) {
parse_state += PAGE;
} else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) {
parse_state += HEX((++hex_count));
if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) ||
(hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) ||
(hex_count == 8)) {
if (hex_count == 2) {
parse_state = 0;
} else {
CLEAR_HEX(parse_state);
parse_state |= ENCOUNTERED_HEX;
}
hex_count = 0;
}
} else if ((parse_state != 0U) && !(
(current_char == '\\' && parse_state == FIRST_SOLIDUS) ||
(current_char == '\'' && parse_state == APOSTROPHE))) {
if (parse_state == APOSTROPHE && current_char != '\'') {
break;
}
throw IfcInvalidTokenException(stream_->tell(), current_char);
} else {
parse_state = hex_count = 0;
}
stream_->increment();
}
}
IfcCharacterDecoder::ConversionMode IfcCharacterDecoder::mode = IfcCharacterDecoder::UTF8;
char IfcCharacterDecoder::substitution_character = '_';
src/ifcparse/IfcCharacterDecoder.h
-3
View File
@@ -54,9 +54,6 @@ class IFC_PARSE_API IfcCharacterDecoder {
static char substitution_character;
IfcCharacterDecoder(IfcParse::FileReader* stream);
~IfcCharacterDecoder();
// Only advances the underlying token stream read pointer
// to the next token.
void skip();
// Gets a decoded string representation at the token stream
// read pointer and advances the underlying token stream.
operator std::string();
src/ifcparse/IfcFile.cpp
+29 -27
View File
@@ -57,32 +57,33 @@ namespace {
template <typename Fn>
void dispatch_token(std::optional<size_t> instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) {
if (t.type == IfcParse::Token_BINARY) {
fn(IfcParse::TokenFunc::asBinary(t));
} else if (IfcParse::TokenFunc::isBool(t)) {
fn(IfcParse::TokenFunc::asBool(t));
} else if (IfcParse::TokenFunc::isLogical(t)) {
fn(IfcParse::TokenFunc::asLogical(t));
} else if (t.type == IfcParse::Token_ENUMERATION) {
auto& s = IfcParse::TokenFunc::asStringRef(t);
if (t.is_binary()) {
fn(t.as_binary());
} else if (t.is_bool()) {
fn(t.as_bool());
} else if (t.is_logical()) {
fn(t.as_logical());
} else if (t.is_enumeration()) {
const auto& s = t.as_string();
if (decl && decl->as_enumeration_type()) {
try {
fn(EnumerationReference(decl->as_enumeration_type(), decl->as_enumeration_type()->lookup_enum_offset(s)));
} catch (IfcParse::IfcException& e) {
Logger::Error("An enumeration literal '" + s + "' is not valid for type '" + decl->name() + "' at offset " + std::to_string(t.startPos));
Logger::Error("An enumeration literal '" + s + "' is not valid for type '" + decl->name() + "' at offset " + std::to_string(t.start_pos));
}
} else {
Logger::Error("An enumeration literal '" + s + "' is not expected at attribute index '" + std::to_string(attribute_id) + "' at offset " + std::to_string(t.startPos));
Logger::Error("An enumeration literal '" + s + "' is not expected at attribute index '" + std::to_string(attribute_id) + "' at offset " + std::to_string(t.start_pos));
}
} else if (t.type == IfcParse::Token_FLOAT) {
fn(IfcParse::TokenFunc::asFloat(t));
} else if (t.type == IfcParse::Token_IDENTIFIER) {
fn(IfcParse::reference_or_simple_type{ IfcParse::InstanceReference{ IfcParse::TokenFunc::asIdentifier(t), t.startPos } });
} else if (t.type == IfcParse::Token_INT) {
fn(IfcParse::TokenFunc::asInt(t));
} else if (t.type == IfcParse::Token_STRING) {
fn(IfcParse::TokenFunc::asStringRef(t));
} else if (t.type == IfcParse::Token_OPERATOR && t.value_char == '*') {
} else if (t.is_int()) {
// @nb make sure is_int() comes before is_float()
fn(t.as_int());
} else if (t.is_float()) {
fn(t.as_float());
} else if (t.is_identifier()) {
fn(IfcParse::reference_or_simple_type{IfcParse::InstanceReference{(int) t.as_identifier(), t.start_pos}});
} else if (t.is_string()) {
fn(t.as_string());
} else if (t.is_operator('*')) {
// This is only in place for the validator
fn(Derived{});
}
@@ -681,22 +682,22 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<I
unsigned current_id = 0;
while (good_ && !lexer_->stream->eof() && !current_id) {
if (token_stream_[0].type == IfcParse::Token_IDENTIFIER &&
token_stream_[1].type == IfcParse::Token_OPERATOR &&
if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER &&
token_stream_[1].type == IfcParse::Token::Token_OPERATOR &&
token_stream_[1].value_char == '=' &&
token_stream_[2].type == IfcParse::Token_KEYWORD) {
current_id = (unsigned)TokenFunc::asIdentifier(token_stream_[0]);
token_stream_[2].type == IfcParse::Token::Token_KEYWORD) {
current_id = token_stream_[0].as_identifier();
const IfcParse::declaration* entity_type;
try {
entity_type = schema_->declaration_by_name(TokenFunc::asStringRef(token_stream_[2]));
entity_type = schema_->declaration_by_name(token_stream_[2].as_string());
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].startPos));
Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos));
current_id = 0;
goto advance;
}
if (entity_type->as_entity() == nullptr) {
Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].startPos));
Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos));
goto advance;
}
@@ -745,7 +746,7 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<I
Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
}
if (!lexer_->stream->eof() && next_token.type == Token_NONE) {
if (!lexer_->stream->eof() && !next_token) {
good_ = file_open_status::INVALID_SYNTAX;
break;
}
@@ -755,6 +756,7 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<I
// Free pages in front of cursor when variable-width tokens are materialized into entity instance data objects
(stream_ ? stream_ : (lexer_)->stream)->dropPages();
lexer_->resetPool();
return return_value;
}
src/ifcparse/IfcParse.cpp
+169 -254
View File
@@ -153,6 +153,45 @@ size_t IfcSpfLexer::skipComment() const {
return index;
}
std::string& IfcSpfLexer::getTempString() const {
const size_t idx = pool_index++;
const size_t slice = idx >> 4;
const size_t offset = idx & 0xF;
while (stringpool_.size() <= slice) {
stringpool_.push_back(std::make_unique<std::array<std::string, 16>>());
}
return (*stringpool_[slice])[offset];
}
namespace {
bool parse_int_(const char* pStart, int& val) {
char* pEnd;
long result = strtol(pStart, &pEnd, 10);
if (*pEnd != 0) {
return false;
}
val = (int)result;
return true;
}
bool parse_float_(const char* pStart, double& val) {
char* pEnd;
#ifdef _MSC_VER
double result = _strtod_l(pStart, &pEnd, locale);
#else
double result = strtod_l(pStart, &pEnd, locale);
#endif
if (*pEnd != 0) {
return false;
}
val = result;
return true;
}
} // namespace
//
// Returns the offset of the current Token and moves cursor to next
//
@@ -169,7 +208,6 @@ Token IfcSpfLexer::Next() {
return Token{};
}
auto& str = GetTempString();
auto pos = stream->tell();
char character = stream->read();
@@ -182,14 +220,30 @@ Token IfcSpfLexer::Next() {
character == '$' ||
character == '*')
{
return OperatorTokenPtr(this, pos, character);
return Token(pos, character);
}
auto& str = getTempString();
if (character == '\'') {
// If a string is encountered defer processing to the IfcCharacterDecoder
str = *decoder_;
return Token(pos, Token::Token_STRING, str);
} else {
str.assign(&character, 1);
auto ttype = Token::Token_NONE;
if (character == '"' || character == '.') {
if (character == '"') {
ttype = Token::Token_BINARY;
} else {
ttype = Token::Token_ENUMERATION;
}
str.clear();
} else if (character == '#') {
ttype = Token::Token_IDENTIFIER;
str.clear();
} else {
str.assign(&character, 1);
}
while (!stream->eof()) {
// Read character and increment pointer if not starting a new token
@@ -203,267 +257,157 @@ Token IfcSpfLexer::Next() {
break;
}
if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
str.push_back(character);
if ((ttype == Token::Token_BINARY && character == '"') || (ttype == Token::Token_ENUMERATION && character == '.')) {
// Skip
} else {
str.push_back(character);
}
}
stream->increment();
}
}
return GeneralTokenPtr(this, pos, str);
}
//
// Reads a std::string from the file at specified offset
// Omits whitespace and comments
//
void IfcSpfLexer::TokenString(size_t offset, std::string& buffer) {
buffer.clear();
auto local_stream = *this->stream;
local_stream.seek(offset);
while (!local_stream.eof()) {
char character = local_stream.peek();
if (!buffer.empty() && (character == '(' ||
character == ')' ||
character == '=' ||
character == ',' ||
character == ';' ||
character == '/')) {
break;
if (ttype == Token::Token_ENUMERATION && str.size() == 1 && (str[0] == 'T' || str[0] == 'F' || str[0] == 'U')) {
popPoolEntry();
return Token(pos, Token::Token_BOOL, str[0]);
} else if (ttype == Token::Token_IDENTIFIER) {
int int_val;
if (!parse_int_(str.c_str(), int_val)) {
throw IfcInvalidTokenException(pos, str, "instance name");
}
popPoolEntry();
return Token(pos, ttype, int_val);
} else if (ttype == Token::Token_NONE && !str.empty()) {
int int_val;
double float_val;
auto& first = str.front();
if ((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z')) {
ttype = Token::Token_KEYWORD;
return Token(pos, ttype, str);
} else if (parse_int_(str.c_str(), int_val)) {
ttype = Token::Token_INT;
popPoolEntry();
return Token(pos, ttype, int_val);
} else if (parse_float_(str.c_str(), float_val)) {
ttype = Token::Token_FLOAT;
popPoolEntry();
return Token(pos, float_val);
}
} else if (ttype == Token::Token_BINARY || ttype == Token::Token_ENUMERATION) {
return Token(pos, ttype, str);
}
local_stream.increment();
if (character == ' ' ||
character == '\r' ||
character == '\n' ||
character == '\t') {
continue;
}
if (character == '\'') {
// todo, make decoder use local offset ptr
auto local_offset = local_stream.tell();
buffer = decoder_->get(local_offset);
break;
}
buffer.push_back(character);
throw IfcInvalidTokenException(pos, str, "valid token");
}
}
//Note: according to STEP standard, there may be newlines in tokens
/*
inline void RemoveTokenSeparators(FileReader* stream, size_t start, size_t end, std::string& oDestination) {
oDestination.clear();
for (unsigned i = start; i < end; i++) {
char character = stream->get(i);
if (character == ' ' ||
character == '\r' ||
character == '\n' ||
character == '\t') {
continue;
}
oDestination += character;
}
}
*/
bool ParseInt(const char* pStart, int& val) {
char* pEnd;
long result = strtol(pStart, &pEnd, 10);
if (*pEnd != 0) {
return false;
}
val = (int)result;
return true;
bool Token::is_operator() {
return type == Token_OPERATOR;
}
bool ParseFloat(const char* pStart, double& val) {
char* pEnd;
#ifdef _MSC_VER
double result = _strtod_l(pStart, &pEnd, locale);
#else
double result = strtod_l(pStart, &pEnd, locale);
#endif
if (*pEnd != 0) {
return false;
}
val = result;
return true;
bool Token::is_operator(char character) {
return type == Token_OPERATOR && value_char == character;
}
bool ParseBool(const char* pStart, int& val) {
if (strlen(pStart) != 3 || pStart[0] != '.' || pStart[2] != '.') {
return false;
}
char mid = pStart[1];
if (mid == 'T') {
val = 1;
} else if (mid == 'F') {
val = 0;
} else if (mid == 'U') {
val = 2;
} else {
return false;
}
return true;
bool Token::is_identifier() {
return type == Token_IDENTIFIER;
}
Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, size_t start, char data) {
Token token(lexer, start, Token_OPERATOR);
token.value_char = data;
return token;
bool Token::is_string() {
return type == Token_STRING;
}
Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, size_t start, const std::string& tokenStr) {
Token token(lexer, start, Token_NONE);
//determine type of the token
const char& first = tokenStr.front();
if (first == '#') {
token.type = Token_IDENTIFIER;
if (!ParseInt(tokenStr.c_str() + 1, token.value_int)) {
Logger::Message(Logger::LOG_ERROR, "Token '" + tokenStr + "' at offset " + std::to_string(token.startPos) + " is not valid");
token.type = Token_OPERATOR;
token.value_char = '$';
}
} else if (first == '\'') {
token.type = Token_STRING;
} else if (first == '.') {
token.type = Token_ENUMERATION;
if (ParseBool(tokenStr.c_str(), token.value_int)) { //bool is also enumeration
token.type = Token_BOOL;
}
} else if (first == '"') {
token.type = Token_BINARY;
} else if (ParseInt(tokenStr.c_str(), token.value_int)) {
token.type = Token_INT;
} else if (ParseFloat(tokenStr.c_str(), token.value_double)) {
token.type = Token_FLOAT;
} else {
token.type = Token_KEYWORD;
}
return token;
bool Token::is_enumeration() {
// @nb this is a bit confusing?
return type == Token_ENUMERATION || type == Token_BOOL;
}
bool TokenFunc::isOperator(const Token& token) {
return token.type == Token_OPERATOR;
bool Token::is_binary() {
return type == Token_BINARY;
}
bool TokenFunc::isOperator(const Token& token, char character) {
return token.type == Token_OPERATOR && token.value_char == character;
bool Token::is_keyword() {
return type == Token_KEYWORD;
}
bool TokenFunc::isIdentifier(const Token& token) {
return token.type == Token_IDENTIFIER;
bool Token::is_int() {
return type == Token_INT;
}
bool TokenFunc::isString(const Token& token) {
return token.type == Token_STRING;
bool Token::is_bool() {
// Bool and logical share the same storage type, just logical unknown is stored as 'U'.
return type == Token_BOOL && value_char != 'U';
}
bool TokenFunc::isEnumeration(const Token& token) {
return token.type == Token_ENUMERATION || token.type == Token_BOOL;
bool Token::is_logical() {
return type == Token_BOOL;
}
bool TokenFunc::isBinary(const Token& token) {
return token.type == Token_BINARY;
}
bool TokenFunc::isKeyword(const Token& token) {
return token.type == Token_KEYWORD;
}
bool TokenFunc::isInt(const Token& token) {
return token.type == Token_INT;
}
bool TokenFunc::isBool(const Token& token) {
// Bool and logical share the same storage type, just logical unknown is stored as 2.
return token.type == Token_BOOL && token.value_int != 2;
}
bool TokenFunc::isLogical(const Token& token) {
return token.type == Token_BOOL;
}
bool TokenFunc::isFloat(const Token& token) {
bool Token::is_float() {
#ifdef PERMISSIVE_FLOAT
/// NB: We are being more permissive here then allowed by the standard
return token.type == Token_FLOAT || token.type == Token_INT;
return type == Token_FLOAT || type == Token_INT;
#else
return token.type == Token_FLOAT;
return type == Token_FLOAT;
#endif
}
int TokenFunc::asInt(const Token& token) {
if (token.type != Token_INT) {
throw IfcInvalidTokenException(token.startPos, toString(token), "integer");
int Token::as_int() {
if (type != Token_INT) {
throw IfcInvalidTokenException(start_pos, to_string(), "integer");
}
return token.value_int;
return value_int;
}
int TokenFunc::asIdentifier(const Token& token) {
if (token.type != Token_IDENTIFIER) {
throw IfcInvalidTokenException(token.startPos, toString(token), "instance name");
unsigned Token::as_identifier() {
if (type != Token_IDENTIFIER) {
throw IfcInvalidTokenException(start_pos, to_string(), "instance name");
}
return token.value_int;
return (unsigned) value_int;
}
bool TokenFunc::asBool(const Token& token) {
if (token.type != Token_BOOL) {
throw IfcInvalidTokenException(token.startPos, toString(token), "boolean");
bool Token::as_bool() {
if (type != Token_BOOL) {
throw IfcInvalidTokenException(start_pos, to_string(), "boolean");
}
return token.value_int == 1;
return value_char == 'T';
}
boost::logic::tribool TokenFunc::asLogical(const Token& token) {
if (token.type != Token_BOOL) {
throw IfcInvalidTokenException(token.startPos, toString(token), "boolean");
boost::logic::tribool Token::as_logical() {
if (type != Token_BOOL) {
throw IfcInvalidTokenException(start_pos, to_string(), "logical");
}
if (token.value_int == 0) {
if (value_int == 'F') {
return false;
}
if (token.value_int == 1) {
if (value_int == 'T') {
return true;
}
return boost::logic::indeterminate;
}
double TokenFunc::asFloat(const Token& token) {
double Token::as_float() {
#ifdef PERMISSIVE_FLOAT
if (token.type == Token_INT) {
if (type == Token_INT) {
/// NB: We are being more permissive here then allowed by the standard
return token.value_int;
return value_int;
} // ----> continues beyond preprocessor directive
#endif
if (token.type == Token_FLOAT) {
return token.value_double;
if (type == Token_FLOAT) {
return value_double;
}
throw IfcInvalidTokenException(token.startPos, toString(token), "real");
throw IfcInvalidTokenException(start_pos, to_string(), "real");
}
const std::string& TokenFunc::asStringRef(const Token& token) {
if (token.type == Token_NONE) {
throw IfcParse::IfcException("Null token encountered, premature end of file?");
const std::string& Token::as_string() {
if (is_string() || is_enumeration() || is_binary() || is_keyword()) {
// @todo quotes
return *value_string;
}
std::string& str = token.lexer->GetTempString();
token.lexer->TokenString(token.startPos, str);
if ((isString(token) || isEnumeration(token) || isBinary(token)) && !str.empty()) {
//remove start+end characters in-place
str.erase(str.end() - 1);
str.erase(str.begin());
}
return str;
throw IfcInvalidTokenException(start_pos, to_string(), "string");
}
std::string TokenFunc::asString(const Token& token) {
if (isString(token) || isEnumeration(token) || isBinary(token)) {
return asStringRef(token);
}
throw IfcInvalidTokenException(token.startPos, toString(token), "string");
}
boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) {
const std::string& str = asStringRef(token);
boost::dynamic_bitset<> Token::as_binary() {
const std::string& str = as_string();
if (str.empty()) {
throw IfcException("Token is not a valid binary sequence");
}
@@ -475,7 +419,7 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) {
}
++it;
unsigned i = ((unsigned)str.size() - 1) * 4 - n;
unsigned i = (str.size() - 1) * 4 - n;
boost::dynamic_bitset<> bitset(i);
for (; it != str.end(); ++it) {
@@ -494,26 +438,18 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) {
return bitset;
}
std::string TokenFunc::toString(const Token& token) {
std::string Token::to_string() {
std::string result;
if (token.type == Token_OPERATOR) {
result.push_back(token.value_char);
} else if (token.type == Token_INT) {
result = std::to_string(token.value_int);
} else if (token.type == Token_BOOL) {
if (token.value_int == 1) {
result = ".T.";
} else if (token.value_int == 0) {
result = ".F.";
} else {
result = ".U.";
}
} else if (token.type == Token_FLOAT) {
if (type == Token_OPERATOR || type == Token_BOOL) {
result.push_back(value_char);
} else if (type == Token_INT) {
result = std::to_string(value_int);
} else if (type == Token_FLOAT) {
std::ostringstream oss;
oss << std::setprecision(15) << token.value_double;
oss << std::setprecision(15) << value_double;
result = oss.str();
} else {
token.lexer->TokenString(token.startPos, result);
return as_string();
}
return result;
}
@@ -525,34 +461,28 @@ std::string TokenFunc::toString(const Token& token) {
void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
Token next = tokens->Next();
/*
if (TokenFunc::isOperator(next, '(')) {
next = tokens->Next();
}
*/
size_t attribute_index_within_data = 0;
size_t return_value = 0;
while ((next.startPos != 0U) || (next.lexer != nullptr)) {
if (TokenFunc::isOperator(next, ',')) {
while (next) {
if (next.is_operator(',')) {
if (attribute_index == -1) {
attribute_index_within_data += 1;
}
} else if (TokenFunc::isOperator(next, ')')) {
} else if (next.is_operator(')')) {
break;
} else if (TokenFunc::isOperator(next, '(')) {
} else if (next.is_operator('(')) {
return_value++;
load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
} else {
return_value++;
if (TokenFunc::isIdentifier(next) && entity && entity_instance_name) {
if (next.is_identifier() && entity && entity_instance_name) {
register_inverse(*entity_instance_name, entity, next.value_int, attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
}
if (TokenFunc::isKeyword(next)) {
if (next.is_keyword()) {
try {
const auto* decl = (schema ? schema : file->schema())->declaration_by_name(TokenFunc::asStringRef(next));
const auto* decl = (schema ? schema : file->schema())->declaration_by_name(next.as_string());
parse_context ps;
tokens->Next();
// The only case we know where a defined type contains entity
@@ -568,7 +498,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
// @todo do we need express::Base here? Or should we just push InstanceData?
context.push(simple_type_instance);
} catch (IfcException& e) {
Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(next.startPos));
Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + " at offset " + std::to_string(next.start_pos));
// #4070 We didn't actually capture an aggregate entry, undo length increment.
return_value--;
}
@@ -580,25 +510,10 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
}
}
//
// Reads an Entity from the list of Tokens at the specified offset in the file
//
std::shared_ptr<InstanceData> IfcParse::impl::in_memory_file_storage::read(unsigned int i) {
Token datatype = tokens->Next();
if (!TokenFunc::isKeyword(datatype)) {
throw IfcException("Unexpected token while parsing entity");
}
const IfcParse::declaration* ty = file->schema()->declaration_by_name(TokenFunc::asStringRef(datatype));
parse_context pc;
tokens->Next();
load(i, ty->as_entity(), pc, -1);
return pc.construct(file, i, *references_to_resolve, ty, std::nullopt, -1);
}
void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const {
auto old_offset = tokens->stream->tell();
Token semilocon = tokens->Next();
if (!TokenFunc::isOperator(semilocon, ';')) {
if (!semilocon.is_operator(';')) {
tokens->stream->seek(old_offset);
}
}
@@ -1354,8 +1269,8 @@ bool IfcParse::InstanceStreamer::hasSemicolon() const {
} catch (const std::out_of_range&) {
return false;
}
while (t.type != Token_NONE) {
if (TokenFunc::isOperator(t, ';')) {
while (t.type != Token::Token_NONE) {
if (t.is_operator(';')) {
return true;
}
try {
@@ -1378,8 +1293,8 @@ size_t IfcParse::InstanceStreamer::semicolonCount() const {
} catch (const std::out_of_range&) {
return false;
}
while (t.type != Token_NONE) {
if (TokenFunc::isOperator(t, ';')) {
while (t.type != Token::Token_NONE) {
if (t.is_operator(';')) {
count++;
}
try {
src/ifcparse/IfcParse.h
+13 -62
View File
@@ -47,64 +47,6 @@ extern const char *IFCOPENSHELL_VERSION;
namespace IfcParse {
/// Provides functions to convert Tokens to binary data
/// Tokens are merely offsets to where they can be read in the file
class IFC_PARSE_API TokenFunc {
private:
static bool startsWith(const Token& token, char character);
public:
/// Returns the offset at which the token is read from the file
// static unsigned int Offset(const Token& t);
/// Returns whether the token can be interpreted as a string
static bool isString(const Token& token);
/// Returns whether the token can be interpreted as an identifier
static bool isIdentifier(const Token& token);
/// Returns whether the token can be interpreted as a syntactical operator
static bool isOperator(const Token& token);
/// Returns whether the token is a given operator
static bool isOperator(const Token& token, char character);
/// Returns whether the token can be interpreted as an enumerated value
static bool isEnumeration(const Token& token);
/// Returns whether the token can be interpreted as a datatype name
static bool isKeyword(const Token& token);
/// Returns whether the token can be interpreted as an integer
static bool isInt(const Token& token);
/// Returns whether the token can be interpreted as a boolean
static bool isBool(const Token& token);
/// Returns whether the token can be interpreted as a logical
static bool isLogical(const Token& token);
/// Returns whether the token can be interpreted as a floating point number
static bool isFloat(const Token& token);
/// Returns whether the token can be interpreted as a binary type
static bool isBinary(const Token& token);
/// Returns the token interpreted as an integer
static int asInt(const Token& token);
/// Returns the token interpreted as an identifier
static int asIdentifier(const Token& token);
/// Returns the token interpreted as an boolean (.T. or .F.)
static bool asBool(const Token& token);
/// Returns the token interpreted as an logical (.T. or .F. or .U.)
static boost::logic::tribool asLogical(const Token& token);
/// Returns the token as a floating point number
static double asFloat(const Token& token);
/// Returns the token as a string (without the dot or apostrophe)
static std::string asString(const Token& token);
/// Returns the token as a string in internal buffer (for optimization purposes)
static const std::string& asStringRef(const Token& token);
/// Returns the token as a string (without the dot or apostrophe)
static boost::dynamic_bitset<> asBinary(const Token& token);
/// Returns a string representation of the token (including the dot or apostrophe)
static std::string toString(const Token& token);
};
//
// Functions for creating Tokens from an arbitary file offset
// The first 4 bits are reserved for Tokens of type ()=,;$*
//
Token OperatorTokenPtr(IfcSpfLexer* tokens, size_t start, char data);
Token GeneralTokenPtr(IfcSpfLexer* tokens, size_t start, const std::string& data);
/// A stream of tokens to be read from a FileReader.
class IFC_PARSE_API IfcSpfLexer {
private:
@@ -112,17 +54,26 @@ class IFC_PARSE_API IfcSpfLexer {
size_t skipWhitespace() const;
size_t skipComment() const;
mutable std::vector<std::unique_ptr<std::array<std::string, 16>>> stringpool_;
mutable size_t pool_index = 0;
public:
std::string& GetTempString() const {
static thread_local std::string string;
return string;
std::string& getTempString() const;
void resetPool() const {
pool_index = 0;
}
void popPoolEntry() {
if (pool_index > 0) {
--pool_index;
}
}
FileReader* stream;
// IfcFile* file;
IfcSpfLexer(FileReader* stream);
Token Next();
~IfcSpfLexer();
void TokenString(size_t offset, std::string& result);
// void TokenString(size_t offset, std::string& result);
};
IFC_PARSE_API std::vector<express::Base> traverse(const express::Base& instance, int max_level = -1);
src/ifcparse/IfcSpfHeader.cpp
+2 -2
View File
@@ -45,7 +45,7 @@ namespace {
void IfcSpfHeader::readSemicolon() {
if (storage_ != nullptr) {
if (!TokenFunc::isOperator(storage_->tokens->Next(), ';')) {
if (!storage_->tokens->Next().is_operator(';')) {
throw IfcException(std::string("Expected ;"));
}
} else {
@@ -55,7 +55,7 @@ void IfcSpfHeader::readSemicolon() {
void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) {
if (storage_ != nullptr) {
if (TokenFunc::asStringRef(storage_->tokens->Next()) != term) {
if (storage_->tokens->Next().as_string() != term) {
throw IfcException(std::string("Expected " + term));
}
if (trail == TRAILING_SEMICOLON) {
src/ifcparse/storage.h
+57 -25
View File
@@ -134,37 +134,71 @@ namespace IfcParse {
class IfcSpfLexer;
class FileReader;
enum TokenType {
Token_NONE,
Token_STRING,
Token_IDENTIFIER,
Token_OPERATOR,
Token_ENUMERATION,
Token_KEYWORD,
Token_INT,
Token_BOOL,
Token_FLOAT,
Token_BINARY
};
struct Token {
IfcSpfLexer* lexer; //TODO: remove it from here
size_t startPos;
enum TokenType {
Token_NONE,
Token_STRING,
Token_IDENTIFIER,
Token_OPERATOR,
Token_ENUMERATION,
Token_KEYWORD,
Token_INT,
Token_BOOL,
Token_FLOAT,
Token_BINARY
};
size_t start_pos;
TokenType type;
union {
char value_char; //types: OPERATOR
int value_int; //types: INT, IDENTIFIER
double value_double; //types: FLOAT
const std::string* value_string; //types: STR, ENUM, KEYWORD; lifetime managed by IfcSpfLexer::string_pool_
};
Token() : lexer(0),
startPos(0),
type(Token_NONE) {
}
Token(IfcSpfLexer* _lexer, size_t _startPos, TokenType _type)
: lexer(_lexer),
startPos(_startPos),
type(_type) {
Token() : start_pos(0),
type(Token_NONE) {}
Token(size_t start, TokenType ty, const std::string& str)
: start_pos(start), type(ty), value_string(&str) {}
Token(size_t start, TokenType ty, int i)
: start_pos(start), type(ty), value_int(i) {}
Token(size_t start, double d)
: start_pos(start), type(Token_FLOAT), value_double(d) {}
Token(size_t start, char op)
: start_pos(start), type(Token_OPERATOR), value_char(op) {}
Token(size_t start, TokenType ty, char c)
: start_pos(start), type(ty), value_char(c) {}
bool is_string();
bool is_identifier();
bool is_operator();
bool is_operator(char character);
bool is_enumeration();
bool is_keyword();
bool is_int();
bool is_bool();
bool is_logical();
bool is_float();
bool is_binary();
int as_int();
unsigned as_identifier();
bool as_bool();
boost::logic::tribool as_logical();
double as_float();
const std::string& as_string();
boost::dynamic_bitset<> as_binary();
std::string to_string();
operator bool() const {
return type != Token_NONE;
}
};
@@ -275,8 +309,6 @@ namespace IfcParse {
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, const express::Base&, int attribute_index);
// @todo is this still used
std::shared_ptr<InstanceData> read(unsigned int index);
void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
file_open_status good_ = file_open_status::SUCCESS;