Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-11 18:16:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement multi_threaded implementation in Iterator

Browse Source
This commit is contained in:
Thomas Krijnen
2019-07-03 12:10:27 +02:00
parent ae2feb2848
commit 3ebca0516e
13 changed files with 434 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/IfcCharacterDecoder.cpp
+146 -96
View File
@@ -85,112 +85,162 @@ IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::IfcSpfStream* f) {
IfcCharacterDecoder::~IfcCharacterDecoder() {
}
IfcCharacterDecoder::operator std::string() {
unsigned int parse_state = 0;
builder_.clear();
builder_.push_back('\'');
char current_char;
int codepage = 1;
unsigned int hex = 0;
unsigned int hex_count = 0;
namespace {
static unsigned int reference_helper = 0;
while ( (current_char = file->Peek()) != 0 ) {
if ( EXPECTS_CHARACTER(parse_state) ) {
builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80));
parse_state = 0;
} else if ( current_char == '\'' && ! parse_state ) {
parse_state = APOSTROPHE;
} else if ( current_char == '\\' && ! parse_state ) {
parse_state = FIRST_SOLIDUS;
} else if ( current_char == '\\' && EXPECTS_SOLIDUS(parse_state) ) {
if ( parse_state & ALPHABET_DEFINITION ||
parse_state & IGNORED_DIRECTIVE ||
parse_state & ENDEXTENDED_0 ) parse_state = hex = hex_count = 0;
else if ( parse_state & ENCOUNTERED_HEX ) {
parse_state += THIRD_SOLIDUS;
parse_state -= ENCOUNTERED_HEX;
class pure_impure_helper {
private:
bool pure_;
IfcParse::IfcSpfStream* stream_;
unsigned int& pointer_;
std::wstring builder_;
char peek() {
if (pure_) {
return stream_->peek_at(pointer_);
} else {
return stream_->Peek();
}
else parse_state += SECOND_SOLIDUS;
} else if ( current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state) ) {
parse_state += ENDEXTENDED_X;
} else if ( current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state) ) {
parse_state += ENDEXTENDED_0;
} else if ( current_char == 'X' && EXPECTS_ARBITRARY(parse_state) ) {
parse_state += ARBITRARY;
} else if ( current_char == '2' && EXPECTS_ARBITRARY2(parse_state) ) {
parse_state += EXTENDED2;
} else if ( current_char == '4' && EXPECTS_ARBITRARY2(parse_state) ) {
parse_state += EXTENDED2 + EXTENDED4;
} else if ( current_char == 'P' && EXPECTS_ALPHABET(parse_state) ) {
parse_state += ALPHABET;
} else if ( (current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state) ) {
parse_state += IGNORED_DIRECTIVE;
} else if ( IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state) ) {
codepage = current_char - 0x40;
parse_state += ALPHABET_DEFINITION;
} else if ( current_char == 'S' && EXPECTS_PAGE(parse_state) ) {
parse_state += PAGE;
} else if ( IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state) ) {
hex <<= 4;
parse_state += HEX((++hex_count));
hex += HEX_TO_INT(current_char);
if ( (hex_count == 2 && !(parse_state & EXTENDED2)) ||
(hex_count == 4 && !(parse_state & EXTENDED4)) ||
(hex_count == 8) )
{
builder_.push_back(hex);
if ( hex_count == 2 ) parse_state = 0;
else {
CLEAR_HEX(parse_state);
parse_state |= ENCOUNTERED_HEX;
}
hex = hex_count = 0;
}
} else if ( parse_state && !(
(current_char == '\\' && parse_state == FIRST_SOLIDUS) ||
(current_char == '\'' && parse_state == APOSTROPHE)
) ) {
if ( parse_state == APOSTROPHE && current_char != '\'' ) break;
throw IfcInvalidTokenException(file->Tell(), current_char);
} else {
parse_state = hex = hex_count = 0;
builder_.push_back(current_char);
}
file->Inc();
}
builder_.push_back('\'');
if (mode == UTF8) {
return IfcUtil::convert_utf8(builder_);
} else if (mode == SUBSTITUTE) {
std::string r;
r.reserve(builder_.size());
const char& sub = substitution_character;
std::transform(builder_.begin(), builder_.end(), std::back_inserter(r), [&sub](wchar_t c) {
if (c >= 0x20 && c <= 0x7e) {
return (char)c;
unsigned int tell() {
if (pure_) {
return pointer_;
} else {
return sub;
return stream_->Tell();
}
});
return r;
} else if (mode == ESCAPE) {
std::stringstream str;
str << std::hex << std::setw(4) << std::setfill('0');
std::for_each(builder_.begin(), builder_.end(), [&str](wchar_t c) {
if (c >= 0x20 && c <= 0x7e) {
str.put((char)c);
}
void increment() {
if (pure_) {
stream_->increment_at(pointer_);
} else {
str << "\\u" << c;
stream_->Inc();
}
});
return str.str();
} else {
throw IfcParse::IfcException("Invalid conversion mode");
}
}
public:
pure_impure_helper(IfcParse::IfcSpfStream* stream)
: pure_(false), stream_(stream), pointer_(reference_helper)
{}
pure_impure_helper(IfcParse::IfcSpfStream* stream, unsigned int& pointer)
: pure_(true), stream_(stream), pointer_(pointer)
{}
std::string get(IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
unsigned int parse_state = 0;
builder_.clear();
builder_.push_back('\'');
char current_char;
int codepage = 1;
unsigned int hex = 0;
unsigned int hex_count = 0;
while ((current_char = peek()) != 0) {
if (EXPECTS_CHARACTER(parse_state)) {
builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80));
parse_state = 0;
} else if (current_char == '\'' && !parse_state) {
parse_state = APOSTROPHE;
} else if (current_char == '\\' && !parse_state) {
parse_state = FIRST_SOLIDUS;
} else if (current_char == '\\' && EXPECTS_SOLIDUS(parse_state)) {
if (parse_state & ALPHABET_DEFINITION ||
parse_state & IGNORED_DIRECTIVE ||
parse_state & ENDEXTENDED_0) parse_state = hex = hex_count = 0;
else if (parse_state & ENCOUNTERED_HEX) {
parse_state += THIRD_SOLIDUS;
parse_state -= ENCOUNTERED_HEX;
} else parse_state += SECOND_SOLIDUS;
} else if (current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state)) {
parse_state += ENDEXTENDED_X;
} else if (current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state)) {
parse_state += ENDEXTENDED_0;
} else if (current_char == 'X' && EXPECTS_ARBITRARY(parse_state)) {
parse_state += ARBITRARY;
} else if (current_char == '2' && EXPECTS_ARBITRARY2(parse_state)) {
parse_state += EXTENDED2;
} else if (current_char == '4' && EXPECTS_ARBITRARY2(parse_state)) {
parse_state += EXTENDED2 + EXTENDED4;
} else if (current_char == 'P' && EXPECTS_ALPHABET(parse_state)) {
parse_state += ALPHABET;
} else if ((current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state)) {
parse_state += IGNORED_DIRECTIVE;
} else if (IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state)) {
codepage = current_char - 0x40;
parse_state += ALPHABET_DEFINITION;
} else if (current_char == 'S' && EXPECTS_PAGE(parse_state)) {
parse_state += PAGE;
} else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) {
hex <<= 4;
parse_state += HEX((++hex_count));
hex += HEX_TO_INT(current_char);
if ((hex_count == 2 && !(parse_state & EXTENDED2)) ||
(hex_count == 4 && !(parse_state & EXTENDED4)) ||
(hex_count == 8)) {
builder_.push_back(hex);
if (hex_count == 2) parse_state = 0;
else {
CLEAR_HEX(parse_state);
parse_state |= ENCOUNTERED_HEX;
}
hex = hex_count = 0;
}
} else if (parse_state && !(
(current_char == '\\' && parse_state == FIRST_SOLIDUS) ||
(current_char == '\'' && parse_state == APOSTROPHE)
)) {
if (parse_state == APOSTROPHE && current_char != '\'') break;
throw IfcInvalidTokenException(tell(), current_char);
} else {
parse_state = hex = hex_count = 0;
builder_.push_back(current_char);
}
increment();
}
builder_.push_back('\'');
if (mode == IfcParse::IfcCharacterDecoder::UTF8) {
return IfcUtil::convert_utf8(builder_);
} else if (mode == IfcParse::IfcCharacterDecoder::SUBSTITUTE) {
std::string r;
r.reserve(builder_.size());
std::transform(builder_.begin(), builder_.end(), std::back_inserter(r), [&substitution_character](wchar_t c) {
if (c >= 0x20 && c <= 0x7e) {
return (char)c;
} else {
return substitution_character;
}
});
return r;
} else if (mode == IfcParse::IfcCharacterDecoder::ESCAPE) {
std::stringstream str;
str << std::hex << std::setw(4) << std::setfill('0');
std::for_each(builder_.begin(), builder_.end(), [&str](wchar_t c) {
if (c >= 0x20 && c <= 0x7e) {
str.put((char)c);
} else {
str << "\\u" << c;
}
});
return str.str();
} else {
throw IfcParse::IfcException("Invalid conversion mode");
}
}
};
}
void IfcCharacterDecoder::dryRun() {
IfcCharacterDecoder::operator std::string() {
return pure_impure_helper(file).get(mode, substitution_character);
}
std::string IfcCharacterDecoder::get(unsigned int& ptr) {
return pure_impure_helper(file, ptr).get(mode, substitution_character);
}
void IfcCharacterDecoder::skip() {
unsigned int parse_state = 0;
char current_char;
unsigned int hex_count = 0;
src/ifcparse/IfcCharacterDecoder.h
+8 -2
View File
@@ -43,7 +43,6 @@ namespace IfcParse {
class IFC_PARSE_API IfcCharacterDecoder {
private:
IfcParse::IfcSpfStream* file;
std::wstring builder_;
int codepage_;
public:
enum ConversionMode {SUBSTITUTE, UTF8, ESCAPE};
@@ -51,8 +50,15 @@ namespace IfcParse {
static char substitution_character;
IfcCharacterDecoder(IfcParse::IfcSpfStream* file);
~IfcCharacterDecoder();
void dryRun();
// Only advances the underlying token stream read pointer
// to the next token.
void skip();
// Gets a decoded string representation at the token stream
// read pointer and advances the underlying token stream.
operator std::string();
// Gets a decoded string representation at the offset provided,
// does not mutate the underlying token stream read pointer.
std::string get(unsigned int&);
};
}
src/ifcparse/IfcFile.h
+2 -1
View File
@@ -199,8 +199,9 @@ public:
std::string createTimestamp() const;
void load(const IfcEntityInstanceData&);
size_t load(unsigned entity_instance_name, Argument**& attributes, size_t num_attributes);
void seek_to(const IfcEntityInstanceData& data);
void try_read_semicolon();
void register_inverse(unsigned, Token);
void register_inverse(unsigned, IfcUtil::IfcBaseClass*);
src/ifcparse/IfcLogger.cpp
+4
View File
@@ -29,6 +29,7 @@
#include <boost/property_tree/json_parser.hpp>
#include <boost/version.hpp>
#include <mutex>
#include <iostream>
#include <algorithm>
@@ -114,6 +115,9 @@ void Logger::SetOutput(std::wostream* l1, std::wostream* l2) {
}
void Logger::Message(Logger::Severity type, const std::string& message, const IfcUtil::IfcBaseClass* instance) {
static std::mutex m;
std::lock_guard<std::mutex> lk(m);
if (type > max_severity) {
max_severity = type;
}
src/ifcparse/IfcParse.cpp
+55 -27
View File
@@ -17,17 +17,6 @@
* *
********************************************************************************/
#include <set>
#include <algorithm>
#include <string>
#include <stdio.h>
#include <stdlib.h>
#include <ctime>
#include <boost/circular_buffer.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/math/special_functions/fpclassify.hpp>
#include "../ifcparse/IfcCharacterDecoder.h"
#include "../ifcparse/IfcParse.h"
#include "../ifcparse/IfcException.h"
@@ -42,6 +31,18 @@
#include <boost/filesystem/path.hpp>
#endif
#include <set>
#include <ctime>
#include <mutex>
#include <string>
#include <stdio.h>
#include <stdlib.h>
#include <algorithm>
#include <boost/circular_buffer.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/math/special_functions/fpclassify.hpp>
#define PERMISSIVE_FLOAT
using namespace IfcParse;
@@ -251,9 +252,11 @@ void IfcSpfStream::Inc() {
eof = true;
return;
}
/// @todo: Shouldn't this be a loop of some kind
const char current = IfcSpfStream::Peek();
if ( current == '\n' || current == '\r' ) IfcSpfStream::Inc();
if (current == '\n' || current == '\r') {
// NB this is recursive. It might as well be a loop.
IfcSpfStream::Inc();
}
}
IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream *s, IfcParse::IfcFile* f) {
@@ -331,34 +334,46 @@ Token IfcSpfLexer::Next() {
len ++;
// If a string is encountered defer processing to the IfcCharacterDecoder
if ( c == '\'' ) decoder->dryRun();
if ( c == '\'' ) decoder->skip();
}
if ( len ) return GeneralTokenPtr(this, pos, stream->Tell());
else return NoneTokenPtr();
}
bool IfcSpfStream::is_eof_at(unsigned int local_ptr) {
return local_ptr >= len;
}
void IfcSpfStream::increment_at(unsigned int& local_ptr) {
if (++local_ptr == len) {
return;
}
const char current = IfcSpfStream::peek_at(local_ptr);
if (current == '\n' || current == '\r') IfcSpfStream::increment_at(local_ptr);
}
char IfcSpfStream::peek_at(unsigned int local_ptr) {
return buffer[local_ptr];
}
//
// Reads a std::string from the file at specified offset
// Omits whitespace and comments
//
void IfcSpfLexer::TokenString(unsigned int offset, std::string &buffer) {
const bool was_eof = stream->eof;
unsigned int old_offset = stream->Tell();
stream->Seek(offset);
buffer.clear();
while ( ! stream->eof ) {
char c = stream->Peek();
while (!stream->is_eof_at(offset)) {
char c = stream->peek_at(offset);
if ( buffer.size() && (c == '(' || c == ')' || c == '=' || c == ',' || c == ';' || c == '/') ) break;
stream->Inc();
stream->increment_at(offset);
if ( c == ' ' || c == '\r' || c == '\n' || c == '\t' ) continue;
else if ( c == '\'' ) {
buffer = *decoder;
// todo, make decoder use local offset ptr
buffer = decoder->get(offset);
break;
}
else buffer.push_back(c);
}
if ( was_eof ) stream->eof = true;
else stream->Seek(old_offset);
}
//Note: according to STEP standard, there may be newlines in tokens
@@ -887,14 +902,16 @@ IfcEntityInstanceData* IfcParse::read(unsigned int i, IfcFile* f, boost::optiona
return e;
}
void IfcParse::IfcFile::load(const IfcEntityInstanceData& data) {
void IfcParse::IfcFile::seek_to(const IfcEntityInstanceData& data) {
if (tokens->stream->Tell() != data.offset_in_file()) {
tokens->stream->Seek(data.offset_in_file());
Token datatype = tokens->Next();
if (!TokenFunc::isKeyword(datatype)) throw IfcException("Unexpected token while parsing entity instance");
}
tokens->Next();
load(data.id(), data.attributes(), data.getArgumentCount());
}
void IfcParse::IfcFile::try_read_semicolon() {
unsigned int old_offset = tokens->stream->Tell();
Token semilocon = tokens->Next();
if (!TokenFunc::isOperator(semilocon, ';')) {
@@ -984,15 +1001,26 @@ unsigned IfcEntityInstanceData::set_id(boost::optional<unsigned> i) {
// Returns the entities of Entity type that have this entity in their ArgumentList
//
IfcEntityList::ptr IfcEntityInstanceData::getInverse(const IfcParse::declaration* type, int attribute_index) const {
static std::mutex m;
std::lock_guard<std::mutex> lk(m);
return file->getInverse(id_, type, attribute_index);
}
void IfcEntityInstanceData::load() const {
static std::recursive_mutex m;
std::lock_guard<std::recursive_mutex> lk(m);
// type_ is 0 for header entities which have their size predetermined in code
Argument** tmp_data = nullptr;
if (type_ != 0) {
attributes_ = new Argument*[getArgumentCount()];
tmp_data = new Argument*[getArgumentCount()];
}
file->load(*this);
file->seek_to(*this);
file->load(id(), tmp_data, getArgumentCount());
file->try_read_semicolon();
// @todo does this need to be atomic somehow?
attributes_ = tmp_data;
}
IfcEntityInstanceData::IfcEntityInstanceData(const IfcEntityInstanceData& e) {
src/ifcparse/IfcParse.h
+15 -3
View File
@@ -49,6 +49,17 @@
#include "../ifcparse/IfcSpfStream.h"
/* gcc doesn't know _Thread_local from C11 yet */
#ifdef __GNUC__
# define my_thread_local __thread
#elif __STDC_VERSION__ >= 201112L
# define my_thread_local _Thread_local
#elif defined(_MSC_VER)
# define my_thread_local __declspec(thread)
#else
# error Cannot define thread_local
#endif
namespace IfcParse {
class IfcFile;
@@ -141,12 +152,13 @@ namespace IfcParse {
class IFC_PARSE_API IfcSpfLexer {
private:
IfcCharacterDecoder* decoder;
//storage for temporary string without allocation
mutable std::string _tempString;
unsigned int skipWhitespace();
unsigned int skipComment();
public:
std::string &GetTempString() const { return _tempString; }
std::string &GetTempString() const {
static my_thread_local std::string s;
return s;
}
IfcSpfStream* stream;
IfcFile* file;
IfcSpfLexer(IfcSpfStream* s, IfcFile* f);
src/ifcparse/IfcSpfStream.h
+4
View File
@@ -72,6 +72,10 @@ namespace IfcParse {
void Seek(unsigned int offset);
/// Returns the cursor position
unsigned int Tell();
bool is_eof_at(unsigned int);
void increment_at(unsigned int&);
char peek_at(unsigned int);
};
}