From 3ebca0516e8484b7369e03737da23dcba31ce45f Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Wed, 3 Jul 2019 12:10:27 +0200 Subject: [PATCH] Implement multi_threaded implementation in Iterator --- src/ifcconvert/IfcConvert.cpp | 6 +- src/ifcgeom/IfcGeomIteratorImplementation.cpp | 4 +- src/ifcgeom/IfcGeomIteratorImplementation.h | 218 +++++++++++++--- src/ifcgeom_schema_agnostic/IfcGeomIterator.h | 8 +- .../IteratorImplementation.cpp | 4 +- .../IteratorImplementation.h | 8 +- src/ifcparse/IfcCharacterDecoder.cpp | 242 +++++++++++------- src/ifcparse/IfcCharacterDecoder.h | 10 +- src/ifcparse/IfcFile.h | 3 +- src/ifcparse/IfcLogger.cpp | 4 + src/ifcparse/IfcParse.cpp | 82 ++++-- src/ifcparse/IfcParse.h | 18 +- src/ifcparse/IfcSpfStream.h | 4 + 13 files changed, 434 insertions(+), 177 deletions(-) diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index 49f707a93c..bca7396422 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -218,9 +218,13 @@ int main(int argc, char** argv) { ifc_options.add_options() ("calculate-quantities", "Calculate or fix the physical quantity definitions " "based on an interpretation of the geometry when exporting IFC"); + + size_t num_threads; po::options_description geom_options("Geometry options"); geom_options.add_options() + ("threads,j", po::value(&num_threads)->default_value(1), + "Number of parallel processing threads for geometry interpretation.") ("plan", "Specifies whether to include curves in the output result. Typically " "these are representations of type Plan or Axis. Excluded by default.") @@ -730,7 +734,7 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } - IfcGeom::Iterator context_iterator(settings, ifc_file, filter_funcs); + IfcGeom::Iterator context_iterator(settings, ifc_file, filter_funcs, num_threads); if (!context_iterator.initialize()) { /// @todo It would be nice to know and print separate error prints for a case where we found no entities /// and for a case we found no entities that satisfy our filtering criteria. diff --git a/src/ifcgeom/IfcGeomIteratorImplementation.cpp b/src/ifcgeom/IfcGeomIteratorImplementation.cpp index d693dbfa68..3d3616bf21 100644 --- a/src/ifcgeom/IfcGeomIteratorImplementation.cpp +++ b/src/ifcgeom/IfcGeomIteratorImplementation.cpp @@ -14,8 +14,8 @@ namespace IfcGeom { namespace { template struct MAKE_TYPE_NAME(factory_t) { - IfcGeom::IteratorImplementation* operator()(const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters) const { - return new IfcGeom::MAKE_TYPE_NAME(IteratorImplementation_)(settings, file, filters); + IfcGeom::IteratorImplementation* operator()(const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters, size_t num_threads) const { + return new IfcGeom::MAKE_TYPE_NAME(IteratorImplementation_)(settings, file, filters, num_threads); } }; } diff --git a/src/ifcgeom/IfcGeomIteratorImplementation.h b/src/ifcgeom/IfcGeomIteratorImplementation.h index 84e1ffdcec..6c79da4bcc 100644 --- a/src/ifcgeom/IfcGeomIteratorImplementation.h +++ b/src/ifcgeom/IfcGeomIteratorImplementation.h @@ -64,6 +64,10 @@ #include #include +#include +#include +#include + #include #include @@ -92,12 +96,46 @@ #undef max #endif +namespace { + template + struct geometry_conversion_task { + int index; + IfcSchema::IfcRepresentation *representation; + IfcSchema::IfcProduct::list::ptr products; + std::vector*> breps; + std::vector*> elements; + }; + + template + void create_element( + IfcGeom::MAKE_TYPE_NAME(Kernel)* kernel, + const IfcGeom::IteratorSettings& settings, + geometry_conversion_task* rep) + { + IfcSchema::IfcRepresentation *representation = rep->representation; + IfcSchema::IfcProduct *product = *rep->products->begin(); + rep->breps = { kernel->create_brep_for_representation_and_product(settings, representation, product) }; + // @todo based on settings + rep->elements = { rep->breps[0] ? new IfcGeom::TriangulationElement(*rep->breps[0]) : nullptr }; + + for (auto it = rep->products->begin() + 1; it != rep->products->end(); ++it) { + rep->breps.push_back(kernel->create_brep_for_processed_representation(settings, representation, *it, rep->breps[0])); + rep->elements.push_back(rep->breps.back() ? new IfcGeom::TriangulationElement(*rep->breps.back()) : nullptr); + } + } +} + namespace IfcGeom { template class MAKE_TYPE_NAME(IteratorImplementation_) : public IteratorImplementation { private: + size_t num_threads_; + std::vector> tasks_; + std::vector*> all_processed_elements_; + typename std::vector*>::const_iterator task_result_iterator_; + MAKE_TYPE_NAME(IteratorImplementation_)(const MAKE_TYPE_NAME(IteratorImplementation_)&); // N/I MAKE_TYPE_NAME(IteratorImplementation_)& operator=(const MAKE_TYPE_NAME(IteratorImplementation_)&); // N/I @@ -282,16 +320,90 @@ namespace IfcGeom { representation_iterator = representations->begin(); ifcproducts.reset(); - if (!create()) { - return false; - } - done = 0; total = representations->size(); + if (num_threads_ != 1) { + collect(); + process_concurrently(); + } else { + if (!create()) { + return false; + } + } + return true; } + void collect() { + int i = 0; + IfcSchema::IfcProduct::list* previous = nullptr; + while (auto rp = get_next_task()) { + // Note that get_next_task() mutates the state of the iterator + // we use that capture all products that can be processed as + // part of this representation and then keep iterating until + // the underlying list of products changes. + if (ifcproducts.get() != previous) { + previous = ifcproducts.get(); + geometry_conversion_task t; + t.index = i++; + t.representation = *representation_iterator; + t.products = ifcproducts; + tasks_.emplace_back(t); + } + + _nextShape(); + } + } + + void process_concurrently() { + unsigned int conc_threads = std::thread::hardware_concurrency(); + if (conc_threads > (unsigned int)tasks_.size()) { + conc_threads = (unsigned int)tasks_.size(); + } + + std::vector kernel_pool; + kernel_pool.reserve(conc_threads); + for (unsigned i = 0; i < conc_threads; ++i) { + kernel_pool.push_back(new MAKE_TYPE_NAME(Kernel)(kernel)); + } + + std::vector> threadpool; + + for (auto& rep : tasks_) { + auto K = kernel_pool[threadpool.size()]; + + while (threadpool.size() == conc_threads) { + for (int i = 0; i < (int)threadpool.size(); i++) { + std::future &fu = threadpool[i]; + std::future_status status; + status = fu.wait_for(std::chrono::seconds(0)); + if (status == std::future_status::ready) { + fu.get(); + std::swap(threadpool[i], threadpool.back()); + threadpool.pop_back(); + std::swap(kernel_pool[i], kernel_pool.back()); + K = kernel_pool.back(); + break; + } // if + } // for + } // while + + std::future fu = std::async(std::launch::async, create_element, K, std::ref(settings), &rep); + threadpool.emplace_back(std::move(fu)); + } + + for (std::future &fu : threadpool) { + fu.get(); + } + + for (auto& rep : tasks_) { + all_processed_elements_.insert(all_processed_elements_.end(), rep.elements.begin(), rep.elements.end()); + } + + task_result_iterator_ = all_processed_elements_.begin(); + } + /// Computes model's bounding box (bounds_min and bounds_max). /// @note Can take several minutes for large files. void compute_bounds() @@ -403,13 +515,13 @@ namespace IfcGeom { return associated_single_materials.size() == 1; } - BRepElement* create_shape_model_for_next_entity() { + boost::optional> get_next_task() { for (;;) { IfcSchema::IfcRepresentation* representation; - if ( representation_iterator == representations->end() ) { + if (representation_iterator == representations->end()) { representations.reset(); - return 0; // reached the end of our list of representations + return boost::none; // reached the end of our list of representations } representation = *representation_iterator; @@ -417,20 +529,20 @@ namespace IfcGeom { // Init. the list of filtered IfcProducts for this representation ifcproducts = IfcSchema::IfcProduct::list::ptr(new IfcSchema::IfcProduct::list); IfcSchema::IfcProduct::list::ptr unfiltered_products = kernel.products_represented_by(representation); - // Include only the desired products for processing. - for (IfcSchema::IfcProduct::list::it jt = unfiltered_products->begin(); jt != unfiltered_products->end(); ++jt) { - IfcSchema::IfcProduct* prod = *jt; - if (boost::all(filters_, filter_match(prod))) { - ifcproducts->push(prod); - } - } + // Include only the desired products for processing. + for (IfcSchema::IfcProduct::list::it jt = unfiltered_products->begin(); jt != unfiltered_products->end(); ++jt) { + IfcSchema::IfcProduct* prod = *jt; + if (boost::all(filters_, filter_match(prod))) { + ifcproducts->push(prod); + } + } - if (ifcproducts->size() == 0) { - _nextShape(); - continue; - } + if (ifcproducts->size() == 0) { + _nextShape(); + continue; + } - geometry_reuse_ok_for_current_representation_ = reuse_ok_(ifcproducts); + geometry_reuse_ok_for_current_representation_ = reuse_ok_(ifcproducts); IfcSchema::IfcRepresentationMap::list::ptr maps = representation->RepresentationMap(); @@ -450,14 +562,14 @@ namespace IfcGeom { // Check if this represenation has (or will be) processed as part its mapped representation bool representation_processed_as_mapped_item = false; - IfcSchema::IfcRepresentation* representation_mapped_to = kernel.representation_mapped_to(representation); + IfcSchema::IfcRepresentation* representation_mapped_to = kernel.representation_mapped_to(representation); if (representation_mapped_to) { - representation_processed_as_mapped_item = geometry_reuse_ok_for_current_representation_ && ( - ok_mapped_representations->contains(representation_mapped_to) || reuse_ok_(kernel.products_represented_by(representation_mapped_to))); + representation_processed_as_mapped_item = geometry_reuse_ok_for_current_representation_ && ( + ok_mapped_representations->contains(representation_mapped_to) || reuse_ok_(kernel.products_represented_by(representation_mapped_to))); } if (representation_processed_as_mapped_item) { - ok_mapped_representations->push(representation_mapped_to); + ok_mapped_representations->push(representation_mapped_to); _nextShape(); continue; } @@ -466,13 +578,28 @@ namespace IfcGeom { } // Have we reached the end of our list of IfcProducts? - if ( ifcproduct_iterator == ifcproducts->end() ) { + if (ifcproduct_iterator == ifcproducts->end()) { _nextShape(); continue; } IfcSchema::IfcProduct* product = *ifcproduct_iterator; - Logger::SetProduct(product); + + + return std::make_pair(representation, product); + } + } + + BRepElement* create_shape_model_for_next_entity() { + for (;;) { + auto rp = get_next_task(); + if (!rp) { + return nullptr; + } + auto representation = rp->first; + auto product = rp->second; + + Logger::SetProduct(product); BRepElement* element; if (ifcproduct_iterator == ifcproducts->begin() || !geometry_reuse_ok_for_current_representation_) { @@ -520,13 +647,24 @@ namespace IfcGeom { /// Moves to the next shape representation, create its geometry, and returns the associated product. /// Use get() to retrieve the created geometry. IfcUtil::IfcBaseClass* next() { - // Increment the iterator over the list of products using the current - // shape representation - if (ifcproducts) { - ++ifcproduct_iterator; - } + if (num_threads_ != 1) { + do { + task_result_iterator_++; + } while (task_result_iterator_ != all_processed_elements_.end() && *task_result_iterator_ == nullptr); + if (task_result_iterator_ == all_processed_elements_.end()) { + return nullptr; + } else { + return (*task_result_iterator_)->product(); + } + } else { + // Increment the iterator over the list of products using the current + // shape representation + if (ifcproducts) { + ++ifcproduct_iterator; + } - return create(); + return create(); + } } /// Gets the representation of the current geometrical entity. @@ -534,9 +672,18 @@ namespace IfcGeom { { // TODO: Test settings and throw Element* ret = 0; - if (current_triangulation) { ret = current_triangulation; } - else if (current_serialization) { ret = current_serialization; } - else if (current_shape_model) { ret = current_shape_model; } + + if (num_threads_ != 1) { + ret = *task_result_iterator_; + } else { + if (current_triangulation) { + ret = current_triangulation; + } else if (current_serialization) { + ret = current_serialization; + } else if (current_shape_model) { + ret = current_shape_model; + } + } // If we want to organize the element considering their hierarchy if (settings.get(IteratorSettings::SEARCH_FLOOR)) @@ -721,11 +868,12 @@ namespace IfcGeom { bool owns_ifc_file; public: - MAKE_TYPE_NAME(IteratorImplementation_)(const IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters) + MAKE_TYPE_NAME(IteratorImplementation_)(const IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters, size_t num_threads) : settings(settings) , ifc_file(file) , filters_(filters) , owns_ifc_file(false) + , num_threads_(num_threads) { _initialize(); } diff --git a/src/ifcgeom_schema_agnostic/IfcGeomIterator.h b/src/ifcgeom_schema_agnostic/IfcGeomIterator.h index 4658d37780..9d46b39ac3 100644 --- a/src/ifcgeom_schema_agnostic/IfcGeomIterator.h +++ b/src/ifcgeom_schema_agnostic/IfcGeomIterator.h @@ -83,19 +83,19 @@ namespace IfcGeom { IteratorImplementation* implementation_; public: - Iterator(const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file) + Iterator(const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, size_t num_threads = 1) : file_(file) , settings_(settings) { - implementation_ = iterator_implementations().construct(file_->schema()->name(), settings, file, filters_); + implementation_ = iterator_implementations().construct(file_->schema()->name(), settings, file, filters_, num_threads); } - Iterator(const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters) + Iterator(const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters, size_t num_threads = 1) : file_(file) , settings_(settings) , filters_(filters) { - implementation_ = iterator_implementations().construct(file_->schema()->name(), settings, file, filters_); + implementation_ = iterator_implementations().construct(file_->schema()->name(), settings, file, filters_, num_threads); } bool initialize() { diff --git a/src/ifcgeom_schema_agnostic/IteratorImplementation.cpp b/src/ifcgeom_schema_agnostic/IteratorImplementation.cpp index 10608d8cb7..cf2f89f2df 100644 --- a/src/ifcgeom_schema_agnostic/IteratorImplementation.cpp +++ b/src/ifcgeom_schema_agnostic/IteratorImplementation.cpp @@ -31,14 +31,14 @@ void IteratorFactoryImplementation::bind(const std::string& schema_name, } template -IfcGeom::IteratorImplementation* IteratorFactoryImplementation::construct(const std::string& schema_name, const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters) { +IfcGeom::IteratorImplementation* IteratorFactoryImplementation::construct(const std::string& schema_name, const IfcGeom::IteratorSettings& settings, IfcParse::IfcFile* file, const std::vector& filters, size_t num_threads) { const std::string schema_name_lower = boost::to_lower_copy(schema_name); typename std::map::type>::const_iterator it; it = this->find(schema_name_lower); if (it == this->end()) { throw IfcParse::IfcException("No geometry iterator registered for " + schema_name); } - return it->second(settings, file, filters); + return it->second(settings, file, filters, num_threads); } diff --git a/src/ifcgeom_schema_agnostic/IteratorImplementation.h b/src/ifcgeom_schema_agnostic/IteratorImplementation.h index 69ddbb618f..5622c0eec8 100644 --- a/src/ifcgeom_schema_agnostic/IteratorImplementation.h +++ b/src/ifcgeom_schema_agnostic/IteratorImplementation.h @@ -23,9 +23,9 @@ namespace IfcGeom { class BRepElement; } -typedef boost::function3*, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&> iterator_float_float_fn; -typedef boost::function3*, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&> iterator_float_double_fn; -typedef boost::function3*, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&> iterator_double_double_fn; +typedef boost::function4*, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&, size_t> iterator_float_float_fn; +typedef boost::function4*, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&, size_t> iterator_float_double_fn; +typedef boost::function4*, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&, size_t> iterator_double_double_fn; template struct get_factory_type {}; @@ -50,7 +50,7 @@ class IteratorFactoryImplementation : public std::map::type fn); - IfcGeom::IteratorImplementation* construct(const std::string& schema_name, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&); + IfcGeom::IteratorImplementation* construct(const std::string& schema_name, const IfcGeom::IteratorSettings&, IfcParse::IfcFile*, const std::vector&, size_t); }; template diff --git a/src/ifcparse/IfcCharacterDecoder.cpp b/src/ifcparse/IfcCharacterDecoder.cpp index c0957f7684..3aef95d15f 100644 --- a/src/ifcparse/IfcCharacterDecoder.cpp +++ b/src/ifcparse/IfcCharacterDecoder.cpp @@ -85,112 +85,162 @@ IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::IfcSpfStream* f) { IfcCharacterDecoder::~IfcCharacterDecoder() { } -IfcCharacterDecoder::operator std::string() { - unsigned int parse_state = 0; - builder_.clear(); - builder_.push_back('\''); - char current_char; - int codepage = 1; - unsigned int hex = 0; - unsigned int hex_count = 0; +namespace { + static unsigned int reference_helper = 0; - while ( (current_char = file->Peek()) != 0 ) { - if ( EXPECTS_CHARACTER(parse_state) ) { - builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80)); - parse_state = 0; - } else if ( current_char == '\'' && ! parse_state ) { - parse_state = APOSTROPHE; - } else if ( current_char == '\\' && ! parse_state ) { - parse_state = FIRST_SOLIDUS; - } else if ( current_char == '\\' && EXPECTS_SOLIDUS(parse_state) ) { - if ( parse_state & ALPHABET_DEFINITION || - parse_state & IGNORED_DIRECTIVE || - parse_state & ENDEXTENDED_0 ) parse_state = hex = hex_count = 0; - else if ( parse_state & ENCOUNTERED_HEX ) { - parse_state += THIRD_SOLIDUS; - parse_state -= ENCOUNTERED_HEX; + class pure_impure_helper { + private: + bool pure_; + IfcParse::IfcSpfStream* stream_; + unsigned int& pointer_; + std::wstring builder_; + + char peek() { + if (pure_) { + return stream_->peek_at(pointer_); + } else { + return stream_->Peek(); } - else parse_state += SECOND_SOLIDUS; - } else if ( current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state) ) { - parse_state += ENDEXTENDED_X; - } else if ( current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state) ) { - parse_state += ENDEXTENDED_0; - } else if ( current_char == 'X' && EXPECTS_ARBITRARY(parse_state) ) { - parse_state += ARBITRARY; - } else if ( current_char == '2' && EXPECTS_ARBITRARY2(parse_state) ) { - parse_state += EXTENDED2; - } else if ( current_char == '4' && EXPECTS_ARBITRARY2(parse_state) ) { - parse_state += EXTENDED2 + EXTENDED4; - } else if ( current_char == 'P' && EXPECTS_ALPHABET(parse_state) ) { - parse_state += ALPHABET; - } else if ( (current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state) ) { - parse_state += IGNORED_DIRECTIVE; - } else if ( IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state) ) { - codepage = current_char - 0x40; - parse_state += ALPHABET_DEFINITION; - } else if ( current_char == 'S' && EXPECTS_PAGE(parse_state) ) { - parse_state += PAGE; - } else if ( IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state) ) { - hex <<= 4; - parse_state += HEX((++hex_count)); - hex += HEX_TO_INT(current_char); - if ( (hex_count == 2 && !(parse_state & EXTENDED2)) || - (hex_count == 4 && !(parse_state & EXTENDED4)) || - (hex_count == 8) ) - { - builder_.push_back(hex); - if ( hex_count == 2 ) parse_state = 0; - else { - CLEAR_HEX(parse_state); - parse_state |= ENCOUNTERED_HEX; - } - hex = hex_count = 0; - } - } else if ( parse_state && !( - (current_char == '\\' && parse_state == FIRST_SOLIDUS) || - (current_char == '\'' && parse_state == APOSTROPHE) - ) ) { - if ( parse_state == APOSTROPHE && current_char != '\'' ) break; - throw IfcInvalidTokenException(file->Tell(), current_char); - } else { - parse_state = hex = hex_count = 0; - builder_.push_back(current_char); } - file->Inc(); - } - builder_.push_back('\''); - if (mode == UTF8) { - return IfcUtil::convert_utf8(builder_); - } else if (mode == SUBSTITUTE) { - std::string r; - r.reserve(builder_.size()); - const char& sub = substitution_character; - std::transform(builder_.begin(), builder_.end(), std::back_inserter(r), [&sub](wchar_t c) { - if (c >= 0x20 && c <= 0x7e) { - return (char)c; + unsigned int tell() { + if (pure_) { + return pointer_; } else { - return sub; + return stream_->Tell(); } - }); - return r; - } else if (mode == ESCAPE) { - std::stringstream str; - str << std::hex << std::setw(4) << std::setfill('0'); - std::for_each(builder_.begin(), builder_.end(), [&str](wchar_t c) { - if (c >= 0x20 && c <= 0x7e) { - str.put((char)c); + } + + void increment() { + if (pure_) { + stream_->increment_at(pointer_); } else { - str << "\\u" << c; + stream_->Inc(); } - }); - return str.str(); - } else { - throw IfcParse::IfcException("Invalid conversion mode"); - } + } + + public: + pure_impure_helper(IfcParse::IfcSpfStream* stream) + : pure_(false), stream_(stream), pointer_(reference_helper) + {} + + pure_impure_helper(IfcParse::IfcSpfStream* stream, unsigned int& pointer) + : pure_(true), stream_(stream), pointer_(pointer) + {} + + std::string get(IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) { + unsigned int parse_state = 0; + builder_.clear(); + builder_.push_back('\''); + char current_char; + int codepage = 1; + unsigned int hex = 0; + unsigned int hex_count = 0; + + while ((current_char = peek()) != 0) { + if (EXPECTS_CHARACTER(parse_state)) { + builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80)); + parse_state = 0; + } else if (current_char == '\'' && !parse_state) { + parse_state = APOSTROPHE; + } else if (current_char == '\\' && !parse_state) { + parse_state = FIRST_SOLIDUS; + } else if (current_char == '\\' && EXPECTS_SOLIDUS(parse_state)) { + if (parse_state & ALPHABET_DEFINITION || + parse_state & IGNORED_DIRECTIVE || + parse_state & ENDEXTENDED_0) parse_state = hex = hex_count = 0; + else if (parse_state & ENCOUNTERED_HEX) { + parse_state += THIRD_SOLIDUS; + parse_state -= ENCOUNTERED_HEX; + } else parse_state += SECOND_SOLIDUS; + } else if (current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state)) { + parse_state += ENDEXTENDED_X; + } else if (current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state)) { + parse_state += ENDEXTENDED_0; + } else if (current_char == 'X' && EXPECTS_ARBITRARY(parse_state)) { + parse_state += ARBITRARY; + } else if (current_char == '2' && EXPECTS_ARBITRARY2(parse_state)) { + parse_state += EXTENDED2; + } else if (current_char == '4' && EXPECTS_ARBITRARY2(parse_state)) { + parse_state += EXTENDED2 + EXTENDED4; + } else if (current_char == 'P' && EXPECTS_ALPHABET(parse_state)) { + parse_state += ALPHABET; + } else if ((current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state)) { + parse_state += IGNORED_DIRECTIVE; + } else if (IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state)) { + codepage = current_char - 0x40; + parse_state += ALPHABET_DEFINITION; + } else if (current_char == 'S' && EXPECTS_PAGE(parse_state)) { + parse_state += PAGE; + } else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) { + hex <<= 4; + parse_state += HEX((++hex_count)); + hex += HEX_TO_INT(current_char); + if ((hex_count == 2 && !(parse_state & EXTENDED2)) || + (hex_count == 4 && !(parse_state & EXTENDED4)) || + (hex_count == 8)) { + builder_.push_back(hex); + if (hex_count == 2) parse_state = 0; + else { + CLEAR_HEX(parse_state); + parse_state |= ENCOUNTERED_HEX; + } + hex = hex_count = 0; + } + } else if (parse_state && !( + (current_char == '\\' && parse_state == FIRST_SOLIDUS) || + (current_char == '\'' && parse_state == APOSTROPHE) + )) { + if (parse_state == APOSTROPHE && current_char != '\'') break; + throw IfcInvalidTokenException(tell(), current_char); + } else { + parse_state = hex = hex_count = 0; + builder_.push_back(current_char); + } + increment(); + } + builder_.push_back('\''); + + if (mode == IfcParse::IfcCharacterDecoder::UTF8) { + return IfcUtil::convert_utf8(builder_); + } else if (mode == IfcParse::IfcCharacterDecoder::SUBSTITUTE) { + std::string r; + r.reserve(builder_.size()); + std::transform(builder_.begin(), builder_.end(), std::back_inserter(r), [&substitution_character](wchar_t c) { + if (c >= 0x20 && c <= 0x7e) { + return (char)c; + } else { + return substitution_character; + } + }); + return r; + } else if (mode == IfcParse::IfcCharacterDecoder::ESCAPE) { + std::stringstream str; + str << std::hex << std::setw(4) << std::setfill('0'); + std::for_each(builder_.begin(), builder_.end(), [&str](wchar_t c) { + if (c >= 0x20 && c <= 0x7e) { + str.put((char)c); + } else { + str << "\\u" << c; + } + }); + return str.str(); + } else { + throw IfcParse::IfcException("Invalid conversion mode"); + } + } + }; } -void IfcCharacterDecoder::dryRun() { +IfcCharacterDecoder::operator std::string() { + return pure_impure_helper(file).get(mode, substitution_character); +} + +std::string IfcCharacterDecoder::get(unsigned int& ptr) { + return pure_impure_helper(file, ptr).get(mode, substitution_character); +} + +void IfcCharacterDecoder::skip() { unsigned int parse_state = 0; char current_char; unsigned int hex_count = 0; diff --git a/src/ifcparse/IfcCharacterDecoder.h b/src/ifcparse/IfcCharacterDecoder.h index d178bae71d..9aaff758f3 100644 --- a/src/ifcparse/IfcCharacterDecoder.h +++ b/src/ifcparse/IfcCharacterDecoder.h @@ -43,7 +43,6 @@ namespace IfcParse { class IFC_PARSE_API IfcCharacterDecoder { private: IfcParse::IfcSpfStream* file; - std::wstring builder_; int codepage_; public: enum ConversionMode {SUBSTITUTE, UTF8, ESCAPE}; @@ -51,8 +50,15 @@ namespace IfcParse { static char substitution_character; IfcCharacterDecoder(IfcParse::IfcSpfStream* file); ~IfcCharacterDecoder(); - void dryRun(); + // Only advances the underlying token stream read pointer + // to the next token. + void skip(); + // Gets a decoded string representation at the token stream + // read pointer and advances the underlying token stream. operator std::string(); + // Gets a decoded string representation at the offset provided, + // does not mutate the underlying token stream read pointer. + std::string get(unsigned int&); }; } diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index f613c938c9..9021850341 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -199,8 +199,9 @@ public: std::string createTimestamp() const; - void load(const IfcEntityInstanceData&); size_t load(unsigned entity_instance_name, Argument**& attributes, size_t num_attributes); + void seek_to(const IfcEntityInstanceData& data); + void try_read_semicolon(); void register_inverse(unsigned, Token); void register_inverse(unsigned, IfcUtil::IfcBaseClass*); diff --git a/src/ifcparse/IfcLogger.cpp b/src/ifcparse/IfcLogger.cpp index 8320c0eb04..9ab29e74a4 100644 --- a/src/ifcparse/IfcLogger.cpp +++ b/src/ifcparse/IfcLogger.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -114,6 +115,9 @@ void Logger::SetOutput(std::wostream* l1, std::wostream* l2) { } void Logger::Message(Logger::Severity type, const std::string& message, const IfcUtil::IfcBaseClass* instance) { + static std::mutex m; + std::lock_guard lk(m); + if (type > max_severity) { max_severity = type; } diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index c97e4095c1..cb6cadf604 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -17,17 +17,6 @@ * * ********************************************************************************/ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - #include "../ifcparse/IfcCharacterDecoder.h" #include "../ifcparse/IfcParse.h" #include "../ifcparse/IfcException.h" @@ -42,6 +31,18 @@ #include #endif +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + #define PERMISSIVE_FLOAT using namespace IfcParse; @@ -251,9 +252,11 @@ void IfcSpfStream::Inc() { eof = true; return; } - /// @todo: Shouldn't this be a loop of some kind const char current = IfcSpfStream::Peek(); - if ( current == '\n' || current == '\r' ) IfcSpfStream::Inc(); + if (current == '\n' || current == '\r') { + // NB this is recursive. It might as well be a loop. + IfcSpfStream::Inc(); + } } IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream *s, IfcParse::IfcFile* f) { @@ -331,34 +334,46 @@ Token IfcSpfLexer::Next() { len ++; // If a string is encountered defer processing to the IfcCharacterDecoder - if ( c == '\'' ) decoder->dryRun(); + if ( c == '\'' ) decoder->skip(); } if ( len ) return GeneralTokenPtr(this, pos, stream->Tell()); else return NoneTokenPtr(); } +bool IfcSpfStream::is_eof_at(unsigned int local_ptr) { + return local_ptr >= len; +} + +void IfcSpfStream::increment_at(unsigned int& local_ptr) { + if (++local_ptr == len) { + return; + } + const char current = IfcSpfStream::peek_at(local_ptr); + if (current == '\n' || current == '\r') IfcSpfStream::increment_at(local_ptr); +} + +char IfcSpfStream::peek_at(unsigned int local_ptr) { + return buffer[local_ptr]; +} + // // Reads a std::string from the file at specified offset // Omits whitespace and comments // void IfcSpfLexer::TokenString(unsigned int offset, std::string &buffer) { - const bool was_eof = stream->eof; - unsigned int old_offset = stream->Tell(); - stream->Seek(offset); buffer.clear(); - while ( ! stream->eof ) { - char c = stream->Peek(); + while (!stream->is_eof_at(offset)) { + char c = stream->peek_at(offset); if ( buffer.size() && (c == '(' || c == ')' || c == '=' || c == ',' || c == ';' || c == '/') ) break; - stream->Inc(); + stream->increment_at(offset); if ( c == ' ' || c == '\r' || c == '\n' || c == '\t' ) continue; else if ( c == '\'' ) { - buffer = *decoder; + // todo, make decoder use local offset ptr + buffer = decoder->get(offset); break; } else buffer.push_back(c); } - if ( was_eof ) stream->eof = true; - else stream->Seek(old_offset); } //Note: according to STEP standard, there may be newlines in tokens @@ -887,14 +902,16 @@ IfcEntityInstanceData* IfcParse::read(unsigned int i, IfcFile* f, boost::optiona return e; } -void IfcParse::IfcFile::load(const IfcEntityInstanceData& data) { +void IfcParse::IfcFile::seek_to(const IfcEntityInstanceData& data) { if (tokens->stream->Tell() != data.offset_in_file()) { tokens->stream->Seek(data.offset_in_file()); Token datatype = tokens->Next(); if (!TokenFunc::isKeyword(datatype)) throw IfcException("Unexpected token while parsing entity instance"); } tokens->Next(); - load(data.id(), data.attributes(), data.getArgumentCount()); +} + +void IfcParse::IfcFile::try_read_semicolon() { unsigned int old_offset = tokens->stream->Tell(); Token semilocon = tokens->Next(); if (!TokenFunc::isOperator(semilocon, ';')) { @@ -984,15 +1001,26 @@ unsigned IfcEntityInstanceData::set_id(boost::optional i) { // Returns the entities of Entity type that have this entity in their ArgumentList // IfcEntityList::ptr IfcEntityInstanceData::getInverse(const IfcParse::declaration* type, int attribute_index) const { + static std::mutex m; + std::lock_guard lk(m); + return file->getInverse(id_, type, attribute_index); } void IfcEntityInstanceData::load() const { + static std::recursive_mutex m; + std::lock_guard lk(m); + // type_ is 0 for header entities which have their size predetermined in code + Argument** tmp_data = nullptr; if (type_ != 0) { - attributes_ = new Argument*[getArgumentCount()]; + tmp_data = new Argument*[getArgumentCount()]; } - file->load(*this); + file->seek_to(*this); + file->load(id(), tmp_data, getArgumentCount()); + file->try_read_semicolon(); + // @todo does this need to be atomic somehow? + attributes_ = tmp_data; } IfcEntityInstanceData::IfcEntityInstanceData(const IfcEntityInstanceData& e) { diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index b05474990e..61803d23cd 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -49,6 +49,17 @@ #include "../ifcparse/IfcSpfStream.h" + /* gcc doesn't know _Thread_local from C11 yet */ +#ifdef __GNUC__ +# define my_thread_local __thread +#elif __STDC_VERSION__ >= 201112L +# define my_thread_local _Thread_local +#elif defined(_MSC_VER) +# define my_thread_local __declspec(thread) +#else +# error Cannot define thread_local +#endif + namespace IfcParse { class IfcFile; @@ -141,12 +152,13 @@ namespace IfcParse { class IFC_PARSE_API IfcSpfLexer { private: IfcCharacterDecoder* decoder; - //storage for temporary string without allocation - mutable std::string _tempString; unsigned int skipWhitespace(); unsigned int skipComment(); public: - std::string &GetTempString() const { return _tempString; } + std::string &GetTempString() const { + static my_thread_local std::string s; + return s; + } IfcSpfStream* stream; IfcFile* file; IfcSpfLexer(IfcSpfStream* s, IfcFile* f); diff --git a/src/ifcparse/IfcSpfStream.h b/src/ifcparse/IfcSpfStream.h index 820b69d275..2172842a9f 100644 --- a/src/ifcparse/IfcSpfStream.h +++ b/src/ifcparse/IfcSpfStream.h @@ -72,6 +72,10 @@ namespace IfcParse { void Seek(unsigned int offset); /// Returns the cursor position unsigned int Tell(); + + bool is_eof_at(unsigned int); + void increment_at(unsigned int&); + char peek_at(unsigned int); }; }