Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-05 23:41:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement streaming scan through file and use in rocksdb serializer and python

Browse Source
This commit is contained in:
Thomas Krijnen
2025-08-25 12:45:10 +02:00
parent 5a597d1a84
commit d2c7c1532c
16 changed files with 1410 additions and 778 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcconvert/IfcConvert.cpp
+15 -3
View File
@@ -266,6 +266,7 @@ int main(int argc, char** argv) {
#ifdef WITH_HDF5
("cache-file", new po::typed_value<path_t, char_t>(&cache_file), "geometry cache file")
#endif
("stream", "Use streaming conversion (currently supported with conversion to RocksDB)")
;
po::options_description ifc_options("IFC options");
@@ -712,16 +713,27 @@ int main(int argc, char** argv) {
else if (output_extension == RDB) {
int exit_code = EXIT_FAILURE;
try {
if (init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) {
if (vmap.count("stream")) {
time_t start, end;
time(&start);
RocksDbSerializer s(ifc_file, IfcUtil::path::to_utf8(output_filename));
RocksDbSerializer s(IfcUtil::path::to_utf8(input_filename), IfcUtil::path::to_utf8(output_filename), true);
Logger::Status("Populating RocksDB Key-Value store...");
s.finalize();
time(&end);
Logger::Status("Done! Conversion took " + format_duration(start, end));
exit_code = EXIT_SUCCESS;
}
} else {
if (init_input_file(IfcUtil::path::to_utf8(input_filename), ifc_file, no_progress || quiet, mmap)) {
time_t start, end;
time(&start);
RocksDbSerializer s(ifc_file, IfcUtil::path::to_utf8(output_filename));
Logger::Status("Populating RocksDB Key-Value store...");
s.finalize();
time(&end);
Logger::Status("Done! Conversion took " + format_duration(start, end));
exit_code = EXIT_SUCCESS;
}
}
} catch (const std::exception& e) {
Logger::Error(e);
}
src/ifcopenshell-python/ifcopenshell/__init__.py
+7
View File
@@ -292,6 +292,13 @@ def guess_format(path: Path) -> Union[str, None]:
return None
def stream2(path: Union[Path, str]):
streamer = ifcopenshell_wrapper.InstanceStreamer(str(path))
while streamer:
if inst := streamer.read_instance_py():
yield inst
version_core = ifcopenshell_wrapper.version()
__version__ = version = "0.0.0"
get_log = ifcopenshell_wrapper.get_log
src/ifcparse/IfcEntityInstanceData.cpp
+1 -1
View File
@@ -74,7 +74,7 @@ namespace {
} else {
std::string str;
if (!array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str).ok()) {
return TypeEncoder::encode_type<boost::blank>() - 'A';
return TypeEncoder::encode_type<Blank>() - 'A';
}
return (size_t) str[0] - 'A';
}
src/ifcparse/IfcEntityInstanceData.h
+10 -4
View File
@@ -414,19 +414,25 @@ class IFC_PARSE_API IfcEntityInstanceData {
{}
IfcEntityInstanceData(IfcEntityInstanceData&& other) noexcept
: storage_(other.storage_)
: storage_(std::exchange(other.storage_, nullptr))
{}
// No copy-constructor anymore because we need the instance for storage model context
// No copy-constructor/-assignment anymore because we need the instance for storage model context
IfcEntityInstanceData(const IfcEntityInstanceData&) = delete;
IfcEntityInstanceData& operator=(const IfcEntityInstanceData&) = delete;
IfcEntityInstanceData& operator=(IfcEntityInstanceData&& other) {
IfcEntityInstanceData& operator=(IfcEntityInstanceData&& other) noexcept {
if (this != &other) {
storage_ = other.storage_;
delete storage_;
storage_ = std::exchange(other.storage_, nullptr);
}
return *this;
}
~IfcEntityInstanceData() {
delete storage_;
}
AttributeValue get_attribute_value(void* storage, const IfcParse::declaration*, std::size_t identity, size_t index) const;
template<typename T>
src/ifcparse/IfcFile.cpp
+111 -15
View File
@@ -3,7 +3,7 @@
IfcParse::parse_context::~parse_context() {
for (auto& t : tokens_) {
boost::apply_visitor([](auto& v) {
std::visit([](auto& v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, parse_context*>) {
delete v;
}
@@ -31,14 +31,14 @@ namespace {
// Specialization when there are multiple types in the variant
template<typename T, typename First, typename... Rest>
struct is_type_in_variant<boost::variant<First, Rest...>, T>
struct is_type_in_variant<std::variant<First, Rest...>, T>
{
static constexpr bool value = std::is_same<T, First>::value || is_type_in_variant<boost::variant<Rest...>, T>::value;
static constexpr bool value = std::is_same<T, First>::value || is_type_in_variant<std::variant<Rest...>, T>::value;
};
// Specialization when there is only one type left in the variant
template<typename T, typename Last>
struct is_type_in_variant<boost::variant<Last>, T>
struct is_type_in_variant<std::variant<Last>, T>
{
static constexpr bool value = std::is_same<T, Last>::value;
};
@@ -107,7 +107,7 @@ namespace {
return;
}
typedef boost::variant<
typedef std::variant<
Blank,
std::vector<int>,
@@ -125,21 +125,21 @@ namespace {
auto append_to_aggregate_storage = [&aggregate_storage](const auto& v) {
if constexpr (is_type_in_variant_v<possible_aggregation_types_t, std::vector<std::decay_t<decltype(v)>>>) {
if (aggregate_storage.which() == 0) {
if (aggregate_storage.index() == 0) {
aggregate_storage = std::vector<std::decay_t<decltype(v)>>{ v };
} else {
if (auto* vec_ptr = boost::get<std::vector<std::decay_t<decltype(v)>>>(&aggregate_storage)) {
if (auto* vec_ptr = std::get_if<std::vector<std::decay_t<decltype(v)>>>(&aggregate_storage)) {
vec_ptr->push_back(v);
} else {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, int>) {
auto* vec_ptr2 = boost::get<std::vector<double>>(&aggregate_storage);
auto* vec_ptr2 = std::get_if<std::vector<double>>(&aggregate_storage);
if (vec_ptr2) {
// double[] + int
vec_ptr2->push_back((double) v);
}
}
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, double>) {
auto* vec_ptr2 = boost::get<std::vector<int>>(&aggregate_storage);
auto* vec_ptr2 = std::get_if<std::vector<int>>(&aggregate_storage);
if (vec_ptr2) {
// int[] -> double[] + double
std::vector<double> ps(vec_ptr2->begin(), vec_ptr2->end());
@@ -149,7 +149,7 @@ namespace {
}
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::vector<int>>) {
auto* vec_ptr2 = boost::get<std::vector<std::vector<double>>>(&aggregate_storage);
auto* vec_ptr2 = std::get_if<std::vector<std::vector<double>>>(&aggregate_storage);
if (vec_ptr2) {
// double[][] + int[]
std::vector<double> vd(v.begin(), v.end());
@@ -157,7 +157,7 @@ namespace {
}
}
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::vector<double>>) {
auto* vec_ptr2 = boost::get<std::vector<std::vector<int>>>(&aggregate_storage);
auto* vec_ptr2 = std::get_if<std::vector<std::vector<int>>>(&aggregate_storage);
if (vec_ptr2) {
// int[][] -> double[][] + double[]
std::vector<std::vector<double>> vvd;
@@ -171,7 +171,7 @@ namespace {
}
// @todo would be cool if we can trace this back to file offset
auto current = boost::apply_visitor([](auto v) {
auto current = std::visit([](auto v) {
if constexpr (!std::is_same_v<decltype(v), Blank>) {
return std::string(typeid(typename decltype(v)::value_type).name());
} else {
@@ -206,7 +206,7 @@ namespace {
};
for (auto& t : p.tokens_) {
boost::apply_visitor([&aggregate_storage, &append_to_aggregate_storage, aggr, instance_id, attribute_id](const auto& v) {
std::visit([&aggregate_storage, &append_to_aggregate_storage, aggr, instance_id, attribute_id](const auto& v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::Token>) {
// @todo get aggregate of enumeration
dispatch_token(instance_id, attribute_id, v, aggr && aggr->type_of_element()->as_named_type() ? aggr->type_of_element()->as_named_type()->declared_type() : nullptr, append_to_aggregate_storage);
@@ -221,7 +221,7 @@ namespace {
}, t);
}
boost::apply_visitor(fn, aggregate_storage);
std::visit(fn, aggregate_storage);
}
}
@@ -274,7 +274,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
auto index = (uint8_t) std::distance(tokens_.begin(), it);
boost::apply_visitor([this, &storage, name, &references_to_resolve, index, param_type](const auto& v) {
std::visit([this, &storage, name, &references_to_resolve, index, param_type](const auto& v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::Token>) {
dispatch_token(name, index, v, param_type && param_type->as_named_type() ? param_type->as_named_type()->declared_type() : nullptr, [this, &storage, name, &references_to_resolve, index](auto v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::reference_or_simple_type>) {
@@ -552,8 +552,104 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
if (line.find("MANIFEST-") == 0) {
return FT_ROCKSDB;
}
return FT_UNKNOWN;
} else {
// @todo just return SPF for now, but ideally this will be augmented with all other options
return FT_IFCSPF;
}
}
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> IfcParse::InstanceStreamer::read_instance() {
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> return_value;
if (header_ && yielded_header_instances_ < 3) {
if (yielded_header_instances_ == 0) {
return_value.emplace(
0,
&header_->file_description()->declaration(),
std::move(header_->file_description()->data())
);
} else if (yielded_header_instances_ == 1) {
return_value.emplace(
0,
&header_->file_name()->declaration(),
std::move(header_->file_name()->data())
);
} else if (yielded_header_instances_ == 2) {
return_value.emplace(
0,
&header_->file_schema()->declaration(),
std::move(header_->file_schema()->data())
);
}
yielded_header_instances_ += 1;
return return_value;
}
unsigned current_id = 0;
while (good_ && !lexer_->stream->eof && !current_id) {
if (token_stream_[0].type == IfcParse::Token_IDENTIFIER &&
token_stream_[1].type == IfcParse::Token_OPERATOR &&
token_stream_[1].value_char == '=' &&
token_stream_[2].type == IfcParse::Token_KEYWORD) {
current_id = (unsigned)TokenFunc::asIdentifier(token_stream_[0]);
const IfcParse::declaration* entity_type;
try {
entity_type = schema_->declaration_by_name(TokenFunc::asStringRef(token_stream_[2]));
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].startPos));
goto advance;
}
if (entity_type->as_entity() == nullptr) {
Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].startPos));
goto advance;
}
parse_context ps;
lexer_->Next();
try {
storage_.load(current_id, entity_type->as_entity(), ps, -1);
} catch (const IfcInvalidTokenException& e) {
good_ = file_open_status::INVALID_SYNTAX;
Logger::Error(e);
break;
}
/// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code?
// Update the status after every 1000 instances parsed
if (((++progress_) % 1000) == 0) {
std::stringstream ss;
ss << "\r#" << current_id;
Logger::Status(ss.str(), false);
}
auto data = ps.construct(current_id, references_to_resolve_, entity_type, boost::none);
return_value.emplace(
(size_t)current_id,
entity_type,
std::move(data)
);
}
advance:
Token next_token;
try {
next_token = lexer_->Next();
} catch (const IfcException& e) {
Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated");
} catch (...) {
Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
}
if (!lexer_->stream->eof && next_token.type == Token_NONE) {
good_ = file_open_status::INVALID_SYNTAX;
break;
}
token_stream_.push_back(next_token);
}
return return_value;
}
src/ifcparse/IfcFile.h
+93 -522
View File
@@ -24,18 +24,14 @@
#include "IfcParse.h"
#include "IfcSchema.h"
#include "IfcSpfHeader.h"
#include "rocksdb_map_adapter.h"
#include "rocksdb_set_view.h"
#include "map_variant.h"
#include "map_transformer.h"
#include "set_to_map_transformer.h"
#include "storage.h"
#include "file_open_status.h"
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/random_access_index.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <boost/unordered_map.hpp>
#include <boost/variant.hpp>
#include <boost/circular_buffer.hpp>
#include <iterator>
#include <map>
@@ -76,520 +72,6 @@ namespace {
namespace IfcParse {
class IFC_PARSE_API file_open_status {
public:
enum file_open_enum {
SUCCESS,
READ_ERROR,
NO_HEADER,
UNSUPPORTED_SCHEMA,
INVALID_SYNTAX
};
private:
file_open_enum error_;
public:
file_open_status(file_open_enum error)
: error_(error) {}
operator file_open_enum() const {
return error_;
}
file_open_enum value() const {
return error_;
}
operator bool() const {
return error_ == SUCCESS;
}
};
struct InstanceReference {
int v;
size_t file_offset;
operator int() const {
return v;
}
};
typedef boost::variant<InstanceReference, IfcUtil::IfcBaseClass*> reference_or_simple_type;
typedef std::list<std::pair<MutableAttributeValue, boost::variant<reference_or_simple_type, std::vector<reference_or_simple_type>, std::vector<std::vector<reference_or_simple_type>>>>> unresolved_references;
struct parse_context {
std::list<
boost::variant<
IfcUtil::IfcBaseClass*,
Token,
parse_context*
>> tokens_;
parse_context() {};
~parse_context();
parse_context(const parse_context&) = delete;
parse_context& operator=(const parse_context&) = delete;
parse_context(parse_context&&) = default;
parse_context& operator=(parse_context&&) = default;
parse_context& push();
void push(Token t);
void push(IfcUtil::IfcBaseClass* inst);
IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size);
};
#include <variant>
#include <iterator>
#include <type_traits>
#include <iostream>
#include <vector>
#include <list>
#ifndef SWIG
template <typename... Iterators>
class variant_iterator {
public:
// The variant type holding one of the underlying iterators.
using variant_type = std::variant<Iterators...>;
// Assuming that all iterator types have the same value_type, difference_type, etc.
using value_type = std::common_type_t<typename std::iterator_traits<Iterators>::value_type...>;
using difference_type = std::common_type_t<typename std::iterator_traits<Iterators>::difference_type...>;
using pointer = value_type*;
using reference = value_type&;
// For simplicity, we use input_iterator_tag; if all underlying iterators support more,
// you could compute the common iterator_category.
using iterator_category = std::input_iterator_tag;
// Default constructor.
variant_iterator() = default;
// Construct from any one of the underlying iterator types.
template <typename Iterator>
variant_iterator(Iterator it) : it_(it) { }
// Dereference operator.
decltype(auto) operator*() const {
return std::visit([](const auto& iter) -> decltype(auto) {
return *iter;
}, it_);
}
// Arrow operator.
decltype(auto) operator->() const {
return std::visit([](const auto& iter) -> decltype(auto) {
return iter.operator->();
}, it_);
}
// Pre-increment operator.
variant_iterator& operator++() {
std::visit([](auto& iter) { ++iter; }, it_);
return *this;
}
// Post-increment operator.
variant_iterator operator++(int) {
variant_iterator temp(*this);
++(*this);
return temp;
}
// Pre-decrement operator.
variant_iterator& operator--() {
std::visit([](auto& iter) { --iter; }, it_);
return *this;
}
// Post-decrement operator.
variant_iterator operator--(int) {
variant_iterator temp(*this);
--(*this);
return temp;
}
// Equality comparison.
friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) {
return lhs.it_ == rhs.it_;
}
// Inequality comparison.
friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) {
return !(lhs == rhs);
}
private:
variant_type it_;
};
#endif
namespace impl {
struct in_memory_file_storage {
IfcParse::IfcSpfLexer* tokens;
IfcParse::IfcSpfStream* stream;
IfcParse::IfcFile* file;
unresolved_references references_to_resolve;
typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_instance_by_name_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> type_instance_by_name_t;
typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_instance_by_guid_t;
typedef std::tuple<int, short, short> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef std::map<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
typedef entity_instance_by_name_t::iterator iterator;
in_memory_file_storage() : tokens(nullptr), stream(nullptr), file(nullptr) {}
in_memory_file_storage(const in_memory_file_storage&) = delete;
in_memory_file_storage(const in_memory_file_storage&&) = delete;
class type_iterator : public entities_by_type_t::const_iterator {
public:
using iterator_category = std::forward_iterator_tag;
using value_type = entities_by_type_t::key_type;
using difference_type = typename entities_by_type_t::const_iterator::difference_type;
using pointer = value_type const*;
using reference = value_type const&;
type_iterator() : entities_by_type_t::const_iterator() {};
type_iterator(const entities_by_type_t::const_iterator& iter)
: entities_by_type_t::const_iterator(iter) {};
entities_by_type_t::key_type const* operator->() const {
return &entities_by_type_t::const_iterator::operator->()->first;
}
entities_by_type_t::key_type const& operator*() const {
return entities_by_type_t::const_iterator::operator*().first;
}
type_iterator& operator++() {
entities_by_type_t::const_iterator::operator++();
return *this;
}
type_iterator operator++(int) {
type_iterator tmp(*this);
operator++();
return tmp;
}
};
static bool guid_map_;
static bool guid_map() { return guid_map_; }
static void guid_map(bool b) { guid_map_ = b; }
entity_instance_by_name_t byid_;
type_instance_by_name_t tbyid_;
entities_by_type_t bytype_excl_;
entities_by_ref_t byref_excl_;
entity_instance_by_guid_t byguid_;
void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
// @todo is this still used
IfcEntityInstanceData read(unsigned int index);
void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
file_open_status good_ = file_open_status::SUCCESS;
IfcUtil::IfcBaseClass* instance_by_id(int id);
void add_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (ty) {
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(new_entity);
}
}
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (ty) {
auto it = bytype_excl_.find(ty);
if (it != bytype_excl_.end()) {
it->second->remove(new_entity);
if (it->second->size() == 0) {
bytype_excl_.erase(ty);
}
}
}
}
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
template <typename T>
T* create();
IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl);
};
class rocks_db_file_storage {
public:
rocksdb::DB* db;
rocksdb::WriteOptions wopts;
rocksdb::ReadOptions ropts;
IfcParse::IfcFile* file;
enum instance_ref {
typedecl_ref,
entityinstance_ref
};
// to make sure that instance pointer are constant during file lifetime
// cache instances because we want stable pointers
// @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side?
typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t;
entity_by_iden_cache_t instance_cache_, type_instance_cache_;
// @todo all these size_ts should probably be uint32_t for consistency with in-mem storage
// lookup id->identity
// typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
// identity_by_id_t byid_;
typedef rocksdb_set_view<size_t> instance_name_view_t;
instance_name_view_t instance_ids_;
typedef set_to_map_transformer<instance_name_view_t, std::function<IfcUtil::IfcBaseClass* (size_t)>> entity_instance_by_name_t;
entity_instance_by_name_t instance_by_name_;
// typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
// storage is now Instance name -> Identity -> Pointer (cached)
// entity_by_id_t byidentity_;
// index in schema to binary serialized ids
typedef rocksdb_map_adapter<size_t, std::string> instance_id_str_by_type_t;
instance_id_str_by_type_t bytype_;
// guid -> id
typedef rocksdb_map_adapter<std::string, size_t> instance_id_by_guid_str_t;
instance_id_by_guid_str_t byguid_internal_;
// guid -> id -> instance
typedef map_transformer<rocksdb_map_adapter<std::string, size_t>, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_instance_by_guid_t;
entity_instance_by_guid_t byguid_;
typedef std::tuple<int, int, int> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef rocksdb_map_adapter<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
entities_by_ref_t byref_excl_;
// @todo naming
rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file);
~rocks_db_file_storage();
bool read_schema(const IfcParse::schema_definition*& schema);
IfcUtil::IfcBaseClass* assert_existance(size_t instanceId, instance_ref r);
// @todo this could be another map_adapter?
/*
class rocksdb_instance_iterator {
private:
rocksdb::Iterator* state_;
rocks_db_file_storage* storage_;
static constexpr char prefix_[] = "i|";
boost::optional<size_t> read_id_() const {
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
char* pEnd;
long result = strtol(sv.data() + 2, &pEnd, 10);
if (*pEnd == '|') {
return (size_t)result;
}
}
return boost::none;
}
public:
rocksdb_instance_iterator()
: state_(nullptr)
, storage_(nullptr)
{}
rocksdb_instance_iterator(rocks_db_file_storage* fs)
: storage_(fs)
{
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
}
}
rocksdb_instance_iterator& operator++() {
if (!state_) {
return *this;
}
auto last_id = read_id_();
while (state_->Valid()) {
state_->Next();
// Stop if we've left the prefix range.
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
break;
}
if (read_id_() != last_id) {
break;
}
}
return *this;
}
rocksdb_instance_iterator operator++(int) {
rocksdb_instance_iterator temp = *this;
++(*this);
return temp;
}
bool operator==(const rocksdb_instance_iterator& other) const {
if (state_ == nullptr && other.state_ == nullptr) {
return true;
} else {
return read_id_() == other.read_id_();
}
}
bool operator!=(const rocksdb_instance_iterator& other) const {
return !(*this == other);
}
IfcUtil::IfcBaseClass* operator*() const;
};
*/
// @todo merge iterators (template?)
class rocksdb_types_iterator {
private:
rocksdb::Iterator* state_;
const rocks_db_file_storage* storage_;
static constexpr char prefix_[] = "t|";
boost::optional<size_t> read_id_() const {
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
char* pEnd;
long result = strtol(sv.data() + 2, &pEnd, 10);
if (*pEnd == '|') {
return (size_t)result;
}
}
return boost::none;
}
public:
using iterator_category = std::forward_iterator_tag;
using value_type = const IfcParse::declaration*;
// @todo ?
using difference_type = ptrdiff_t;
using pointer = value_type const*;
using reference = value_type const&;
rocksdb_types_iterator()
: state_(nullptr)
, storage_(nullptr)
{}
rocksdb_types_iterator(const rocks_db_file_storage* fs)
: storage_(fs)
{
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
}
}
rocksdb_types_iterator& operator++() {
if (!state_) {
return *this;
}
auto last_id = read_id_();
while (state_->Valid()) {
state_->Next();
// Stop if we've left the prefix range.
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
break;
}
if (read_id_() != last_id) {
break;
}
}
return *this;
}
rocksdb_types_iterator operator++(int) {
rocksdb_types_iterator temp = *this;
++(*this);
return temp;
}
bool operator==(const rocksdb_types_iterator& other) const {
if (state_ == nullptr && other.state_ == nullptr) {
return true;
} else {
return read_id_() == other.read_id_();
}
}
bool operator!=(const rocksdb_types_iterator& other) const {
return !(*this == other);
}
value_type const& operator*() const;
value_type const* operator->() const {
return &operator*();
}
};
// @todo rocksdb_instance_iterator?
using const_iterator = entity_instance_by_name_t::iterator;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
// @todo a bit hard as a map because of value_type being an aggregate
void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
IfcUtil::IfcBaseClass* instance_by_id(int id);
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
template <typename T>
T* create();
IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl);
};
}
enum filetype {
FT_IFCSPF,
FT_IFCXML,
@@ -601,6 +83,95 @@ enum filetype {
filetype guess_file_type(const std::string& fn);
class InstanceStreamer {
private:
IfcSpfStream* stream_;
IfcSpfLexer* lexer_;
IfcSpfHeader* header_;
boost::circular_buffer<Token> token_stream_;
const IfcParse::schema_definition* schema_;
const IfcParse::declaration* ifcroot_type_;
IfcParse::impl::in_memory_file_storage storage_;
IfcParse::file_open_status good_ = IfcParse::file_open_status::SUCCESS;
int progress_;
IfcParse::unresolved_references references_to_resolve_;
int yielded_header_instances_ = 0;
public:
operator bool() const {
return good_ && !lexer_->stream->eof;
}
IfcParse::file_open_status status() const {
return good_;
}
const IfcParse::unresolved_references& references() const {
return references_to_resolve_;
}
IfcParse::unresolved_references& references() {
return references_to_resolve_;
}
const IfcParse::impl::in_memory_file_storage::entities_by_ref_t& inverses() const {
return storage_.byref_excl_;
}
IfcParse::impl::in_memory_file_storage::entities_by_ref_t& inverses() {
return storage_.byref_excl_;
}
InstanceStreamer(const std::string& fn)
: stream_(new IfcSpfStream(fn))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, ifcroot_type_(nullptr)
, progress_(0)
{
good_ = file_open_status::NO_HEADER;
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
try {
schema_ = IfcParse::schema_by_name(header_->file_schema()->schema_identifiers().front());
good_ = file_open_status::SUCCESS;
} catch (const IfcParse::IfcException&) {
}
}
storage_.file = nullptr;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
}
InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer)
: stream_(nullptr)
, lexer_(lexer)
, header_(nullptr)
, token_stream_(3, Token{})
, schema_(schema)
, ifcroot_type_(schema->declaration_by_name("IfcRoot"))
, progress_(0)
{
storage_.file = nullptr;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
}
~InstanceStreamer() {
delete stream_;
if (stream_) {
delete lexer_;
}
delete header_;
}
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> read_instance();
};
/// This class provides access to the entity instances in an IFC file
/// The file takes ownership of instances added to this file and deletes them when the file is deleted.
class IFC_PARSE_API IfcFile {
@@ -661,7 +232,7 @@ private:
~IfcFile();
file_open_status good() const { return good_; }
IfcParse::file_open_status good() const { return good_; }
/// Returns the first entity in the range of instances contained in the model,
/// in arbitrary order
src/ifcparse/IfcParse.cpp
+80 -144
View File
@@ -31,7 +31,6 @@
#include <algorithm>
#include <boost/algorithm/string.hpp>
#include <boost/circular_buffer.hpp>
#include <boost/variant.hpp>
#include <boost/math/special_functions/fpclassify.hpp>
#include <ctime>
@@ -263,8 +262,7 @@ void IfcSpfStream::Inc() {
}
}
IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_, IfcParse::IfcFile* file_) {
file = file_;
IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_) {
stream = stream_;
decoder_ = new IfcCharacterDecoder(stream_);
}
@@ -318,14 +316,14 @@ unsigned int IfcSpfLexer::skipComment() const {
Token IfcSpfLexer::Next() {
if (stream->eof) {
return NoneTokenPtr();
return Token{};
}
while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
}
if (stream->eof) {
return NoneTokenPtr();
return Token{};
}
unsigned int pos = stream->Tell();
@@ -369,7 +367,7 @@ Token IfcSpfLexer::Next() {
if (len != 0) {
t = GeneralTokenPtr(this, pos, stream->Tell());
} else {
t = NoneTokenPtr();
t = Token{};
}
// std::wcout << "token: " << pos << " " << TokenFunc::asStringRef(t).c_str() << std::endl;
return t;
@@ -525,7 +523,6 @@ Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end
return token;
}
Token IfcParse::NoneTokenPtr() { return Token(); }
bool TokenFunc::isOperator(const Token& token) {
return token.type == Token_OPERATOR;
@@ -717,11 +714,11 @@ void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name,
if (TokenFunc::isKeyword(next)) {
try {
const auto* decl = file->schema()->declaration_by_name(TokenFunc::asStringRef(next));
const auto* decl = (schema ? schema : file->schema())->declaration_by_name(TokenFunc::asStringRef(next));
parse_context ps;
tokens->Next();
load(0, nullptr, ps, -1);
auto* simple_type_instance = file->schema()->instantiate(decl, ps.construct(-1, references_to_resolve, decl, boost::none));
auto* simple_type_instance = (schema ? schema : file->schema())->instantiate(decl, ps.construct(-1, *references_to_resolve, decl, boost::none));
//@todo decide addEntity(((IfcUtil::IfcBaseClass*)*entity));
context.push(simple_type_instance);
simple_type_instance->file_ = file;
@@ -750,7 +747,7 @@ IfcEntityInstanceData IfcParse::impl::in_memory_file_storage::read(unsigned int
parse_context pc;
tokens->Next();
load(i, ty->as_entity(), pc, -1);
return IfcEntityInstanceData(pc.construct(i, references_to_resolve, ty, boost::none));
return IfcEntityInstanceData(pc.construct(i, *references_to_resolve, ty, boost::none));
}
void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const {
@@ -814,7 +811,12 @@ void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from,
if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) {
std::vector<uint32_t> vals(s.size() / sizeof(uint32_t));
memcpy(vals.data(), s.data(), s.size());
vals.erase(std::find(vals.begin(), vals.end(), (uint32_t)id_from));
auto it = std::find(vals.begin(), vals.end(), (uint32_t)id_from);
if (it != vals.end()) {
vals.erase(it);
} else {
Logger::Error("Unregistering non-existant inverse #" + std::to_string(id_from) + " on instance #" + std::to_string(inst_id) + " at attribute " + std::to_string(attribute_index));
}
s.resize(vals.size() * sizeof(uint32_t));
memcpy(s.data(), vals.data(), s.size());
db->Put(wopts, key, s);
@@ -1348,7 +1350,9 @@ IfcFile::IfcFile(const std::string& path, filetype ty) {
} else {
throw std::runtime_error("Unsupported file format");
}
ifcroot_type_ = schema_->declaration_by_name("IfcRoot");
if (schema_) {
ifcroot_type_ = schema_->declaration_by_name("IfcRoot");
}
}
#endif
@@ -1405,16 +1409,15 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
// number parsing. See comment above on line 41.
init_locale();
tokens = 0;
stream = s;
if (!stream->valid) {
tokens = nullptr;
if (!s->valid) {
// @todo set good on parent file
good_ = file_open_status::READ_ERROR;
return;
}
// @todo file ptr arg removed?
tokens = new IfcSpfLexer(stream, nullptr);
tokens = new IfcSpfLexer(s);
std::vector<std::string> schemas;
@@ -1447,182 +1450,117 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
auto ifcroot_type_ = schema->declaration_by_name("IfcRoot");
boost::circular_buffer<Token> token_stream(3, Token());
InstanceStreamer streamer(schema, tokens);
IfcUtil::IfcBaseClass* instance = nullptr;
unsigned current_id = 0;
int progress = 0;
Logger::Status("Scanning file...");
int paren_stack_depth = 0;
int attribute_index = -1;
while (streamer) {
while (!stream->eof) {
if (token_stream[0].type == IfcParse::Token_IDENTIFIER &&
token_stream[1].type == IfcParse::Token_OPERATOR &&
token_stream[1].value_char == '=' &&
token_stream[2].type == IfcParse::Token_KEYWORD) {
attribute_index = 0;
auto inst = streamer.read_instance();
current_id = (unsigned)TokenFunc::asIdentifier(token_stream[0]);
const IfcParse::declaration* entity_type;
try {
entity_type = schema->declaration_by_name(TokenFunc::asStringRef(token_stream[2]));
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream[2].startPos));
goto advance;
}
if (entity_type->as_entity() == nullptr) {
Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream[2].startPos));
goto advance;
}
parse_context ps;
tokens->Next();
try {
load(current_id, entity_type->as_entity(), ps, -1);
} catch (const IfcInvalidTokenException& e) {
good_ = file_open_status::INVALID_SYNTAX;
Logger::Error(e);
break;
}
instance = schema->instantiate(entity_type, ps.construct(current_id, references_to_resolve, entity_type, boost::none));
instance->file_ = file;
instance->id_ = current_id;
/// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code?
// Update the status after every 1000 instances parsed
if (((++progress) % 1000) == 0) {
std::stringstream ss;
ss << "\r#" << current_id;
Logger::Status(ss.str(), false);
}
if (instance->declaration().is(*ifcroot_type_)) {
try {
// @nb here we know we're using in-memory so 'nullptr, nullptr, 0' is safe
const std::string guid = instance->data().get_attribute_value(nullptr, nullptr, 0, 0);
if (byguid_.find(guid) != byguid_.end()) {
std::stringstream ss;
ss << "Instance encountered with non-unique GlobalId " << guid;
Logger::Message(Logger::LOG_WARNING, ss.str());
}
byguid_[guid] = instance;
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, ex.what());
}
// this has consumed the instance tokens, set stack depth to 0
paren_stack_depth = 0;
attribute_index = -1;
}
const IfcParse::declaration* ty = &instance->declaration();
{
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(instance);
}
if (byid_.find(current_id) != byid_.end()) {
std::stringstream ss;
ss << "Overwriting instance with name #" << current_id;
Logger::Message(Logger::LOG_WARNING, ss.str());
}
// byidentity_[instance->identity()] = instance;
byid_.insert({ current_id, instance });
// @nb cannot assign to byid_;
// byid_[current_id] = instance;
max_id = (std::max)(max_id, current_id);
} else if (token_stream[0].type == IfcParse::Token_IDENTIFIER && (instance != nullptr)) {
register_inverse(current_id, instance->declaration().as_entity(), token_stream[0].value_int, attribute_index);
} else if (token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == '(') {
paren_stack_depth++;
} else if (token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == ')') {
paren_stack_depth--;
if (paren_stack_depth == 0) {
attribute_index = -1;
}
} else if (paren_stack_depth == 1 && token_stream[0].type == IfcParse::Token_OPERATOR && token_stream[0].value_char == ',') {
attribute_index++;
}
advance:
Token next_token;
try {
next_token = tokens->Next();
} catch (const IfcException& e) {
Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated");
} catch (...) {
Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
}
if (!stream->eof && next_token.type == Token_NONE) {
good_ = file_open_status::INVALID_SYNTAX;
if (!inst) {
// No more instances to read
break;
}
auto current_id = std::get<0>(*inst);
auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst)));
instance->file_ = file;
instance->id_ = current_id;
if (instance->declaration().is(*ifcroot_type_)) {
try {
// @nb here we know we're using in-memory so 'nullptr, nullptr, 0' is safe
const std::string guid = instance->data().get_attribute_value(nullptr, nullptr, 0, 0);
if (byguid_.find(guid) != byguid_.end()) {
std::stringstream ss;
ss << "Instance encountered with non-unique GlobalId " << guid;
Logger::Message(Logger::LOG_WARNING, ss.str());
}
byguid_[guid] = instance;
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, ex.what());
}
}
token_stream.push_back(next_token);
const IfcParse::declaration* ty = &instance->declaration();
{
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(instance);
}
if (byid_.find(current_id) != byid_.end()) {
std::stringstream ss;
ss << "Overwriting instance with name #" << current_id;
Logger::Message(Logger::LOG_WARNING, ss.str());
}
// byidentity_[instance->identity()] = instance;
byid_.insert({ current_id, instance });
// @nb cannot assign to byid_;
// byid_[current_id] = instance;
max_id = (std::max)(max_id, (unsigned int) current_id);
}
good_ = streamer.status();
byref_excl_ = streamer.inverses();
Logger::Status("\rDone scanning file ");
delete tokens;
if (good_ != file_open_status::SUCCESS) {
references_to_resolve.clear();
return;
}
for (const auto& p : references_to_resolve) {
for (const auto& p : streamer.references()) {
const auto& ref = p.first.name_;
const auto& refattr = p.first.index_;
if (auto* v = boost::get<reference_or_simple_type>(&p.second)) {
if (auto* name = boost::get<InstanceReference>(v)) {
if (auto* v = std::get_if<reference_or_simple_type>(&p.second)) {
if (auto* name = std::get_if<InstanceReference>(v)) {
auto it = byid_.find(*name);
if (it == byid_.end()) {
Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset));
} else {
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, it->second);
}
} else if (auto* inst = boost::get<IfcUtil::IfcBaseClass*>(v)) {
} else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(v)) {
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst);
}
} else if (auto* v = boost::get<std::vector<reference_or_simple_type>>(&p.second)) {
} else if (auto* v = std::get_if<std::vector<reference_or_simple_type>>(&p.second)) {
aggregate_of_instance::ptr instances(new aggregate_of_instance);
instances->reserve(v->size());
for (const auto& vi : *v) {
if (auto* name = boost::get<InstanceReference>(&vi)) {
if (auto* name = std::get_if<InstanceReference>(&vi)) {
auto it = byid_.find(*name);
if (it == byid_.end()) {
Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset));
} else {
instances->push(it->second);
}
} else if (auto* inst = boost::get<IfcUtil::IfcBaseClass*>(&vi)) {
} else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&vi)) {
instances->push(*inst);
}
}
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances);
} else if (auto* v = boost::get<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
} else if (auto* v = std::get_if<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance);
for (const auto& vi : *v) {
std::vector<IfcUtil::IfcBaseClass*> inner;
for (const auto& vii : vi) {
if (auto* name = boost::get<InstanceReference>(&vii)) {
if (auto* name = std::get_if<InstanceReference>(&vii)) {
auto it = byid_.find(*name);
if (it == byid_.end()) {
Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found at offset " + std::to_string(name->file_offset));
} else {
inner.push_back(it->second);
}
} else if (auto* inst = boost::get<IfcUtil::IfcBaseClass*>(&vii)) {
} else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&vii)) {
inner.push_back(*inst);
}
}
@@ -1633,8 +1571,6 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
}
Logger::Status("Done resolving references");
references_to_resolve.clear();
}
void IfcFile::recalculate_id_counter() {
src/ifcparse/IfcParse.h
+3 -38
View File
@@ -34,6 +34,7 @@
#include "IfcCharacterDecoder.h"
#include "IfcSpfStream.h"
#include "macros.h"
#include "storage.h"
#include <boost/dynamic_bitset.hpp>
#include <boost/shared_ptr.hpp>
@@ -50,41 +51,6 @@
namespace IfcParse {
class IfcFile;
class IfcSpfLexer;
enum TokenType {
Token_NONE,
Token_STRING,
Token_IDENTIFIER,
Token_OPERATOR,
Token_ENUMERATION,
Token_KEYWORD,
Token_INT,
Token_BOOL,
Token_FLOAT,
Token_BINARY
};
struct Token {
IfcSpfLexer* lexer; //TODO: remove it from here
unsigned startPos;
TokenType type;
union {
char value_char; //types: OPERATOR
int value_int; //types: INT, IDENTIFIER
double value_double; //types: FLOAT
};
Token() : lexer(0),
startPos(0),
type(Token_NONE) {}
Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type)
: lexer(_lexer),
startPos(_startPos),
type(_type) {}
};
/// Provides functions to convert Tokens to binary data
/// Tokens are merely offsets to where they can be read in the file
class IFC_PARSE_API TokenFunc {
@@ -142,7 +108,6 @@ class IFC_PARSE_API TokenFunc {
//
Token OperatorTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
Token GeneralTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
Token NoneTokenPtr();
/// A stream of tokens to be read from a IfcSpfStream.
class IFC_PARSE_API IfcSpfLexer {
@@ -157,8 +122,8 @@ class IFC_PARSE_API IfcSpfLexer {
return string;
}
IfcSpfStream* stream;
IfcFile* file;
IfcSpfLexer(IfcSpfStream* stream, IfcFile* file);
// IfcFile* file;
IfcSpfLexer(IfcSpfStream* stream);
Token Next();
~IfcSpfLexer();
void TokenString(unsigned int offset, std::string& result);
src/ifcparse/IfcSpfHeader.cpp
+67 -38
View File
@@ -30,46 +30,40 @@ static const char* const DATA = "DATA";
using namespace IfcParse;
namespace {
IfcEntityInstanceData read_from_spf_file(IfcFile* f, size_t s) {
return std::visit([f, s](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
parse_context pc;
m.tokens->Next();
m.load(-1, nullptr, pc, -1);
return pc.construct(-1, m.references_to_resolve, nullptr, s);
} else {
// std::unreachable();
return IfcEntityInstanceData(in_memory_attribute_storage(10));
}
}, f->storage_);
IfcEntityInstanceData read_from_spf_file(IfcParse::impl::in_memory_file_storage* storage, size_t s) {
if (storage != nullptr) {
parse_context pc;
storage->tokens->Next();
storage->load(-1, nullptr, pc, -1);
return pc.construct(-1, *storage->references_to_resolve, nullptr, s);
} else {
// std::unreachable();
return IfcEntityInstanceData(in_memory_attribute_storage(10));
}
}
} // namespace
void IfcSpfHeader::readSemicolon() {
if (storage_ != nullptr) {
if (!TokenFunc::isOperator(storage_->tokens->Next(), ';')) {
throw IfcException(std::string("Expected ;"));
}
} else {
// std::unreachable();
}
}
void IfcSpfHeader::readSemicolon() {
std::visit([](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
if (!TokenFunc::isOperator(m.tokens->Next(), ';')) {
throw IfcException(std::string("Expected ;"));
}
} else {
// std::unreachable();
}
}, file_->storage_);
}
void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) {
std::visit([this, term, trail](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
if (TokenFunc::asStringRef(m.tokens->Next()) != term) {
throw IfcException(std::string("Expected " + term));
}
if (trail == TRAILING_SEMICOLON) {
readSemicolon();
}
} else {
// std::unreachable();
if (storage_ != nullptr) {
if (TokenFunc::asStringRef(storage_->tokens->Next()) != term) {
throw IfcException(std::string("Expected " + term));
}
}, file_->storage_);
if (trail == TRAILING_SEMICOLON) {
readSemicolon();
}
} else {
// std::unreachable();
}
}
IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file)
@@ -86,15 +80,50 @@ IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file)
file_name_->file_ = file_;
file_schema_ = new Header_section_schema::file_schema({});
file_schema_->file_ = file_;
} else {
storage_ = std::visit([this](auto& m) -> decltype(storage_) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
return &m;
}
return nullptr;
}, file_->storage_);
}
}
IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcSpfLexer* lexer)
{
storage_ = new impl::in_memory_file_storage;
storage_->tokens = lexer;
file_ = nullptr;
// overwritten later in IfcFile::setDefaultHeaderValues() when we know the schema identifier
file_description_ = new Header_section_schema::file_description({}, "");
file_description_->file_ = file_;
file_name_ = new Header_section_schema::file_name("", "", {}, {}, "", "", "");
file_name_->file_ = file_;
file_schema_ = new Header_section_schema::file_schema({});
file_schema_->file_ = file_;
}
IfcParse::IfcSpfHeader::~IfcSpfHeader() {
delete file_schema_;
delete file_name_;
delete file_description_;
}
void IfcParse::IfcSpfHeader::file(IfcParse::IfcFile* file)
{
this->file_ = file;
if (file != nullptr) {
storage_ = std::visit([this](auto& m) -> decltype(storage_) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
return &m;
}
return nullptr;
}, file_->storage_);
}
}
void IfcSpfHeader::read() {
readTerminal(ISO_10303_21, TRAILING_SEMICOLON);
readTerminal(HEADER, TRAILING_SEMICOLON);
@@ -111,19 +140,19 @@ void IfcSpfHeader::read() {
readTerminal(Header_section_schema::file_description::Class().name_uc(), NONE);
delete file_description_;
file_description_ = new Header_section_schema::file_description(read_from_spf_file(file_, Header_section_schema::file_description::Class().attribute_count()));
file_description_ = new Header_section_schema::file_description(read_from_spf_file(storage_, Header_section_schema::file_description::Class().attribute_count()));
file_description_->file_ = file_;
readSemicolon();
readTerminal(Header_section_schema::file_name::Class().name_uc(), NONE);
delete file_name_;
file_name_ = new Header_section_schema::file_name(read_from_spf_file(file_, Header_section_schema::file_name::Class().attribute_count()));
file_name_ = new Header_section_schema::file_name(read_from_spf_file(storage_, Header_section_schema::file_name::Class().attribute_count()));
file_name_->file_ = file_;
readSemicolon();
readTerminal(Header_section_schema::file_schema::Class().name_uc(), NONE);
delete file_schema_;
file_schema_ = new Header_section_schema::file_schema(read_from_spf_file(file_, Header_section_schema::file_schema::Class().attribute_count()));
file_schema_ = new Header_section_schema::file_schema(read_from_spf_file(storage_, Header_section_schema::file_schema::Class().attribute_count()));
file_schema_->file_ = file_;
readSemicolon();
}
src/ifcparse/IfcSpfHeader.h
+5 -1
View File
@@ -23,6 +23,7 @@
#include "ifc_parse_api.h"
#include "IfcEntityInstanceData.h"
#include "Header_section_schema.h"
#include "storage.h"
namespace IfcParse {
@@ -31,6 +32,8 @@ class IfcFile;
class IFC_PARSE_API IfcSpfHeader {
private:
IfcFile* file_;
IfcParse::impl::in_memory_file_storage* storage_ = nullptr;
mutable Header_section_schema::file_description* file_description_;
mutable Header_section_schema::file_name* file_name_;
mutable Header_section_schema::file_schema* file_schema_;
@@ -43,11 +46,12 @@ class IFC_PARSE_API IfcSpfHeader {
public:
explicit IfcSpfHeader(IfcParse::IfcFile* file = nullptr);
explicit IfcSpfHeader(IfcParse::IfcSpfLexer* lexer);
~IfcSpfHeader();
IfcParse::IfcFile* file() { return file_; }
void file(IfcParse::IfcFile* file) { file_ = file; }
void file(IfcParse::IfcFile* file);
void read();
bool tryRead();
src/ifcparse/file_open_status.h
+42
View File
@@ -0,0 +1,42 @@
#ifndef FILE_OPEN_STATUS_H
#define FILE_OPEN_STATUS_H
#include "ifc_parse_api.h"
namespace IfcParse {
class IFC_PARSE_API file_open_status {
public:
enum file_open_enum {
SUCCESS,
READ_ERROR,
NO_HEADER,
UNSUPPORTED_SCHEMA,
INVALID_SYNTAX,
UNKNOWN
};
private:
file_open_enum error_;
public:
file_open_status(file_open_enum error = UNKNOWN)
: error_(error) {
}
operator file_open_enum() const {
return error_;
}
file_open_enum value() const {
return error_;
}
operator bool() const {
return error_ == SUCCESS;
}
};
}
#endif // !FILE_OPEN_STATUS_H
src/ifcparse/parse_ifcxml.cpp
+6 -6
View File
@@ -446,14 +446,14 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
// Create an attribute value from an instance. Potentially NULL in case it is a
// forward reference to an instance not yet encountered.
auto instance_to_attribute = [&state](const boost::variant<std::string, IfcUtil::IfcBaseClass*>& inst_or_ref, size_t attribute_index, IfcUtil::IfcBaseClass*& inst) {
if (inst_or_ref.which() == 0) {
auto instance_to_attribute = [&state](const std::variant<std::string, IfcUtil::IfcBaseClass*>& inst_or_ref, size_t attribute_index, IfcUtil::IfcBaseClass*& inst) {
if (inst_or_ref.index() == 0) {
inst = nullptr;
// This attribute is NULL initially and after parsing the complete
// file populated in a subsequent step.
state->forward_references.push_back(std::make_tuple(inst->as<IfcUtil::IfcBaseEntity>(), attribute_index, boost::get<std::string>(inst_or_ref)));
state->forward_references.push_back(std::make_tuple(inst->as<IfcUtil::IfcBaseEntity>(), attribute_index, std::get<std::string>(inst_or_ref)));
} else {
inst = boost::get<IfcUtil::IfcBaseClass*>(inst_or_ref);
inst = std::get<IfcUtil::IfcBaseClass*>(inst_or_ref);
inst->set_attribute_value(attribute_index, inst);
}
};
@@ -461,7 +461,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
// Create or reference an instance from the file and set attributes based on XML attributes.
auto create_instance = [&state, &attributes](const IfcParse::declaration* decl) {
boost::optional<std::string> id;
boost::variant<std::string, IfcUtil::IfcBaseClass*> rv;
std::variant<std::string, IfcUtil::IfcBaseClass*> rv;
for (auto& pair : attributes) {
if (pair.first == "id" || pair.first == "href" || pair.first == "ref") {
@@ -619,7 +619,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
instance_to_attribute(inst_or_reference, idx, inst);
// @todo
state->stack.back().inst();
state->stack.push_back(stack_node::instance(id, boost::get<IfcUtil::IfcBaseClass*>(inst_or_reference)));
state->stack.push_back(stack_node::instance(id, std::get<IfcUtil::IfcBaseClass*>(inst_or_reference)));
} else if (attribute_type->as_named_type()->declared_type()->as_select_type() != nullptr) {
// Select types cause an additional indirection, so the current stack node is simply repeated
state->stack.push_back(stack_node::select(state->stack.back().inst(), idx));
src/ifcparse/storage.h
+542
View File
@@ -0,0 +1,542 @@
#ifndef STORAGE_H
#define STORAGE_H
#include "rocksdb_map_adapter.h"
#include "rocksdb_set_view.h"
#include "map_variant.h"
#include "map_transformer.h"
#include "set_to_map_transformer.h"
#include "file_open_status.h"
#include <boost/unordered_map.hpp>
#include <variant>
#include <iterator>
#include <type_traits>
#include <iostream>
#include <vector>
#include <list>
#ifndef SWIG
template <typename... Iterators>
class variant_iterator {
public:
// The variant type holding one of the underlying iterators.
using variant_type = std::variant<Iterators...>;
// Assuming that all iterator types have the same value_type, difference_type, etc.
using value_type = std::common_type_t<typename std::iterator_traits<Iterators>::value_type...>;
using difference_type = std::common_type_t<typename std::iterator_traits<Iterators>::difference_type...>;
using pointer = value_type*;
using reference = value_type&;
// For simplicity, we use input_iterator_tag; if all underlying iterators support more,
// you could compute the common iterator_category.
using iterator_category = std::input_iterator_tag;
// Default constructor.
variant_iterator() = default;
// Construct from any one of the underlying iterator types.
template <typename Iterator>
variant_iterator(Iterator it) : it_(it) {}
// Dereference operator.
decltype(auto) operator*() const {
return std::visit([](const auto& iter) -> decltype(auto) {
return *iter;
}, it_);
}
// Arrow operator.
decltype(auto) operator->() const {
return std::visit([](const auto& iter) -> decltype(auto) {
return iter.operator->();
}, it_);
}
// Pre-increment operator.
variant_iterator& operator++() {
std::visit([](auto& iter) { ++iter; }, it_);
return *this;
}
// Post-increment operator.
variant_iterator operator++(int) {
variant_iterator temp(*this);
++(*this);
return temp;
}
// Pre-decrement operator.
variant_iterator& operator--() {
std::visit([](auto& iter) { --iter; }, it_);
return *this;
}
// Post-decrement operator.
variant_iterator operator--(int) {
variant_iterator temp(*this);
--(*this);
return temp;
}
// Equality comparison.
friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) {
return lhs.it_ == rhs.it_;
}
// Inequality comparison.
friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) {
return !(lhs == rhs);
}
private:
variant_type it_;
};
#endif
namespace IfcParse {
struct InstanceReference {
int v;
size_t file_offset;
operator int() const {
return v;
}
};
typedef std::variant<InstanceReference, IfcUtil::IfcBaseClass*> reference_or_simple_type;
typedef std::list<std::pair<MutableAttributeValue, std::variant<reference_or_simple_type, std::vector<reference_or_simple_type>, std::vector<std::vector<reference_or_simple_type>>>>> unresolved_references;
class IfcFile;
class IfcSpfLexer;
class IfcSpfStream;
enum TokenType {
Token_NONE,
Token_STRING,
Token_IDENTIFIER,
Token_OPERATOR,
Token_ENUMERATION,
Token_KEYWORD,
Token_INT,
Token_BOOL,
Token_FLOAT,
Token_BINARY
};
struct Token {
IfcSpfLexer* lexer; //TODO: remove it from here
unsigned startPos;
TokenType type;
union {
char value_char; //types: OPERATOR
int value_int; //types: INT, IDENTIFIER
double value_double; //types: FLOAT
};
Token() : lexer(0),
startPos(0),
type(Token_NONE) {
}
Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type)
: lexer(_lexer),
startPos(_startPos),
type(_type) {
}
};
struct parse_context {
std::list<
std::variant<
IfcUtil::IfcBaseClass*,
Token,
parse_context*
>> tokens_;
parse_context() {};
~parse_context();
parse_context(const parse_context&) = delete;
parse_context& operator=(const parse_context&) = delete;
parse_context(parse_context&&) = default;
parse_context& operator=(parse_context&&) = default;
parse_context& push();
void push(Token t);
void push(IfcUtil::IfcBaseClass* inst);
IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size);
};
namespace impl {
struct in_memory_file_storage {
IfcParse::IfcSpfLexer* tokens;
// IfcParse::IfcSpfStream* stream;
// Either one of these needs to be set
IfcParse::IfcFile* file;
const IfcParse::schema_definition* schema;
unresolved_references* references_to_resolve = nullptr;
typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_instance_by_name_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> type_instance_by_name_t;
typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_instance_by_guid_t;
typedef std::tuple<int, short, short> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef std::map<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
typedef entity_instance_by_name_t::iterator iterator;
in_memory_file_storage() : tokens(nullptr), file(nullptr), schema(nullptr) {}
in_memory_file_storage(const in_memory_file_storage&) = delete;
in_memory_file_storage(const in_memory_file_storage&&) = delete;
class type_iterator : public entities_by_type_t::const_iterator {
public:
using iterator_category = std::forward_iterator_tag;
using value_type = entities_by_type_t::key_type;
using difference_type = typename entities_by_type_t::const_iterator::difference_type;
using pointer = value_type const*;
using reference = value_type const&;
type_iterator() : entities_by_type_t::const_iterator() {};
type_iterator(const entities_by_type_t::const_iterator& iter)
: entities_by_type_t::const_iterator(iter) {};
entities_by_type_t::key_type const* operator->() const {
return &entities_by_type_t::const_iterator::operator->()->first;
}
entities_by_type_t::key_type const& operator*() const {
return entities_by_type_t::const_iterator::operator*().first;
}
type_iterator& operator++() {
entities_by_type_t::const_iterator::operator++();
return *this;
}
type_iterator operator++(int) {
type_iterator tmp(*this);
operator++();
return tmp;
}
};
static bool guid_map_;
static bool guid_map() { return guid_map_; }
static void guid_map(bool b) { guid_map_ = b; }
entity_instance_by_name_t byid_;
type_instance_by_name_t tbyid_;
entities_by_type_t bytype_excl_;
entities_by_ref_t byref_excl_;
entity_instance_by_guid_t byguid_;
void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
// @todo is this still used
IfcEntityInstanceData read(unsigned int index);
void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
file_open_status good_ = file_open_status::SUCCESS;
IfcUtil::IfcBaseClass* instance_by_id(int id);
void add_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (ty) {
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(new_entity);
}
}
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (ty) {
auto it = bytype_excl_.find(ty);
if (it != bytype_excl_.end()) {
it->second->remove(new_entity);
if (it->second->size() == 0) {
bytype_excl_.erase(ty);
}
}
}
}
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
template <typename T>
T* create();
IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl);
};
class rocks_db_file_storage {
public:
rocksdb::DB* db;
rocksdb::WriteOptions wopts;
rocksdb::ReadOptions ropts;
IfcParse::IfcFile* file;
enum instance_ref {
typedecl_ref,
entityinstance_ref
};
// to make sure that instance pointer are constant during file lifetime
// cache instances because we want stable pointers
// @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side?
typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t;
entity_by_iden_cache_t instance_cache_, type_instance_cache_;
// @todo all these size_ts should probably be uint32_t for consistency with in-mem storage
// lookup id->identity
// typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
// identity_by_id_t byid_;
typedef rocksdb_set_view<size_t> instance_name_view_t;
instance_name_view_t instance_ids_;
typedef set_to_map_transformer<instance_name_view_t, std::function<IfcUtil::IfcBaseClass* (size_t)>> entity_instance_by_name_t;
entity_instance_by_name_t instance_by_name_;
// typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
// storage is now Instance name -> Identity -> Pointer (cached)
// entity_by_id_t byidentity_;
// index in schema to binary serialized ids
typedef rocksdb_map_adapter<size_t, std::string> instance_id_str_by_type_t;
instance_id_str_by_type_t bytype_;
// guid -> id
typedef rocksdb_map_adapter<std::string, size_t> instance_id_by_guid_str_t;
instance_id_by_guid_str_t byguid_internal_;
// guid -> id -> instance
typedef map_transformer<rocksdb_map_adapter<std::string, size_t>, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_instance_by_guid_t;
entity_instance_by_guid_t byguid_;
typedef std::tuple<int, int, int> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef rocksdb_map_adapter<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
entities_by_ref_t byref_excl_;
// @todo naming
rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file);
~rocks_db_file_storage();
bool read_schema(const IfcParse::schema_definition*& schema);
IfcUtil::IfcBaseClass* assert_existance(size_t instanceId, instance_ref r);
// @todo this could be another map_adapter?
/*
class rocksdb_instance_iterator {
private:
rocksdb::Iterator* state_;
rocks_db_file_storage* storage_;
static constexpr char prefix_[] = "i|";
boost::optional<size_t> read_id_() const {
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
char* pEnd;
long result = strtol(sv.data() + 2, &pEnd, 10);
if (*pEnd == '|') {
return (size_t)result;
}
}
return boost::none;
}
public:
rocksdb_instance_iterator()
: state_(nullptr)
, storage_(nullptr)
{}
rocksdb_instance_iterator(rocks_db_file_storage* fs)
: storage_(fs)
{
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
}
}
rocksdb_instance_iterator& operator++() {
if (!state_) {
return *this;
}
auto last_id = read_id_();
while (state_->Valid()) {
state_->Next();
// Stop if we've left the prefix range.
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
break;
}
if (read_id_() != last_id) {
break;
}
}
return *this;
}
rocksdb_instance_iterator operator++(int) {
rocksdb_instance_iterator temp = *this;
++(*this);
return temp;
}
bool operator==(const rocksdb_instance_iterator& other) const {
if (state_ == nullptr && other.state_ == nullptr) {
return true;
} else {
return read_id_() == other.read_id_();
}
}
bool operator!=(const rocksdb_instance_iterator& other) const {
return !(*this == other);
}
IfcUtil::IfcBaseClass* operator*() const;
};
*/
// @todo merge iterators (template?)
class rocksdb_types_iterator {
private:
rocksdb::Iterator* state_;
const rocks_db_file_storage* storage_;
static constexpr char prefix_[] = "t|";
boost::optional<size_t> read_id_() const {
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
char* pEnd;
long result = strtol(sv.data() + 2, &pEnd, 10);
if (*pEnd == '|') {
return (size_t)result;
}
}
return boost::none;
}
public:
using iterator_category = std::forward_iterator_tag;
using value_type = const IfcParse::declaration*;
// @todo ?
using difference_type = ptrdiff_t;
using pointer = value_type const*;
using reference = value_type const&;
rocksdb_types_iterator()
: state_(nullptr)
, storage_(nullptr)
{
}
rocksdb_types_iterator(const rocks_db_file_storage* fs)
: storage_(fs)
{
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
}
}
rocksdb_types_iterator& operator++() {
if (!state_) {
return *this;
}
auto last_id = read_id_();
while (state_->Valid()) {
state_->Next();
// Stop if we've left the prefix range.
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
break;
}
if (read_id_() != last_id) {
break;
}
}
return *this;
}
rocksdb_types_iterator operator++(int) {
rocksdb_types_iterator temp = *this;
++(*this);
return temp;
}
bool operator==(const rocksdb_types_iterator& other) const {
if (state_ == nullptr && other.state_ == nullptr) {
return true;
} else {
return read_id_() == other.read_id_();
}
}
bool operator!=(const rocksdb_types_iterator& other) const {
return !(*this == other);
}
value_type const& operator*() const;
value_type const* operator->() const {
return &operator*();
}
};
// @todo rocksdb_instance_iterator?
using const_iterator = entity_instance_by_name_t::iterator;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
// @todo a bit hard as a map because of value_type being an aggregate
void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
IfcUtil::IfcBaseClass* instance_by_id(int id);
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
template <typename T>
T* create();
IfcUtil::IfcBaseClass* create(const IfcParse::declaration* decl);
};
}
}
#endif // STORAGE_H
src/ifcwrap/IfcParseWrapper.i
+205
View File
@@ -34,6 +34,10 @@ private:
%ignore IfcParse::IfcFile::internal_guid_map;
%ignore IfcParse::IfcFile::storage_;
%ignore IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer);
%ignore IfcParse::InstanceStreamer::read_instance;
%ignore in_memory_file_storage;
%ignore rocks_db_file_storage;
@@ -571,6 +575,7 @@ static IfcUtil::ArgumentType helper_fn_attribute_type(const IfcUtil::IfcBaseClas
%include "../ifcparse/ifc_parse_api.h"
%include "../ifcparse/IfcSpfHeader.h"
%include "../ifcparse/IfcFile.h"
%include "../ifcparse/file_open_status.h"
%include "../ifcparse/IfcBaseClass.h"
%include "../ifcparse/IfcSchema.h"
@@ -866,3 +871,203 @@ static IfcUtil::ArgumentType helper_fn_attribute_type(const IfcUtil::IfcBaseClas
}
%}
%extend IfcParse::InstanceStreamer {
PyObject* read_instance_py() {
auto simply_type_to_dictionary = [&](IfcUtil::IfcBaseClass* t) -> PyObject* {
const auto& nm = t->declaration().name();
auto ifc_val = t->get_attribute_value(0);
auto attribute_val_py = ifc_val.apply_visitor([&](const auto& t) {
using U = std::decay_t<decltype(t)>;
if constexpr (is_std_vector_v<U>) {
return pythonize_vector(t);
} else if constexpr (std::is_same_v<U, EnumerationReference>) {
return pythonize(std::string(t.value()));
} else if constexpr (std::is_same_v<U, Derived>) {
if (feature_use_attribute_value_derived) {
return SWIG_NewPointerObj(new attribute_value_derived, SWIGTYPE_p_attribute_value_derived, SWIG_POINTER_OWN);
} else {
Py_INCREF(Py_None);
return static_cast<PyObject*>(Py_None);
}
} else if constexpr (std::is_same_v<U, aggregate_of_instance::ptr>) {
// cannot occur in streaming mode
Py_INCREF(Py_None);
return static_cast<PyObject*>(Py_None);
} else if constexpr (std::is_same_v<U, aggregate_of_aggregate_of_instance::ptr>) {
// cannot occur in streaming mode
Py_INCREF(Py_None);
return static_cast<PyObject*>(Py_None);
} else if constexpr (std::is_same_v<U, empty_aggregate_t> || std::is_same_v<U, empty_aggregate_of_aggregate_t> || std::is_same_v<U, Blank>) {
Py_INCREF(Py_None);
return static_cast<PyObject*>(Py_None);
} else {
return pythonize(t);
}
});
PyObject* val = PyDict_New();
{
const std::string& key_cpp = "type";
auto name_py = pythonize(key_cpp);
auto value_py = pythonize(nm);
PyDict_SetItem(val, name_py, value_py);
Py_DECREF(name_py);
Py_DECREF(value_py);
}
{
const std::string& key_cpp = "value";
auto name_py = pythonize(key_cpp);
PyDict_SetItem(val, name_py, attribute_val_py);
Py_DECREF(name_py);
Py_DECREF(attribute_val_py);
}
return val;
};
auto instance_reference_to_dict = [&](int i) -> PyObject* {
PyObject* val = PyDict_New();
const std::string& key_cpp = "ref";
auto name_py = pythonize(key_cpp);
auto value_py = pythonize(i);
PyDict_SetItem(val, name_py, value_py);
Py_DECREF(name_py);
Py_DECREF(value_py);
return val;
};
if (!*self) {
Py_INCREF(Py_None);
return Py_None;
}
auto inst = self->read_instance();
if (!inst) {
Py_INCREF(Py_None);
return Py_None;
}
PyObject* d = PyDict_New();
{
const std::string& key_cpp = "id";
auto name_py = pythonize(key_cpp);
auto value_py = pythonize((int) std::get<0>(*inst));
PyDict_SetItem(d, name_py, value_py);
Py_DECREF(name_py);
Py_DECREF(value_py);
}
{
const std::string& key_cpp = "type";
auto name_py = pythonize(key_cpp);
auto value_py = pythonize(std::get<1>(*inst)->name());
PyDict_SetItem(d, name_py, value_py);
Py_DECREF(name_py);
Py_DECREF(value_py);
}
{
const auto* decl = std::get<1>(*inst);
const auto& data = std::get<2>(*inst);
for (size_t i = 0; i < decl->as_entity()->attribute_count(); i++) {
auto val = data.get_attribute_value(nullptr, decl, 0, i);
// sets dict member, returns void
val.apply_visitor([&](const auto& t) -> void {
using T = std::decay_t<decltype(t)>;
PyObject* attribute_val_py;
if constexpr (std::is_same_v<T, IfcUtil::IfcBaseClass*>) {
attribute_val_py = simply_type_to_dictionary(t);
} else {
using U = std::decay_t<decltype(t)>;
if constexpr (is_std_vector_v<U>) {
attribute_val_py = pythonize_vector(t);
} else if constexpr (std::is_same_v<U, EnumerationReference>) {
attribute_val_py = pythonize(std::string(t.value()));
} else if constexpr (std::is_same_v<U, Derived>) {
if (feature_use_attribute_value_derived) {
attribute_val_py = SWIG_NewPointerObj(new attribute_value_derived, SWIGTYPE_p_attribute_value_derived, SWIG_POINTER_OWN);
} else {
Py_INCREF(Py_None);
attribute_val_py = static_cast<PyObject*>(Py_None);
}
} else if constexpr (std::is_same_v<U, aggregate_of_instance::ptr>) {
// cannot occur in streaming mode
Py_INCREF(Py_None);
attribute_val_py = static_cast<PyObject*>(Py_None);
} else if constexpr (std::is_same_v<U, aggregate_of_aggregate_of_instance::ptr>) {
// cannot occur in streaming mode
Py_INCREF(Py_None);
attribute_val_py = static_cast<PyObject*>(Py_None);
} else if constexpr (std::is_same_v<U, empty_aggregate_t> || std::is_same_v<U, empty_aggregate_of_aggregate_t> || std::is_same_v<U, Blank>) {
Py_INCREF(Py_None);
attribute_val_py = static_cast<PyObject*>(Py_None);
} else {
attribute_val_py = pythonize(t);
}
}
{
auto name_py = pythonize(decl->as_entity()->attribute_by_index(i)->name());
PyDict_SetItem(d, name_py, attribute_val_py);
Py_DECREF(name_py);
Py_DECREF(attribute_val_py);
}
});
}
for (auto& p : $self->references()) {
int index = p.first.index_;
auto name_py = pythonize(decl->as_entity()->attribute_by_index(index)->name());
std::visit([&](const auto& v) -> void {
PyObject* attribute_val_py;
using T = std::decay_t<decltype(v)>;
if constexpr (std::is_same_v<T, IfcParse::reference_or_simple_type>) {
if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&v)) {
// So this never happens?
} else if (auto* name = std::get_if<IfcParse::InstanceReference>(&v)) {
attribute_val_py = instance_reference_to_dict(*name);
}
} else if constexpr (std::is_same_v<T, std::vector<IfcParse::reference_or_simple_type>>) {
attribute_val_py = PyTuple_New(v.size());
size_t idx = 0;
for (auto const& inner : v) {
if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&inner)) {
PyTuple_SetItem(attribute_val_py, idx++, simply_type_to_dictionary(*inst));
} else if (auto* name = std::get_if<IfcParse::InstanceReference>(&inner)) {
PyTuple_SetItem(attribute_val_py, idx++, instance_reference_to_dict(*name));
}
}
} else if constexpr (std::is_same_v<T, std::vector<std::vector<IfcParse::reference_or_simple_type>>>) {
attribute_val_py = PyTuple_New(v.size());
size_t outer_idx = 0;
for (auto const& inner : v) {
PyObject* inner_py = PyTuple_New(inner.size());
size_t idx = 0;
for (auto const& innermost : inner) {
if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&innermost)) {
PyTuple_SetItem(inner_py, idx++, simply_type_to_dictionary(*inst));
} else if (auto* name = std::get_if<IfcParse::InstanceReference>(&innermost)) {
PyTuple_SetItem(inner_py, idx++, instance_reference_to_dict(*name));
}
}
PyTuple_SetItem(attribute_val_py, outer_idx++, inner_py);
}
}
PyDict_SetItem(d, name_py, attribute_val_py);
Py_DECREF(name_py);
Py_DECREF(attribute_val_py);
}, p.second);
}
}
$self->references().clear();
$self->inverses().clear();
return d;
}
}
src/serializers/RocksDbSerializer.cpp
+219 -5
View File
@@ -14,7 +14,8 @@ RocksDbSerializer::RocksDbSerializer(IfcParse::IfcFile* file, const std::string&
options.create_if_missing = true;
options.merge_operator.reset(new ConcatenateIdMergeOperator());
rocksdb::Status status = rocksdb::DB::Open(options, rocksdb_filename, &db_);*/
output_file_ = new IfcParse::IfcFile(file_->schema(), IfcParse::FT_ROCKSDB, rocksdb_filename_);
output_file_ = new IfcParse::IfcFile(file->schema(), IfcParse::FT_ROCKSDB, rocksdb_filename_);
// We promise never to add the same instance twice
output_file_->check_existance_before_adding = false;
@@ -22,8 +23,212 @@ RocksDbSerializer::RocksDbSerializer(IfcParse::IfcFile* file, const std::string&
output_file_->calculate_unit_factors = false;
}
void RocksDbSerializer::finalize()
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream)
: file_(input_filename)
, rocksdb_filename_(rocksdb_filename)
{
}
namespace {
// @nb copied from IfcEntityInstanceData.cpp but operating on unresolved instances
bool serialize(std::string& val, const IfcParse::reference_or_simple_type& t)
{
auto s = sizeof(size_t);
val.resize(s + 2);
val[0] = TypeEncoder::encode_type<IfcUtil::IfcBaseClass*>();
// 1 = entity - stored by id (entity name)
// 2 = type - stored by identity (internal counter in class)
val[1] = t.index() == 0 ? 'i' : 't';
size_t iden;
if (auto* name = std::get_if<IfcParse::InstanceReference>(&t)) {
iden = *name;
} else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&t)) {
iden = (*inst)->identity();
}
memcpy(val.data() + 2, &iden, s);
return true;
}
bool serialize(std::string& val, const std::vector<IfcParse::reference_or_simple_type>& t)
{
// no attempt at alignment
val.resize(t.size() * (sizeof(size_t) + 1) + 1);
val[0] = TypeEncoder::encode_type<aggregate_of_instance::ptr>();
char* ptr = val.data() + 1;
for (auto it = t.begin(); it != t.end(); ++it) {
*ptr = it->index() == 0 ? 'i' : 't';
ptr++;
size_t iden;
if (auto* name = std::get_if<IfcParse::InstanceReference>(&*it)) {
iden = *name;
} else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&*it)) {
iden = (*inst)->identity();
}
memcpy(ptr, &iden, sizeof(size_t));
ptr += sizeof(size_t);
}
return true;
}
bool serialize(std::string& val, const std::vector<std::vector<IfcParse::reference_or_simple_type>>& t)
{
// @todo
return false;
}
}
namespace {
template <typename T>
std::string to_string_fixed_width(const T& t, size_t w) {
// @todo currently inactive
std::ostringstream oss;
oss << /*std::setfill('0') << std::setw(w) <<*/ t;
return oss.str();
}
}
void RocksDbSerializer::write_streaming_() {
const auto& input_filename = std::get<std::string>(file_);
IfcParse::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr);
std::string tmp;
IfcParse::InstanceStreamer streamer(input_filename);
while (streamer) {
auto inst = streamer.read_instance();
if (inst) {
// name can be zero in case of header instances
auto name = std::get<0>(*inst);
const auto* decl = std::get<1>(*inst);
const auto& data = std::get<2>(*inst);
const bool is_header = decl->schema() == &Header_section_schema::get_schema();
std::vector<IfcUtil::IfcBaseClass*> simple_type_instances;
for (size_t i = 0; i < decl->as_entity()->attribute_count(); i++) {
auto val = data.get_attribute_value(nullptr, decl, 0, i);
val.apply_visitor([&](const auto& t) {
using T = std::decay_t<decltype(t)>;
if constexpr (std::is_same_v<T, IfcUtil::IfcBaseClass*>) {
// instance is per definition a simple type here, because instance
// references are not resolved yet, but provided in vector of
// references
simple_type_instances.push_back(t);
}
rocks_db_attribute_storage{}.set(&storage, decl, name, i, t);
});
}
for (auto& p : streamer.references()) {
// @nb cast to int in order not be interpreted as a char when appending to string
int index = p.first.index_;
std::visit([&](const auto& v) {
serialize(tmp, v);
using T = std::decay_t<decltype(v)>;
if constexpr (std::is_same_v<T, IfcParse::reference_or_simple_type>) {
if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&v)) {
// So this never happens?
simple_type_instances.push_back(*inst);
}
} else if constexpr (std::is_same_v<T, std::vector<IfcParse::reference_or_simple_type>>) {
for (auto const& inner : v) {
if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&inner)) {
simple_type_instances.push_back(*inst);
}
}
} else if constexpr (std::is_same_v<T, std::vector<std::vector<IfcParse::reference_or_simple_type>>>) {
for (auto const& inner : v) {
for (auto const& innermost : inner) {
if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(&innermost)) {
simple_type_instances.push_back(*inst);
}
}
}
}
}, p.second);
storage.db->Put(
storage.wopts,
(is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) +
(is_header ? decl->name() : std::to_string(p.first.name_)) + "|" +
std::to_string(index), tmp);
auto write_inverse = [&](const IfcParse::reference_or_simple_type& v) {
if (auto* ref = std::get_if<IfcParse::InstanceReference>(&v)) {
auto key = "v|" + to_string_fixed_width(*ref, 10) + "|" + to_string_fixed_width(decl->index_in_schema(), 4) + "|" + to_string_fixed_width(index, 2);
static std::string s;
uint32_t vv = name;
s.resize(sizeof(uint32_t));
memcpy(s.data(), &vv, sizeof(uint32_t));
storage.db->Merge(
storage.wopts, key, s);
}
};
std::visit([&](auto const& val) {
using T = std::decay_t<decltype(val)>;
if constexpr (std::is_same_v<T, IfcParse::reference_or_simple_type>) {
write_inverse(val);
} else if constexpr (std::is_same_v<T, std::vector<IfcParse::reference_or_simple_type>>) {
std::for_each(val.begin(), val.end(), write_inverse);
} else if constexpr (std::is_same_v<T, std::vector<std::vector<IfcParse::reference_or_simple_type>>>) {
for (auto const& inner : val) {
std::for_each(inner.begin(), inner.end(), write_inverse);
}
}
}, p.second);
}
for (const auto* inst : simple_type_instances) {
std::string s(sizeof(size_t), ' ');
size_t v = inst->declaration().index_in_schema();
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Put(
storage.wopts,
(inst->declaration().as_entity() ? "i|" : "t|") + std::to_string(inst->identity()) + "|_", s);
auto val = inst->get_attribute_value(0);
val.apply_visitor([&](const auto& t) {
rocks_db_attribute_storage{}.set(&storage, &inst->declaration(), inst->identity(), 0, t);
});
// @nb we also need to delete them
delete inst;
}
// Entity type as numeric ref to index_in_schema
if (!is_header) {
std::string s(sizeof(size_t), ' ');
size_t v = decl->index_in_schema();
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Put(
storage.wopts,
(decl->as_entity() ? "i|" : "t|") + std::to_string(name) + "|_", s);
{
size_t v = name;
std::string s(sizeof(size_t), ' ');
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Merge(storage.wopts, "t|" + std::to_string(decl->index_in_schema()), s);
}
}
streamer.references().clear();
streamer.inverses().clear();
}
}
}
void RocksDbSerializer::write_non_streaming_() {
// Build a map of instances and their references/dependencies
std::map<uint32_t, std::set<uint32_t>> dependencies, dependencies_inv;
std::visit([&dependencies](const auto& m) {
@@ -34,9 +239,9 @@ void RocksDbSerializer::finalize()
}
}
}
}, file_->storage_);
}, std::get<IfcParse::IfcFile*>(file_)->storage_);
// Add bottom-rank nodes, inv mapping does not contain them
for (const auto& p : *file_) {
for (const auto& p : *std::get<IfcParse::IfcFile*>(file_)) {
dependencies[p.first];
}
@@ -80,7 +285,7 @@ void RocksDbSerializer::finalize()
// Add them in topological order, so that add() never recurses into something not previously visited
for (auto& i : deps_topo_order) {
output_file_->addEntity(file_->instance_by_id(i), i);
output_file_->addEntity(std::get<IfcParse::IfcFile*>(file_)->instance_by_id(i), i);
}
// Copy inverses
@@ -102,4 +307,13 @@ void RocksDbSerializer::finalize()
delete output_file_;
}
void RocksDbSerializer::finalize() {
if (file_.index() == 0) {
write_non_streaming_();
} else {
write_streaming_();
}
}
#endif
src/serializers/RocksDbSerializer.h
+4 -1
View File
@@ -10,11 +10,14 @@ class SERIALIZERS_API RocksDbSerializer : public Serializer {
private:
rocksdb::DB* db_;
std::string rocksdb_filename_;
IfcParse::IfcFile* file_;
std::variant<IfcParse::IfcFile*, std::string> file_;
IfcParse::IfcFile* output_file_;
void write_streaming_();
void write_non_streaming_();
public:
RocksDbSerializer(IfcParse::IfcFile* file, const std::string& rocksdb_filename);
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream);
virtual ~RocksDbSerializer() {}