Exploration of using rocksdb as file+instance storage

This commit is contained in:
Thomas Krijnen
2025-02-21 16:12:16 +01:00
parent cdcbc2ad3b
commit da4e86a1d5
16 changed files with 2012 additions and 547 deletions
+4 -2
View File
@@ -117,6 +117,8 @@ public:
void unset_attribute_value(size_t i);
AttributeValue get_attribute_value(size_t index) const;
uint32_t identity() const { return identity_; }
uint32_t id() const { return id_; }
@@ -131,7 +133,7 @@ class IFC_PARSE_API IfcBaseEntity : public IfcBaseClass {
IfcBaseEntity(IfcEntityInstanceData&& data);
IfcBaseEntity(size_t n)
: IfcBaseClass(IfcEntityInstanceData(storage_t(n)))
: IfcBaseClass(IfcEntityInstanceData(in_memory_attribute_storage(n)))
{}
virtual const IfcParse::declaration& declaration() const = 0;
@@ -172,7 +174,7 @@ class IFC_PARSE_API IfcBaseType : public IfcBaseClass {
{}
IfcBaseType()
: IfcBaseClass(IfcEntityInstanceData(storage_t(1)))
: IfcBaseClass(IfcEntityInstanceData(in_memory_attribute_storage(1)))
{}
virtual const IfcParse::declaration& declaration() const = 0;
+223 -22
View File
@@ -1,5 +1,6 @@
#include "IfcEntityInstanceData.h"
#include "IfcBaseClass.h"
#include "IfcFile.h"
// @todo is size() still needed?
class SizeVisitor {
@@ -28,100 +29,300 @@ public:
int operator()(const aggregate_of_aggregate_of_instance::ptr& i) const { return i->size(); }
};
namespace {
// Trait to detect contiguous containers (vector / string)
template <typename T>
struct is_contiguous_container : std::false_type {};
template <typename T, typename Alloc>
struct is_contiguous_container<std::vector<T, Alloc>> : std::true_type {};
template <typename CharT, typename Traits, typename Alloc>
struct is_contiguous_container<std::basic_string<CharT, Traits, Alloc>> : std::true_type {};
template <typename T>
bool serialize(std::string& val, const T& t) {
return false;
}
template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
bool serialize(std::string& val, const T& t) {
auto s = sizeof(typename T::value_type) * t.size();
val.resize(s);
val[0] = TypeEncoder::encode_type<T>();
memcpy(val.data() + 1, t.data(), s);
return true;
}
bool serialize(std::string& val, const IfcUtil::IfcBaseClass* t) {
auto s = sizeof(size_t);
val.resize(s + 2);
val[0] = TypeEncoder::encode_type<IfcUtil::IfcBaseClass*>();
// 1 = entity - stored by id (entity name)
// 2 = type - stored by identity (internal counter in class)
val[1] = t->declaration().as_entity() ? 1 : 2;
size_t iden = t->declaration().as_entity() ? t->id() : t->identity();
memcpy(val.data() + 2, &iden, s);
return true;
}
bool serialize(std::string& val, const EnumerationReference& v) {
auto s = sizeof(size_t);
val.resize(s * 2 + 1);
val[0] = TypeEncoder::encode_type<EnumerationReference>();
size_t vv = v.enumeration()->index_in_schema();
memcpy(val.data() + 1, &vv, sizeof(size_t));
vv = v.index();
memcpy(val.data() + 1, &vv, sizeof(size_t));
return true;
}
bool serialize(std::string& val, aggregate_of_instance::ptr& t) {
std::vector<size_t> ids;
// @nb this has to be identity, because needs to work for typedecls as well
std::transform(t->begin(), t->end(), std::back_inserter(ids), [](auto& x) { return x->identity(); });
return false;
}
bool serialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
return false;
}
/*
template <typename T>
bool deserialize(std::string& val, const T& t) {}
*/
template <typename T, typename std::enable_if<is_contiguous_container<T>::value, int>::type = 0>
bool deserialize(std::string& val, T& t) {
// @todo vector of vector
if (val[0] != TypeEncoder::encode_type<T>()) {
return false;
}
auto s = (val.size() - 1) / sizeof(typename T::value_type);
t.resize(s);
memcpy(t.data(), val.data() + 1, s * sizeof(typename T::value_type));
return true;
}
template <typename T, typename std::enable_if<std::is_integral_v<T> || std::is_floating_point_v<T>, int>::type = 0>
bool deserialize(std::string& val, T& t) {
if (val[0] != TypeEncoder::encode_type<T>()) {
return false;
}
auto s = (val.size() - 1) / sizeof(T);
memcpy(&t, val.data() + 1, sizeof(T));
return true;
}
bool deserialize(std::string& val, boost::logic::tribool& t) {
if (val[0] != TypeEncoder::encode_type<boost::logic::tribool>()) {
return false;
}
if (val[1] == 0) {
t = false;
} else if (val[1] == 1) {
t = true;
} else if (val[1] == 2) {
t = boost::logic::indeterminate;
} else {
return false;
}
}
bool deserialize(std::string& val, boost::dynamic_bitset<>& t) {
if (val[0] != TypeEncoder::encode_type<boost::dynamic_bitset<>>()) {
return false;
}
t = boost::dynamic_bitset<>(val.substr(1));
return true;
}
bool deserialize(std::string& val, aggregate_of_instance::ptr& t) {
return false;
}
bool deserialize(std::string& val, aggregate_of_aggregate_of_instance::ptr& t) {
return false;
}
}
namespace {
template<typename T>
inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
{
if (storage_model_ == 0) {
return array_.storage_ptr->get<T>(index_);
} else {
T val;
if constexpr (
// the following types cannot be directly deserialized from rocksdb, but need to be constructed
!std::is_same_v<T, EnumerationReference> &&
!std::is_same_v<std::remove_cv_t<std::remove_pointer_t<T>>, IfcUtil::IfcBaseClass>)
{
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
deserialize(str, val);
}
return val;
}
}
template<typename T>
inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
{
if (storage_model_ == 0) {
return array_.storage_ptr->has<T>(index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
return str[0] == TypeEncoder::encode_type<T>();
}
}
inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
{
if (storage_model_ == 0) {
return array_.storage_ptr->index(index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
return (size_t) str[0] - 'A';
}
}
}
AttributeValue::operator int() const
{
return array_->get<int>(index_);
return dispatch_get_<int>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator bool() const
{
return array_->get<bool>(index_);
return dispatch_get_<bool>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator double() const
{
return array_->get<double>(index_);
return dispatch_get_<double>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator boost::logic::tribool() const
{
if (array_->has<bool>(index_)) {
return array_->get<bool>(index_);
if (dispatch_has_<bool>(array_, storage_model_, instance_name_, index_)) {
return dispatch_get_<bool>(array_, storage_model_, instance_name_, index_);
}
return array_->get<boost::logic::tribool>(index_);
return dispatch_get_<boost::logic::tribool>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator std::string() const
{
if (array_->has<EnumerationReference>(index_)) {
if (dispatch_has_<EnumerationReference>(array_, storage_model_, instance_name_, index_)) {
// @todo this is silly, but the way things currently work,
// @todo also we don't really need to store a reference to the enumeration type, when this same type is already stored on the definition of the entity and no other value can be provided.
return array_->get<EnumerationReference>(index_).value();
if (storage_model_ == 0) {
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, index_).value();
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = schema_->declarations()[v]->as_enumeration_type();
memcpy(&v, str.data() + 5, sizeof(size_t));
return decl->lookup_enum_value(v);
}
}
return dispatch_get_<std::string>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator EnumerationReference() const
{
if (storage_model_ == 0) {
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = schema_->declarations()[v]->as_enumeration_type();
memcpy(&v, str.data() + 5, sizeof(size_t));
return EnumerationReference(decl, v);
}
return array_->get<std::string>(index_);
}
AttributeValue::operator boost::dynamic_bitset<>() const
{
return array_->get<boost::dynamic_bitset<>>(index_);
return dispatch_get_<boost::dynamic_bitset<>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator IfcUtil::IfcBaseClass* () const
{
return array_->get<IfcUtil::IfcBaseClass*>(index_);
if (storage_model_ == 0) {
return dispatch_get_<IfcUtil::IfcBaseClass*>(array_, storage_model_, instance_name_, index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "a|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = schema_->declarations()[v]->as_enumeration_type();
memcpy(&v, str.data() + 5, sizeof(size_t));
return array_.db_ptr->assert_existance(v);
}
}
AttributeValue::operator std::vector<int>() const
{
return array_->get<std::vector<int>>(index_);
return dispatch_get_<std::vector<int>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator std::vector<double>() const
{
return array_->get<std::vector<double>>(index_);
return dispatch_get_<std::vector<double>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator std::vector<std::string>() const
{
return array_->get<std::vector<std::string>>(index_);
return dispatch_get_<std::vector<std::string>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator std::vector<boost::dynamic_bitset<>>() const
{
return array_->get<std::vector<boost::dynamic_bitset<>>>(index_);
return dispatch_get_<std::vector<boost::dynamic_bitset<>>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator boost::shared_ptr<aggregate_of_instance>() const
{
return array_->get<boost::shared_ptr<aggregate_of_instance>>(index_);
return dispatch_get_<boost::shared_ptr<aggregate_of_instance>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator std::vector<std::vector<int>>() const
{
return array_->get<std::vector<std::vector<int>>>(index_);
return dispatch_get_<std::vector<std::vector<int>>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator std::vector<std::vector<double>>() const
{
return array_->get<std::vector<std::vector<double>>>(index_);
return dispatch_get_<std::vector<std::vector<double>>>(array_, storage_model_, instance_name_, index_);
}
AttributeValue::operator boost::shared_ptr<aggregate_of_aggregate_of_instance>() const
{
return array_->get<boost::shared_ptr<aggregate_of_aggregate_of_instance>>(index_);
return dispatch_get_<boost::shared_ptr<aggregate_of_aggregate_of_instance>>(array_, storage_model_, instance_name_, index_);
}
bool AttributeValue::isNull() const
{
return array_->has<Blank>(index_);
return dispatch_has_<Blank>(array_, storage_model_, instance_name_, index_);
}
unsigned int AttributeValue::size() const
{
return array_->apply_visitor(SizeVisitor{}, index_);
// @todo
return array_.storage_ptr->apply_visitor(SizeVisitor{}, index_);
}
IfcUtil::ArgumentType AttributeValue::type() const
{
return static_cast<IfcUtil::ArgumentType>(array_->index(index_));
return static_cast<IfcUtil::ArgumentType>(dispatch_index_(array_, storage_model_, instance_name_, index_));
}
+136 -18
View File
@@ -25,6 +25,13 @@
#include "aggregate_of_instance.h"
#include "IfcSchema.h"
#pragma push_macro("Handle")
#undef Handle
#include <rocksdb/db.h>
#pragma pop_macro("Handle")
#include <boost/optional.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/logic/tribool.hpp>
@@ -36,7 +43,7 @@ private:
size_t index_;
public:
EnumerationReference(const IfcParse::enumeration_type* enumeration, size_t index)
EnumerationReference(const IfcParse::enumeration_type* enumeration = nullptr, size_t index = 0)
: enumeration_(enumeration)
, index_(index)
{}
@@ -58,7 +65,12 @@ class Derived {};
class empty_aggregate_t {};
class empty_aggregate_of_aggregate_t {};
typedef VariantArray <
template<typename... Args>
struct parameter_pack {
static constexpr size_t size = sizeof...(Args);
};
typedef parameter_pack <
// A null argument, it will always serialize to $
Blank,
// @todo Derived is not really necessary anymore, just serialize correctly based on schema
@@ -113,26 +125,75 @@ typedef VariantArray <
std::vector<std::vector<double>>,
// An aggregate of an aggregate of entities. E.g. ((#1, #2), (#3))
aggregate_of_aggregate_of_instance::ptr
> storage_t;
> type_variant_parameter_pack;
template<typename Pack>
struct pack_to_variant_array;
template<typename... Args>
struct pack_to_variant_array<parameter_pack<Args...>> {
using type = VariantArray<Args...>;
};
using in_memory_attribute_storage = pack_to_variant_array<type_variant_parameter_pack>::type;
template <typename Pack>
struct TypeEncoder_t;
template <typename... Types>
struct TypeEncoder_t<parameter_pack<Types...>> {
template <typename U>
static char encode_type() {
return 'A' + ::impl::TypeIndex_v<U, Types...>;
}
};
using TypeEncoder = TypeEncoder_t<type_variant_parameter_pack>;
struct MutableAttributeValue {
int name_;
uint8_t index_;
};
namespace IfcParse {
namespace impl {
class rocks_db_file_storage;
}
}
// short lived
struct AttributeValue {
const storage_t* array_;
uint8_t index_;
uint8_t storage_model_ = 0;
size_t instance_name_;
// @todo couple with db_ptr;
IfcParse::schema_definition* schema_;
union pointer_type {
const in_memory_attribute_storage* storage_ptr;
IfcParse::impl::rocks_db_file_storage* db_ptr;
pointer_type(IfcParse::impl::rocks_db_file_storage* db) : db_ptr(db) {}
pointer_type(const in_memory_attribute_storage* ims) : storage_ptr(ims) {}
};
pointer_type array_;
AttributeValue()
: array_(nullptr)
, index_(0)
: index_(0)
, array_((const in_memory_attribute_storage*)nullptr)
, storage_model_(0)
{}
AttributeValue(const storage_t* arr, uint8_t index)
: array_(arr)
, index_(index)
AttributeValue(const in_memory_attribute_storage* arr, uint8_t index)
: index_(index)
, array_(arr)
, storage_model_(0)
{}
AttributeValue(IfcParse::schema_definition* schema, IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t index)
: index_(index)
, array_(db)
, storage_model_(1)
, instance_name_(instance_name)
, schema_(schema)
{}
operator int() const;
@@ -153,17 +214,60 @@ struct AttributeValue {
operator std::vector<std::vector<double>>() const;
operator boost::shared_ptr<aggregate_of_aggregate_of_instance>() const;
operator EnumerationReference() const;
bool isNull() const;
unsigned int size() const;
IfcUtil::ArgumentType type() const;
};
struct rocks_db_attribute_storage {
private:
IfcParse::impl::rocks_db_file_storage* fs;
template<typename Visitor, std::size_t Index>
auto apply_visitor_impl(Visitor&& visitor, std::size_t idx, std::integral_constant<std::size_t, Index>) const {
return apply_visitor_impl(std::forward<Visitor>(visitor), idx, std::integral_constant<std::size_t, Index - 1>{});
}
template<typename Visitor>
void apply_visitor_impl(Visitor&&, std::size_t, std::integral_constant<std::size_t, 0>) const {
throw std::runtime_error("Invalid variant index");
}
public:
size_t size() const {
// @todo
return 8;
}
template<typename T>
void set(std::size_t index, T&& value) {
// @todo
}
template<typename T>
bool has(std::size_t index) const {
// @todo
return false;
}
template<typename Visitor>
auto apply_visitor(Visitor&& visitor, std::size_t index) const {
return apply_visitor_impl(std::forward<Visitor>(visitor), index, std::integral_constant<std::size_t, type_variant_parameter_pack::size>{});
}
};
class IFC_PARSE_API IfcEntityInstanceData {
public:
storage_t storage_;
std::variant<in_memory_attribute_storage, rocks_db_attribute_storage> storage_;
IfcEntityInstanceData(storage_t&& storage)
IfcEntityInstanceData(in_memory_attribute_storage&& storage)
: storage_(std::move(storage))
{}
IfcEntityInstanceData(rocks_db_attribute_storage&& storage)
: storage_(std::move(storage))
{}
@@ -182,15 +286,29 @@ class IFC_PARSE_API IfcEntityInstanceData {
AttributeValue get_attribute_value(size_t index) const;
/*
template <typename T>
void set_attribute_value(size_t index, const T& t);
template<typename T>
void set_attribute_value(std::size_t index, T&& value) {
std::visit([&index, &value](auto& x) {
return x.set(index, value);
}, storage_);
}
void set_attribute_value(size_t index, AttributeValue&, IfcUtil::ArgumentType attr_type = IfcUtil::Argument_UNKNOWN);
*/
template<typename T>
bool has_attribute_value(std::size_t index) const {
return std::visit([&index](const auto& x) {
return x.has<T>(index);
}, storage_);
}
template<typename Visitor>
auto apply_visitor(Visitor&& visitor, std::size_t index) const {
return std::visit([&index, &visitor](const auto& x) {
return x.apply_visitor(std::forward<Visitor>(visitor), index);
}, storage_);
}
size_t size() const {
return storage_.size();
return std::visit([](const auto& x) { return x.size(); }, storage_);
}
void toString(std::ostream&, bool upper = false, const IfcParse::entity* ent = nullptr) const;
+95 -2
View File
@@ -259,10 +259,10 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
}
if (tokens_.empty()) {
return IfcEntityInstanceData(storage_t(0));
return IfcEntityInstanceData(in_memory_attribute_storage(0));
}
storage_t storage(decl != nullptr
in_memory_attribute_storage storage(decl != nullptr
? (std::min)(parameter_types.size(), tokens_.size())
: tokens_.size()
);
@@ -328,3 +328,96 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
return IfcEntityInstanceData(std::move(storage));
}
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_iterator::operator*() const {
auto it = storage_->byid_.find(*read_id_());
if (it != storage_->byid_.end()) {
// @todo define an implicit std::to_string() in all map adapters with leading 0s
auto jt = storage_->instance_cache_.find(it->second);
if (jt != storage_->instance_cache_.end()) {
return jt->second;
} else {
return storage_->assert_existance(it->second);
}
}
}
const IfcParse::declaration* IfcParse::impl::rocks_db_file_storage::rocksdb_types_iterator::operator*() const {
return storage_->file->schema()->declarations()[*read_id_()];
}
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(size_t instanceId) {
std::string v;
rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instanceId) + "|t", &v);
if (s.ok()) {
size_t s;
memcpy(&s, v.data(), sizeof(size_t));
auto decl = file->schema()->declarations()[s];
IfcEntityInstanceData data(rocks_db_attribute_storage{});
auto inst = file->schema()->instantiate(decl, std::move(data));
inst->id_ = instanceId;
instance_cache_.insert({ inst->identity(), inst });
byid_.insert({ inst->id(), inst->identity() });
return inst;
}
throw std::runtime_error("");
}
#include "rocksdb/merge_operator.h"
namespace {
class ConcatenateIdMergeOperator : public rocksdb::AssociativeMergeOperator {
public:
virtual bool Merge(const rocksdb::Slice& key,
const rocksdb::Slice* existing_value,
const rocksdb::Slice& value,
std::string* new_value,
rocksdb::Logger* logger) const override {
if (existing_value) {
new_value->assign(existing_value->data(), existing_value->size());
new_value->append(value.data(), value.size());
} else {
new_value->assign(value.data(), value.size());
}
return true;
}
virtual const char* Name() const override {
return "ConcatenateIdMergeOperator";
}
};
}
// @todo naming
IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile)
: file(ffile)
// @todo db is not initialized here yet
, byguid_internal_(db, "g|")
, byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
, byid_(db, "d|")
, bytype_(db, "t|")
{
rocksdb::Options options;
options.create_if_missing = true;
options.merge_operator.reset(new ConcatenateIdMergeOperator());
rocksdb::Status status = rocksdb::DB::Open(options, filepath, &db);
}
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int id)
{
// @todo rename assert_existance() -> instance_by_id();
return assert_existance(id);
}
IfcUtil::IfcBaseClass* IfcParse::impl::in_memory_file_storage::instance_by_id(int id)
{
auto it = byid_.find(id);
if (it == byid_.end()) {
throw IfcException("Instance #" + boost::lexical_cast<std::string>(id) + " not found");
}
return it->second;
}
+486 -101
View File
@@ -24,6 +24,9 @@
#include "IfcParse.h"
#include "IfcSchema.h"
#include "IfcSpfHeader.h"
#include "rocksdb_map_adapter.h"
#include "map_variant.h"
#include "map_transformer.h"
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/random_access_index.hpp>
@@ -95,94 +98,469 @@ struct parse_context {
IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size);
};
/// This class provides several static convenience functions and variables
/// and provide access to the entities in an IFC file
#include <variant>
#include <iterator>
#include <type_traits>
#include <iostream>
#include <vector>
#include <list>
template <typename... Iterators>
class variant_iterator {
public:
// The variant type holding one of the underlying iterators.
using variant_type = std::variant<Iterators...>;
// Assuming that all iterator types have the same value_type, difference_type, etc.
using value_type = std::common_type_t<typename std::iterator_traits<Iterators>::value_type...>;
using difference_type = std::common_type_t<typename std::iterator_traits<Iterators>::difference_type...>;
using pointer = value_type*;
using reference = value_type&;
// For simplicity, we use input_iterator_tag; if all underlying iterators support more,
// you could compute the common iterator_category.
using iterator_category = std::input_iterator_tag;
// Default constructor.
variant_iterator() = default;
// Construct from any one of the underlying iterator types.
template <typename Iterator>
variant_iterator(Iterator it) : it_(it) { }
// Dereference operator.
decltype(auto) operator*() const {
return std::visit([](const auto& iter) -> decltype(auto) {
return *iter;
}, it_);
}
// Arrow operator.
decltype(auto) operator->() const {
return std::visit([](const auto& iter) -> decltype(auto) {
return iter.operator->();
}, it_);
}
// Pre-increment operator.
variant_iterator& operator++() {
std::visit([](auto& iter) { ++iter; }, it_);
return *this;
}
// Post-increment operator.
variant_iterator operator++(int) {
variant_iterator temp(*this);
++(*this);
return temp;
}
// Pre-decrement operator.
variant_iterator& operator--() {
std::visit([](auto& iter) { --iter; }, it_);
return *this;
}
// Post-decrement operator.
variant_iterator operator--(int) {
variant_iterator temp(*this);
--(*this);
return temp;
}
// Equality comparison.
friend bool operator==(const variant_iterator& lhs, const variant_iterator& rhs) {
return lhs.it_ == rhs.it_;
}
// Inequality comparison.
friend bool operator!=(const variant_iterator& lhs, const variant_iterator& rhs) {
return !(lhs == rhs);
}
private:
variant_type it_;
};
namespace impl {
struct in_memory_file_storage {
IfcParse::IfcSpfLexer* tokens;
IfcParse::IfcSpfStream* stream;
IfcParse::IfcFile* file;
unresolved_references references_to_resolve;
typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
typedef boost::unordered_map<size_t, size_t> identity_by_id_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_t;
typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_by_guid_t;
typedef std::tuple<int, short, short> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef std::map<inverse_attr_record, std::vector<int>> entities_by_ref_t;
typedef std::map<int, std::vector<int>> entities_by_ref_excl_t;
typedef std::map<unsigned int, aggregate_of_instance::ptr> ref_map_t;
typedef map_transformer<identity_by_id_t, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
typedef entity_by_id_t::iterator iterator;
identity_by_id_t idenbyid_;
in_memory_file_storage()
: byid_(
&idenbyid_,
[this](size_t v) { return byidentity_[v]; },
[](IfcUtil::IfcBaseClass* inst) { return inst->identity(); }
)
{}
class type_iterator : private entities_by_type_t::const_iterator {
public:
using iterator_category = std::forward_iterator_tag;
using value_type = entities_by_type_t::key_type;
using difference_type = typename entities_by_type_t::const_iterator::difference_type;
using pointer = value_type const*;
using reference = value_type const&;
type_iterator() : entities_by_type_t::const_iterator() {};
type_iterator(const entities_by_type_t::const_iterator& iter)
: entities_by_type_t::const_iterator(iter) {};
entities_by_type_t::key_type const* operator->() const {
return &entities_by_type_t::const_iterator::operator->()->first;
}
entities_by_type_t::key_type const& operator*() const {
return entities_by_type_t::const_iterator::operator*().first;
}
type_iterator& operator++() {
entities_by_type_t::const_iterator::operator++();
return *this;
}
type_iterator operator++(int) {
type_iterator tmp(*this);
operator++();
return tmp;
}
bool operator!=(const type_iterator& other) const {
const entities_by_type_t::const_iterator& self_ = *this;
const entities_by_type_t::const_iterator& other_ = other;
return self_ != other_;
}
};
static bool guid_map_;
static bool guid_map() { return guid_map_; }
static void guid_map(bool b) { guid_map_ = b; }
entity_by_id_t byid_;
// this is for simple types
entity_by_iden_t byidentity_;
// entities_by_type_t bytype_;
entities_by_type_t bytype_excl_;
// entities_by_ref_t byref_;
entities_by_ref_t byref_excl_;
entity_by_guid_t byguid_;
void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
// @todo is this still used
IfcEntityInstanceData read(unsigned int index);
void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
file_open_status good_ = file_open_status::SUCCESS;
IfcUtil::IfcBaseClass* instance_by_id(int id);
void add_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(new_entity);
}
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) {
// @todo
}
void add_inverse_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(new_entity);
}
void remove_inverse_ref(IfcUtil::IfcBaseClass* new_entity) {
// @todo
}
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
};
struct rocks_db_file_storage {
// to make sure that instance pointer are constant during file lifetime
// cache instances because we want stable pointers
// @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side?
typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t;
entity_by_iden_cache_t instance_cache_;
// lookup id->identity
typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
identity_by_id_t byid_;
// typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_identity_t;
// storage is now Instance name -> Identity -> Pointer (cached)
// entity_by_identity_t byidentity_;
// index in schema to binary serialized ids
typedef rocksdb_map_adapter<size_t, std::string> instance_id_str_by_type_t;
instance_id_str_by_type_t bytype_;
// guid -> id
typedef rocksdb_map_adapter<std::string, size_t> instance_id_by_guid_str_t;
instance_id_by_guid_str_t byguid_internal_;
// guid -> id -> instance
typedef map_transformer<rocksdb_map_adapter<std::string, size_t>, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_by_guid_t;
entity_by_guid_t byguid_;
rocksdb::DB* db;
IfcParse::IfcFile* file;
// @todo naming
rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile);
bool read_schema(const IfcParse::schema_definition*& schema) {
// @todo
schema = nullptr;
return true;
}
IfcUtil::IfcBaseClass* assert_existance(size_t instanceId);
// @todo this could be another map_adapter?
class rocksdb_instance_iterator {
private:
rocksdb::Iterator* state_;
rocks_db_file_storage* storage_;
static constexpr char prefix_[] = "a|";
boost::optional<size_t> read_id_() const {
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
char* pEnd;
long result = strtol(sv.data() + 2, &pEnd, 10);
if (*pEnd == '|') {
return (size_t)result;
}
}
return boost::none;
}
public:
rocksdb_instance_iterator()
: state_(nullptr)
, storage_(nullptr)
{}
rocksdb_instance_iterator(rocks_db_file_storage* fs)
: storage_(fs)
{
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
}
}
rocksdb_instance_iterator& operator++() {
if (!state_) {
return *this;
}
auto last_id = read_id_();
while (state_->Valid()) {
state_->Next();
// Stop if we've left the prefix range.
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
break;
}
if (read_id_() != last_id) {
break;
}
}
return *this;
}
rocksdb_instance_iterator operator++(int) {
rocksdb_instance_iterator temp = *this;
++(*this);
return temp;
}
bool operator==(const rocksdb_instance_iterator& other) const {
if (state_ == nullptr && other.state_ == nullptr) {
return true;
} else {
return read_id_() == other.read_id_();
}
}
bool operator!=(const rocksdb_instance_iterator& other) const {
return !(*this == other);
}
IfcUtil::IfcBaseClass* operator*() const;
};
// @todo merge iterators (template?)
class rocksdb_types_iterator {
private:
rocksdb::Iterator* state_;
const rocks_db_file_storage* storage_;
static constexpr char prefix_[] = "t|";
boost::optional<size_t> read_id_() const {
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
char* pEnd;
long result = strtol(sv.data() + 2, &pEnd, 10);
if (*pEnd == '|') {
return (size_t)result;
}
}
return boost::none;
}
public:
using iterator_category = std::forward_iterator_tag;
using value_type = const IfcParse::declaration*;
// @todo ?
using difference_type = ptrdiff_t;
using pointer = value_type const*;
using reference = value_type const&;
rocksdb_types_iterator()
: state_(nullptr)
, storage_(nullptr)
{}
rocksdb_types_iterator(const rocks_db_file_storage* fs)
: storage_(fs)
{
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
}
}
rocksdb_types_iterator& operator++() {
if (!state_) {
return *this;
}
auto last_id = read_id_();
while (state_->Valid()) {
state_->Next();
// Stop if we've left the prefix range.
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
delete state_;
state_ = nullptr;
break;
}
if (read_id_() != last_id) {
break;
}
}
return *this;
}
rocksdb_types_iterator operator++(int) {
rocksdb_types_iterator temp = *this;
++(*this);
return temp;
}
bool operator==(const rocksdb_types_iterator& other) const {
if (state_ == nullptr && other.state_ == nullptr) {
return true;
} else {
return read_id_() == other.read_id_();
}
}
bool operator!=(const rocksdb_types_iterator& other) const {
return !(*this == other);
}
const IfcParse::declaration* operator*() const;
};
using const_iterator = rocksdb_types_iterator;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
// @todo a bit hard as a map because of value_type being an aggregate
void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
IfcUtil::IfcBaseClass* instance_by_id(int id);
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
};
}
enum filetype {
ifcspf,
ifcxml,
rocksdb,
autodetect
};
/// This class provides access to the entity instances in an IFC file
/// The file takes ownership of instances added to this file and deletes them when the file is deleted.
class IFC_PARSE_API IfcFile {
public:
unresolved_references references_to_resolve;
typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
typedef boost::unordered_map<unsigned int, IfcUtil::IfcBaseClass*> entity_by_id_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_t;
typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_by_guid_t;
typedef std::tuple<int, short, short> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef std::map<inverse_attr_record, std::vector<int>> entities_by_ref_t;
typedef std::map<int, std::vector<int>> entities_by_ref_excl_t;
typedef std::map<unsigned int, aggregate_of_instance::ptr> ref_map_t;
typedef entity_by_id_t::const_iterator const_iterator;
class type_iterator : private entities_by_type_t::const_iterator {
public:
type_iterator() : entities_by_type_t::const_iterator(){};
type_iterator(const entities_by_type_t::const_iterator& iter)
: entities_by_type_t::const_iterator(iter){};
entities_by_type_t::key_type const* operator->() const {
return &entities_by_type_t::const_iterator::operator->()->first;
}
entities_by_type_t::key_type const& operator*() const {
return entities_by_type_t::const_iterator::operator*().first;
}
type_iterator& operator++() {
entities_by_type_t::const_iterator::operator++();
return *this;
}
type_iterator operator++(int) {
type_iterator tmp(*this);
operator++();
return tmp;
}
bool operator!=(const type_iterator& other) const {
const entities_by_type_t::const_iterator& self_ = *this;
const entities_by_type_t::const_iterator& other_ = other;
return self_ != other_;
}
};
static bool guid_map_;
static bool guid_map() { return guid_map_; }
static void guid_map(bool b) { guid_map_ = b; }
private:
typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_entity_map_t;
// @todo determine the constness of things (probably needs to be all const, we don't want to overwrite)
// @todo we have variant_iterator and MapVariant, we probably need to retain only one?
public:
using const_iterator = variant_iterator<impl::in_memory_file_storage::iterator, impl::rocks_db_file_storage::const_iterator>;
using type_iterator = variant_iterator<impl::in_memory_file_storage::type_iterator, impl::rocks_db_file_storage::rocksdb_types_iterator>;
using storage_t = std::variant<std::monostate, impl::in_memory_file_storage, impl::rocks_db_file_storage>;
// @todo temporarily public for header
storage_t storage_;
private:
file_open_status good_ = file_open_status::SUCCESS;
const IfcParse::schema_definition* schema_;
const IfcParse::declaration* ifcroot_type_;
// std::vector<Argument*> internal_attribute_vector_, internal_attribute_vector_simple_type_;
entity_by_id_t byid_;
// this is for simple types
entity_by_iden_t byidentity_;
// entities_by_type_t bytype_;
entities_by_type_t bytype_excl_;
// entities_by_ref_t byref_;
entities_by_ref_t byref_excl_;
entity_by_guid_t byguid_;
entity_entity_map_t entity_file_map_;
unsigned int MaxId;
unsigned int max_id_;
IfcSpfHeader _header;
void setDefaultHeaderValues();
void initialize_(IfcParse::IfcSpfStream* stream);
void build_inverses_(IfcUtil::IfcBaseClass*);
typedef boost::multi_index_container<
int,
boost::multi_index::indexed_by<
@@ -195,37 +573,39 @@ class IFC_PARSE_API IfcFile {
void process_deletion_();
public:
IfcParse::IfcSpfLexer* tokens;
IfcParse::IfcSpfStream* stream;
#ifdef USE_MMAP
IfcFile(const std::string& path, bool mmap = false);
#else
IfcFile(const std::string& path);
IfcFile(const std::string& path, filetype ty=ifcspf);
#endif
IfcFile(std::istream& stream, int length);
IfcFile(void* data, int length);
IfcFile(IfcParse::IfcSpfStream* stream);
IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"));
/// Deleting the file will also delete all new instances that were added to the file (via memory allocation)
virtual ~IfcFile();
~IfcFile() {
for (const auto& p : byidentity_) {
delete p.second;
}
}
file_open_status good() const { return good_; }
/// Returns the first entity in the file, this probably is the entity
/// with the lowest id (EXPRESS ENTITY_INSTANCE_NAME)
const_iterator begin() const;
/// Returns the last entity in the file, this probably is the entity
/// with the highest id (EXPRESS ENTITY_INSTANCE_NAME)
const_iterator end() const;
/// Returns the first entity in the range of instances contained in the model,
/// in arbitrary order
auto begin() const {
return byid_.begin();
}
/// Returns the first entity in the range of instances contained in the model,
/// in arbitrary order
auto end() const {
return byid_.end();
}
type_iterator types_begin() const;
type_iterator types_end() const;
// type_iterator types_incl_super_begin() const;
// type_iterator types_incl_super_end() const;
/// Returns all entities in the file that match the template argument.
/// NOTE: This also returns subtypes of the requested type, for example:
/// IfcWall will also return IfcWallStandardCase entities
@@ -294,9 +674,9 @@ class IFC_PARSE_API IfcFile {
size_t getTotalInverses(int instance_id);
unsigned int FreshId() { return ++MaxId; }
unsigned int FreshId() { return ++max_id_; }
unsigned int getMaxId() const { return MaxId; }
unsigned int getMaxId() const { return max_id_; }
const IfcParse::declaration* ifcroot_type() const { return ifcroot_type_; }
@@ -305,12 +685,6 @@ class IFC_PARSE_API IfcFile {
IfcUtil::IfcBaseClass* addEntity(IfcUtil::IfcBaseClass* entity, int id = -1);
void addEntities(aggregate_of_instance::ptr entities);
void batch() { batch_mode_ = true; }
void unbatch() {
process_deletion_();
batch_mode_ = false;
}
/// Removes entity instance from file and unsets references.
///
/// Attention when running removeEntity inside a loop over a list of entities to be removed.
@@ -327,20 +701,31 @@ class IFC_PARSE_API IfcFile {
static std::string createTimestamp() ;
void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
void register_inverse(unsigned, const IfcParse::entity* from_entity, Token, int attribute_index);
void register_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
const IfcParse::schema_definition* schema() const { return schema_; }
std::pair<IfcUtil::IfcBaseClass*, double> getUnit(const std::string& unit_type);
void build_inverses();
entity_by_guid_t& internal_guid_map() { return byguid_; };
// @todo variant apply_visitor
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
typedef VariantMap<impl::in_memory_file_storage::entity_by_guid_t, impl::rocks_db_file_storage::entity_by_guid_t> entity_by_guid_t;
entity_by_guid_t byguid_;
typedef VariantMap<impl::in_memory_file_storage::identity_by_id_t, impl::rocks_db_file_storage::identity_by_id_t> identity_by_id_t;
identity_by_id_t byid_;
typedef VariantMap<impl::in_memory_file_storage::entity_by_iden_t, impl::rocks_db_file_storage::entity_by_iden_cache_t> entity_by_iden_t;
entity_by_iden_t byidentity_;
// @todo
entity_by_guid_t internal_guid_map() { return byguid_; };
void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
void build_inverses_(IfcUtil::IfcBaseClass*);
};
#ifdef WITH_IFCXML
+5 -5
View File
@@ -434,17 +434,17 @@ class IFC_PARSE_API IfcHierarchyHelper : public IfcParse::IfcFile {
aggregate_of_instance::ptr related_objects(new aggregate_of_instance);
related_objects->push(related_object);
IfcEntityInstanceData data = IfcEntityInstanceData(storage_t(T::Class().attribute_count()));
data.storage_.set(0, (std::string)IfcParse::IfcGlobalId());
data.storage_.set(1, owner_hist);
IfcEntityInstanceData data = IfcEntityInstanceData(in_memory_attribute_storage(T::Class().attribute_count()));
data.set_attribute_value(0, (std::string)IfcParse::IfcGlobalId());
data.set_attribute_value(1, owner_hist);
int relating_index = 4;
int related_index = 5;
if (T::Class().name() == "IfcRelContainedInSpatialStructure" || std::is_base_of<typename Schema::IfcRelDefines, T>::value) {
// some classes have attributes reversed.
std::swap(relating_index, related_index);
}
data.storage_.set(relating_index, relating_object);
data.storage_.set(related_index, related_objects);
data.set_attribute_value(relating_index, relating_object);
data.set_attribute_value(related_index, related_objects);
T* t = (T*)Schema::get_schema().instantiate(&T::Class(), std::move(data));
addEntity(t);
+453 -335
View File
File diff suppressed because it is too large Load Diff
-2
View File
@@ -164,8 +164,6 @@ class IFC_PARSE_API IfcSpfLexer {
void TokenString(unsigned int offset, std::string& result);
};
IFC_PARSE_API IfcEntityInstanceData read(unsigned int index, IfcFile* file);
IFC_PARSE_API aggregate_of_instance::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level = -1);
IFC_PARSE_API aggregate_of_instance::ptr traverse_breadth_first(IfcUtil::IfcBaseClass* instance, int max_level = -1);
+45 -20
View File
@@ -37,44 +37,69 @@ static const char* const DATA = "DATA";
using namespace IfcParse;
namespace {
IfcEntityInstanceData read_from_file(IfcFile* f, size_t s) {
parse_context pc;
f->tokens->Next();
f->load(-1, nullptr, pc, -1);
return pc.construct(-1, f->references_to_resolve, nullptr, s);
IfcEntityInstanceData read_from_spf_file(IfcFile* f, size_t s) {
return std::visit([f, s](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
parse_context pc;
m.tokens->Next();
m.load(-1, nullptr, pc, -1);
return pc.construct(-1, m.references_to_resolve, nullptr, s);
} else {
// std::unreachable();
return IfcEntityInstanceData(in_memory_attribute_storage(10));
}
}, f->storage_);
}
}
HeaderEntity::HeaderEntity(const char* const datatype, size_t size, IfcFile* file)
: datatype_(datatype)
, file_(file)
, data_(file ? read_from_file(file, size) : IfcEntityInstanceData(storage_t(size)))
, data_(file ? read_from_spf_file(file, size) : IfcEntityInstanceData(in_memory_attribute_storage(size)))
{}
HeaderEntity::~HeaderEntity() {
}
void IfcSpfHeader::readSemicolon() {
if (!TokenFunc::isOperator(file_->tokens->Next(), ';')) {
throw IfcException(std::string("Expected ;"));
}
std::visit([](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
if (!TokenFunc::isOperator(m.tokens->Next(), ';')) {
throw IfcException(std::string("Expected ;"));
}
} else {
// std::unreachable();
}
}, file_->storage_);
}
void IfcSpfHeader::readParen() {
if (!TokenFunc::isOperator(file_->tokens->Next(), '(')) {
throw IfcException(std::string("Expected ("));
}
std::visit([](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
if (!TokenFunc::isOperator(m.tokens->Next(), '(')) {
throw IfcException(std::string("Expected ("));
}
} else {
// std::unreachable();
}
}, file_->storage_);
}
void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) {
if (TokenFunc::asStringRef(file_->tokens->Next()) != term) {
throw IfcException(std::string("Expected " + term));
}
if (trail == TRAILING_SEMICOLON) {
readSemicolon();
} else if (trail == TRAILING_PAREN) {
readParen();
}
std::visit([this, term, trail](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, IfcParse::impl::in_memory_file_storage>) {
if (TokenFunc::asStringRef(m.tokens->Next()) != term) {
throw IfcException(std::string("Expected " + term));
}
if (trail == TRAILING_SEMICOLON) {
readSemicolon();
} else if (trail == TRAILING_PAREN) {
readParen();
}
} else {
// std::unreachable();
}
}, file_->storage_);
}
void IfcSpfHeader::read() {
+10 -18
View File
@@ -39,14 +39,6 @@ class IFC_PARSE_API HeaderEntity {
HeaderEntity(const char* const datatype, size_t size, IfcParse::IfcFile* file);
virtual ~HeaderEntity();
void setValue(unsigned int index, const std::string& string) {
data_.storage_.set(index, string);
}
void setValue(unsigned int index, const std::vector<std::string>& strings) {
data_.storage_.set(index, strings);
}
public:
virtual size_t getArgumentCount() const {
return data_.size();
@@ -71,8 +63,8 @@ class IFC_PARSE_API FileDescription : public HeaderEntity {
std::vector<std::string> description() const { return data_.get_attribute_value(0); }
std::string implementation_level() const { return data_.get_attribute_value(1); }
void description(const std::vector<std::string>& value) { setValue(0, value); }
void implementation_level(const std::string& value) { setValue(1, value); }
void description(const std::vector<std::string>& value) { data_.set_attribute_value(0, value); }
void implementation_level(const std::string& value) { data_.set_attribute_value(1, value); }
};
class IFC_PARSE_API FileName : public HeaderEntity {
@@ -87,13 +79,13 @@ class IFC_PARSE_API FileName : public HeaderEntity {
std::string originating_system() const { return data_.get_attribute_value(5); }
std::string authorization() const { return data_.get_attribute_value(6); }
void name(const std::string& value) { setValue(0, value); }
void time_stamp(const std::string& value) { setValue(1, value); }
void author(const std::vector<std::string>& value) { setValue(2, value); }
void organization(const std::vector<std::string>& value) { setValue(3, value); }
void preprocessor_version(const std::string& value) { setValue(4, value); }
void originating_system(const std::string& value) { setValue(5, value); }
void authorization(const std::string& value) { setValue(6, value); }
void name(const std::string& value) { data_.set_attribute_value(0, value); }
void time_stamp(const std::string& value) { data_.set_attribute_value(1, value); }
void author(const std::vector<std::string>& value) { data_.set_attribute_value(2, value); }
void organization(const std::vector<std::string>& value) { data_.set_attribute_value(3, value); }
void preprocessor_version(const std::string& value) { data_.set_attribute_value(4, value); }
void originating_system(const std::string& value) { data_.set_attribute_value(5, value); }
void authorization(const std::string& value) { data_.set_attribute_value(6, value); }
};
class IFC_PARSE_API FileSchema : public HeaderEntity {
@@ -102,7 +94,7 @@ class IFC_PARSE_API FileSchema : public HeaderEntity {
std::vector<std::string> schema_identifiers() const { return data_.get_attribute_value(0); }
void schema_identifiers(const std::vector<std::string>& value) { setValue(0, value); }
void schema_identifiers(const std::vector<std::string>& value) { data_.set_attribute_value(0, value); }
};
class IFC_PARSE_API IfcSpfHeader {
+1 -1
View File
@@ -209,7 +209,7 @@ IfcUtil::IfcBaseEntity::IfcBaseEntity(IfcEntityInstanceData&& data)
void IfcUtil::IfcBaseEntity::populate_derived() {
for (auto it = declaration().as_entity()->derived().begin(); it != declaration().as_entity()->derived().end(); ++it) {
if (*it) {
this->data().storage_.set(
this->data().set_attribute_value(
std::distance(declaration().as_entity()->derived().begin(), it),
Derived{}
);
+120
View File
@@ -0,0 +1,120 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include <iterator>
#include <type_traits>
#include <utility>
#include <functional>
// map_transformer: wraps a map-like construct so that its iterator returns
// a value_type where the mapped element is transformed via a function.
template <typename BaseMap, typename Transform, typename TransformBack>
class map_transformer {
public:
using key_type = typename BaseMap::key_type;
using base_mapped_type = typename BaseMap::mapped_type;
using transformed_mapped_type = std::invoke_result_t<Transform, base_mapped_type>;
using value_type = std::pair<key_type, transformed_mapped_type>;
private:
BaseMap* base_map_;
Transform transform_;
TransformBack transform_back_;
public:
map_transformer(BaseMap* map, Transform transform, TransformBack transform_back)
: base_map_(map), transform_(transform), transform_back_(transform_back){}
class iterator {
public:
using base_iterator = typename BaseMap::iterator;
using iterator_category = std::forward_iterator_tag;
using difference_type = typename std::iterator_traits<base_iterator>::difference_type;
using key_type = typename BaseMap::key_type;
using transformed_mapped_type = std::invoke_result_t<Transform, typename BaseMap::mapped_type>;
using value_type = std::pair<key_type, transformed_mapped_type>;
private:
base_iterator base_it_;
Transform* transform_ptr_;
mutable value_type cached_value_;
public:
iterator() : base_it_(), transform_ptr_(nullptr) {}
iterator(base_iterator base_it, Transform* transform_ptr)
: base_it_(base_it), transform_ptr_(transform_ptr) {}
// On dereference, return a pair where the key is unchanged and the mapped value
// is the result of applying the transform to the underlying mapped value.
// @todo should these also all be const references so that key/value can be non-copyable (i.e unique_ptr)
value_type operator*() const {
auto base_val = *base_it_;
return { base_val.first, (*transform_ptr_)(base_val.second) };
}
// operator-> uses a mutable cache to return a pointer to the current value.
value_type* operator->() const {
cached_value_ = **this;
return &cached_value_;
}
iterator& operator++() {
++base_it_;
return *this;
}
iterator operator++(int) {
iterator tmp(*this);
++(*this);
return tmp;
}
bool operator==(const iterator& other) const {
return base_it_ == other.base_it_;
}
bool operator!=(const iterator& other) const {
return !(*this == other);
}
};
iterator begin() {
return iterator(base_map_->begin(), &transform_);
}
iterator end() {
return iterator(base_map_->end(), &transform_);
}
iterator find(const key_type& k) {
return iterator(base_map_->find(k), &transform_);
}
// @todo still not sure if this is a good idea, do we want to insert into the transformed map?
std::pair<iterator, bool> insert(const value_type& val) {
auto p = base_map_->insert({ val.first, transform_back_(val.second) });
return { iterator(p.first, &transform_), p.second };
}
size_t erase(const key_type& key) {
return base_map_->erase(key);
}
};
+174
View File
@@ -0,0 +1,174 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include <map>
#include <variant>
#include <tuple>
#include <utility>
#include <cstddef>
#include <string>
#include <iostream>
// VariantMap: A map interface that delegates to one of several map types.
// The underlying maps are referenced by pointers (not moved into the variant).
// All map types must share the same key_type, mapped_type, and value_type.
template <typename... Maps>
class VariantMap {
public:
// The variant holds a pointer to the map
using variant_type = std::variant<std::monostate, Maps*...>;
variant_type map_;
// Deduce common types from the first map type.
// @todo these are not common types, but just the 1st
using key_type = typename std::tuple_element<0, std::tuple<Maps...>>::type::key_type;
using mapped_type = typename std::tuple_element<0, std::tuple<Maps...>>::type::mapped_type;
using value_type = typename std::tuple_element<0, std::tuple<Maps...>>::type::value_type;
using underlying_iterator_variant = std::variant<typename Maps::iterator...>;
class iterator {
public:
using value_type = VariantMap::value_type;
using difference_type = std::ptrdiff_t;
using pointer = value_type*;
using reference = value_type;
using iterator_category = std::forward_iterator_tag;
underlying_iterator_variant it_var;
// mutable cache to support operator-> (so that it->second works)
mutable std::unique_ptr<value_type> cached_value_ptr_;
iterator() = default;
explicit iterator(underlying_iterator_variant v)
: it_var(std::move(v)) {}
iterator(const iterator& other)
: it_var(other.it_var), cached_value_ptr_(nullptr) {}
iterator& operator=(const iterator& other) {
if (this != &other) {
it_var = other.it_var;
cached_value_ptr_.reset(); // clear the cache
}
return *this;
}
value_type operator*() const {
return std::visit([](auto& it) -> value_type { return *it; }, it_var);
}
value_type* operator->() const {
// @todo we need to make a copy here (stored in unique_ptr) because the
// value_type appears to be pair<const K, V> instead of <K, V> or something
// related...
cached_value_ptr_ = std::make_unique<value_type>(**this);
return cached_value_ptr_.get();
}
iterator& operator++() {
std::visit([](auto& it) { ++it; }, it_var);
return *this;
}
iterator operator++(int) {
iterator tmp(*this);
++(*this);
return tmp;
}
bool operator==(const iterator& other) const {
return it_var == other.it_var;
}
bool operator!=(const iterator& other) const {
return !(*this == other);
}
};
VariantMap() {}
template <typename MapT>
VariantMap(MapT* m) : map_(m) {}
iterator begin() const{
return std::visit([](auto m) -> iterator {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
return iterator{};
} else {
return iterator(m->begin());
}
}, map_);
}
iterator end() const {
return std::visit([](auto m) -> iterator {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
return iterator{};
} else {
return iterator(m->end());
}
}, map_);
}
iterator find(const key_type& key) const {
return std::visit([&key](auto m) -> iterator {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
return iterator{};
} else {
return iterator(m->find(key));
}
}, map_);
}
size_t erase(const key_type& key) {
return std::visit([&key](auto m) -> size_t {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
return size_t(0);
} else {
return m->erase(key);
}
}, map_);
}
size_t erase(const iterator& it) {
return std::visit([&it](auto m) -> size_t {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
return size_t(0);
} else {
// @todo erasing by iterator would be more efficient
return m->erase(it->first);
}
}, map_);
}
std::pair<iterator, bool> insert(const value_type& val) {
return std::visit([this, &val](auto m) -> std::pair<iterator, bool> {
// @todo is monostate still necessary here?
if constexpr (!std::is_same_v<std::decay_t<decltype(m)>, std::monostate>) {
auto result = m->insert(val);
return { iterator(result.first), result.second };
} else {
return { end(), false };
}
}, map_);
}
};
+8 -8
View File
@@ -279,7 +279,7 @@ static void end_element(void* user, const xmlChar* tag) {
}
*/
// @todo
// back.inst()->data().storage_.set(back.idx(), elems);
// back.inst()->data().set_attribute_value(back.idx(), elems);
}
if (state->dialect == ifcxml_dialect_ifc2x3 && state->stack.back().ntype() == stack_node::node_instance) {
@@ -325,7 +325,7 @@ static void process_characters(void* user, const xmlChar* character, int len) {
if (!val.empty()) {
// type declaration always at idx 0
visit_any([&state](auto& v) {
state->stack.back().inst()->data().storage_.set(0, v);
state->stack.back().inst()->data().set_attribute_value(0, v);
}, val);
}
} else if (state_type == stack_node::node_header_entry) {
@@ -479,7 +479,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
}
}
auto untyped = IfcEntityInstanceData(storage_t(decl->as_entity() != nullptr ? decl->as_entity()->attribute_count() : 1));
auto untyped = IfcEntityInstanceData(in_memory_attribute_storage(decl->as_entity() != nullptr ? decl->as_entity()->attribute_count() : 1));
const IfcParse::entity* entity = decl->as_entity();
if (entity != nullptr) {
@@ -494,7 +494,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
auto val = parse_attribute_value(attr->type_of_attribute(), pair.second);
if (!val.empty()) {
visit_any([&untyped, idx](auto& v) {
untyped.storage_.set(idx, v);
untyped.set_attribute_value(idx, v);
}, val);
}
} else {
@@ -531,7 +531,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
IfcUtil::IfcBaseClass* inst;
auto inst_ = create_instance(decl);
instance_to_attribute(inst_, state->stack.back().idx(), inst);
// state->stack.back().inst()->data().storage_.set(state->stack.back().idx(), attr);
// state->stack.back().inst()->data().set_attribute_value(state->stack.back().idx(), attr);
state->stack.push_back(stack_node::instance(id, inst));
} else if (state_type == stack_node::node_aggregate) {
@@ -592,7 +592,7 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
if (inst != nullptr) {
int idx = (*found)->entity_reference()->attribute_index(
(*found)->attribute_reference());
inst->data().storage_.set(idx, state->stack.back().inst());
inst->data().set_attribute_value(idx, state->stack.back().inst());
state->stack.push_back(stack_node::instance(id, inst));
} else {
Logger::Error("Unknown attribute " + tagname);
@@ -663,12 +663,12 @@ static void start_element(void* user, const xmlChar* tag, const xmlChar** attrs)
int idx = state->stack.back().inv_attr()->entity_reference()->attribute_index(
state->stack.back().inv_attr()->attribute_reference());
if (inst != nullptr) {
inst->data().storage_.set(idx, state->stack.back().inst());
inst->data().set_attribute_value(idx, state->stack.back().inst());
} else {
Logger::Error("Internal error, inverse attribute not processed");
}
} else if (state_type == stack_node::node_instance_attribute) {
state->stack.back().inst()->data().storage_.set(state->stack.back().idx(), inst);
state->stack.back().inst()->data().set_attribute_value(state->stack.back().idx(), inst);
}
if (entity == nullptr) {
+251
View File
@@ -0,0 +1,251 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <memory>
#include <string>
#include <utility>
#include <iterator>
#include <cstddef>
// Serialization and deserialization primitives
template <typename T>
struct DefaultCodec;
// Specialization for size_t.
template <>
struct DefaultCodec<size_t> {
std::string encode(const size_t& v) const {
std::string s(sizeof(v), 0);
size_t temp = v;
for (size_t i = 0; i < sizeof(v); i++) {
s[sizeof(v) - i - 1] = static_cast<char>(temp & 0xFF);
temp >>= 8;
}
return s;
}
size_t decode(const std::string& s) const {
size_t v = 0;
for (size_t i = 0; i < s.size(); i++) {
v = (v << 8) | static_cast<unsigned char>(s[i]);
}
return v;
}
};
// Specialization for std::string (identity)
template <>
struct DefaultCodec<std::string> {
std::string encode(const std::string& v) const {
return v;
}
std::string decode(const std::string& s) const {
return s;
}
};
template <typename KeyT>
std::string key_to_string(const KeyT& key) {
if constexpr (std::is_same_v<KeyT, std::string>) {
return key;
} else {
return std::to_string(key);
}
}
// Convert from a string to a key. For non-string types, we assume numeric keys.
template <typename KeyT>
KeyT key_from_string(const std::string& s) {
// @todo tuples
if constexpr (std::is_same_v<KeyT, std::string>) {
return s;
} else if constexpr (std::is_integral_v<KeyT>) {
return static_cast<KeyT>(std::stoll(s));
} else {
static_assert(sizeof(KeyT) == 0, "key_from_string not implemented for this type");
}
}
// rocksdb_map_adapter: a std::map-like interface on a RocksDB keyspace with a given prefix.
// The mapped_type is templated and encoded/decoded via Codec.
template <typename KeyT, typename MappedT, typename Codec = DefaultCodec<MappedT>>
class rocksdb_map_adapter {
public:
using key_type = KeyT;
using mapped_type = MappedT;
using value_type = std::pair<key_type, mapped_type>;
private:
rocksdb::DB* db_;
std::string prefix_;
Codec codec_;
public:
rocksdb_map_adapter(rocksdb::DB* db, const std::string& prefix)
: db_(db), prefix_(prefix), codec_(Codec{}) {}
class iterator {
public:
using value_type = std::pair<key_type, mapped_type>;
using difference_type = std::ptrdiff_t;
using iterator_category = std::forward_iterator_tag;
using pointer = value_type*;
using reference = value_type&;
private:
rocksdb::DB* db_;
std::string prefix_;
Codec codec_;
// When it_ is nullptr, this iterator is at end.
std::unique_ptr<rocksdb::Iterator> it_;
mutable value_type cached_value_;
void check_valid() {
if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_)) {
it_.reset();
}
}
public:
iterator() : db_(nullptr), prefix_(), codec_(Codec{}), it_(nullptr) {}
iterator(rocksdb::DB* db, const std::string& prefix,
std::unique_ptr<rocksdb::Iterator> iter, Codec codec = Codec{})
: db_(db), prefix_(prefix), codec_(codec), it_(std::move(iter))
{
check_valid();
}
iterator(const iterator& other)
: db_(other.db_), prefix_(other.prefix_), codec_(other.codec_)
{
if (other.it_) {
std::string curr = other.it_->key().ToString();
it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
it_->Seek(curr);
if (!it_->Valid() || it_->key().ToString() != curr)
it_.reset();
}
}
iterator& operator=(const iterator& other) {
if (this != &other) {
db_ = other.db_;
prefix_ = other.prefix_;
codec_ = other.codec_;
if (other.it_) {
std::string curr = other.it_->key().ToString();
it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
it_->Seek(curr);
if (!it_->Valid() || it_->key().ToString() != curr)
it_.reset();
} else {
it_.reset();
}
}
return *this;
}
value_type operator*() const {
std::string full_key = it_->key().ToString();
std::string key_without_prefix = full_key.substr(prefix_.size());
std::string value_str = it_->value().ToString();
return { key_from_string<key_type>(key_without_prefix), codec_.decode(value_str) };
}
// operator-> uses a mutable cache to return a pointer to the current value.
value_type* operator->() const {
cached_value_ = **this;
return &cached_value_;
}
iterator& operator++() {
if (it_) {
it_->Next();
check_valid();
}
return *this;
}
iterator operator++(int) {
iterator tmp(*this);
++(*this);
return tmp;
}
bool operator==(const iterator& other) const {
if (!it_ && !other.it_) return true;
if (it_ && other.it_)
return it_->key().ToString() == other.it_->key().ToString();
return false;
}
bool operator!=(const iterator& other) const {
return !(*this == other);
}
};
iterator begin() const {
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
iter->Seek(prefix_);
if (iter->Valid() && iter->key().starts_with(prefix_)) {
return iterator(db_, prefix_, std::move(iter), codec_);
}
return end();
}
iterator end() const {
return iterator();
}
iterator find(const key_type& key) const {
std::string key_str = key_to_string(key);
std::string full_key = prefix_ + key_str;
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
iter->Seek(full_key);
if (iter->Valid() && iter->key().ToString() == full_key)
return iterator(db_, prefix_, std::move(iter), codec_);
return end();
}
size_t erase(const key_type& key) {
std::string key_str = key_to_string(key);
std::string full_key = prefix_ + key_str;
rocksdb::Status s = db_->Delete(rocksdb::WriteOptions{}, full_key);
return s.ok() ? 1 : 0;
}
std::pair<iterator, bool> insert(const value_type& val) {
std::string key_str = key_to_string(val.first);
std::string full_key = prefix_ + key_str;
std::string existing;
rocksdb::Status s = db_->Get(rocksdb::ReadOptions{}, full_key, &existing);
if (s.ok()) {
// Key already exists.
return { find(val.first), false };
}
std::string encoded = codec_.encode(val.second);
s = db_->Put(rocksdb::WriteOptions{}, full_key, encoded);
if (!s.ok()) {
return { end(), false };
}
return { find(val.first), true };
}
};
+1 -13
View File
@@ -42,18 +42,6 @@ namespace impl {
template<class T, typename... Args>
struct is_unique_ptr<std::unique_ptr<T, Args...>> : std::true_type {};
/*
// Trait to find index of type in parameter pack
template <typename T, typename... Ts>
struct TypeIndex;
template <typename T, typename... Ts>
struct TypeIndex<T, T, Ts...> : std::integral_constant<std::size_t, 0> {};
template <typename T, typename U, typename... Ts>
struct TypeIndex<T, U, Ts...> : std::integral_constant<std::size_t, 1 + TypeIndex<T, Ts...>::value> {};
template <typename T, typename... Ts>
constexpr std::size_t TypeIndex_v = TypeIndex<T, Ts...>::value;
*/
// Trait to find index of type in parameter pack considering inheritance
template <typename T, typename... Ts>
struct TypeIndex;
@@ -239,7 +227,7 @@ public:
return apply_visitor_impl(std::forward<Visitor>(visitor), index, std::integral_constant<std::size_t, sizeof...(Types)>{});
}
auto size() const {
size_t size() const {
return size_and_indices_ ? size_and_indices_[0] : 0;
}