Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-11 02:02:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Non-streaming rocksdb serializer nearing completion

Browse Source
This commit is contained in:
Thomas Krijnen
2025-03-09 21:20:16 +01:00
parent ce4407ff3c
commit 9f49ad5f71
11 changed files with 672 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/IfcBaseClass.h
+1 -6
View File
@@ -97,12 +97,7 @@ protected:
IfcEntityInstanceData data_;
public:
IfcBaseClass(IfcEntityInstanceData&& data)
: identity_(counter_++)
, id_(0)
, file_(nullptr)
, data_(std::move(data))
{}
IfcBaseClass(IfcEntityInstanceData&& data);
const IfcEntityInstanceData& data() const { return data_; }
IfcEntityInstanceData& data() { return data_; }
src/ifcparse/IfcEntityInstanceData.cpp
+50 -45
View File
@@ -31,7 +31,7 @@ public:
namespace {
template<typename T>
inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
inline T dispatch_get_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, bool is_entity, uint8_t index_)
{
if (storage_model_ == 0) {
return array_.storage_ptr->get<T>(index_);
@@ -43,7 +43,7 @@ namespace {
!std::is_same_v<std::remove_cv_t<std::remove_pointer_t<T>>, IfcUtil::IfcBaseClass>)
{
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
impl::deserialize(array_.db_ptr, str, val);
}
return val;
@@ -51,24 +51,29 @@ namespace {
}
template<typename T>
inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
inline bool dispatch_has_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, bool is_entity, uint8_t index_)
{
if (storage_model_ == 0) {
return array_.storage_ptr->has<T>(index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
if constexpr (std::is_same_v<T, Blank>) {
if (str.size() == 0) {
return true;
}
}
return str[0] == TypeEncoder::encode_type<T>();
}
}
inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, uint8_t index_)
inline size_t dispatch_index_(AttributeValue::pointer_type array_, uint8_t storage_model_, size_t instance_name_, bool is_entity, uint8_t index_)
{
if (storage_model_ == 0) {
return array_.storage_ptr->index(index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
return (size_t) str[0] - 'A';
}
}
@@ -77,57 +82,57 @@ namespace {
AttributeValue::operator int() const
{
return dispatch_get_<int>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<int>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator bool() const
{
return dispatch_get_<bool>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<bool>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator double() const
{
return dispatch_get_<double>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<double>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator boost::logic::tribool() const
{
if (dispatch_has_<bool>(array_, storage_model_, instance_name_, index_)) {
return dispatch_get_<bool>(array_, storage_model_, instance_name_, index_);
if (dispatch_has_<bool>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_)) {
return dispatch_get_<bool>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
return dispatch_get_<boost::logic::tribool>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<boost::logic::tribool>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator std::string() const
{
if (dispatch_has_<EnumerationReference>(array_, storage_model_, instance_name_, index_)) {
if (dispatch_has_<EnumerationReference>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_)) {
// @todo this is silly, but the way things currently work,
// @todo also we don't really need to store a reference to the enumeration type, when this same type is already stored on the definition of the entity and no other value can be provided.
if (storage_model_ == 0) {
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, index_).value();
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_).value();
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = schema_->declarations()[v]->as_enumeration_type();
auto decl = array_.db_ptr->file->schema()->declarations()[v]->as_enumeration_type();
memcpy(&v, str.data() + 1 + sizeof(size_t), sizeof(size_t));
return decl->lookup_enum_value(v);
}
}
return dispatch_get_<std::string>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::string>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator EnumerationReference() const
{
if (storage_model_ == 0) {
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = schema_->declarations()[v]->as_enumeration_type();
auto decl = array_.db_ptr->file->schema()->declarations()[v]->as_enumeration_type();
memcpy(&v, str.data() + 5, sizeof(size_t));
return EnumerationReference(decl, v);
}
@@ -135,24 +140,24 @@ AttributeValue::operator EnumerationReference() const
AttributeValue::operator boost::dynamic_bitset<>() const
{
return dispatch_get_<boost::dynamic_bitset<>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<boost::dynamic_bitset<>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator IfcUtil::IfcBaseClass* () const
{
if (storage_model_ == 0) {
return dispatch_get_<IfcUtil::IfcBaseClass*>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<IfcUtil::IfcBaseClass*>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
} else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
size_t v;
memcpy(&v, str.data() + 2, sizeof(size_t));
if (str[1] == 'e') {
if (str[1] == 'i') {
// entity reference, by #Name
return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_name);
return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::entityinstance_ref);
} else if (str[1] == 't') {
// type reference by Identity
return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_identity);
return array_.db_ptr->assert_existance(v, IfcParse::impl::rocks_db_file_storage::typedecl_ref);
}
}
@@ -160,47 +165,47 @@ AttributeValue::operator IfcUtil::IfcBaseClass* () const
AttributeValue::operator std::vector<int>() const
{
return dispatch_get_<std::vector<int>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::vector<int>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator std::vector<double>() const
{
return dispatch_get_<std::vector<double>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::vector<double>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator std::vector<std::string>() const
{
return dispatch_get_<std::vector<std::string>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::vector<std::string>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator std::vector<boost::dynamic_bitset<>>() const
{
return dispatch_get_<std::vector<boost::dynamic_bitset<>>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::vector<boost::dynamic_bitset<>>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator boost::shared_ptr<aggregate_of_instance>() const
{
return dispatch_get_<boost::shared_ptr<aggregate_of_instance>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<boost::shared_ptr<aggregate_of_instance>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator std::vector<std::vector<int>>() const
{
return dispatch_get_<std::vector<std::vector<int>>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::vector<std::vector<int>>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator std::vector<std::vector<double>>() const
{
return dispatch_get_<std::vector<std::vector<double>>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<std::vector<std::vector<double>>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
AttributeValue::operator boost::shared_ptr<aggregate_of_aggregate_of_instance>() const
{
return dispatch_get_<boost::shared_ptr<aggregate_of_aggregate_of_instance>>(array_, storage_model_, instance_name_, index_);
return dispatch_get_<boost::shared_ptr<aggregate_of_aggregate_of_instance>>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
bool AttributeValue::isNull() const
{
return dispatch_has_<Blank>(array_, storage_model_, instance_name_, index_);
return dispatch_has_<Blank>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
unsigned int AttributeValue::size() const
@@ -211,7 +216,7 @@ unsigned int AttributeValue::size() const
IfcUtil::ArgumentType AttributeValue::type() const
{
return static_cast<IfcUtil::ArgumentType>(dispatch_index_(array_, storage_model_, instance_name_, index_));
return static_cast<IfcUtil::ArgumentType>(dispatch_index_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_));
}
bool impl::serialize(std::string& val, const IfcUtil::IfcBaseClass* t)
@@ -221,8 +226,8 @@ bool impl::serialize(std::string& val, const IfcUtil::IfcBaseClass* t)
val[0] = TypeEncoder::encode_type<IfcUtil::IfcBaseClass*>();
// 1 = entity - stored by id (entity name)
// 2 = type - stored by identity (internal counter in class)
val[1] = t->declaration().as_entity() ? 'e' : 't';
size_t iden = t->declaration().as_entity() ? t->id() : t->identity();
val[1] = t->declaration().as_entity() ? 'i' : 't';
size_t iden = t->id() ? t->id() : t->identity();
memcpy(val.data() + 2, &iden, s);
return true;
}
@@ -243,12 +248,12 @@ bool impl::serialize(std::string& val, const aggregate_of_instance::ptr& t)
{
// no attempt at alignment
val.resize(t->size() * (sizeof(size_t) + 1) + 1);
val[0] = TypeEncoder::encode_type<EnumerationReference>();
val[0] = TypeEncoder::encode_type<aggregate_of_instance::ptr>();
char* ptr = val.data() + 1;
for (auto it = t->begin(); it != t->end(); ++it) {
*ptr = (*it)->declaration().as_entity() ? 'e' : 't';
*ptr = (*it)->declaration().as_entity() ? 'i' : 't';
ptr++;
size_t iden = (*it)->declaration().as_entity() ? (*it)->id() : (*it)->identity();
size_t iden = (*it)->id() ? (*it)->id() : (*it)->identity();
memcpy(ptr, &iden, sizeof(size_t));
ptr += sizeof(size_t);
}
@@ -328,10 +333,10 @@ bool impl::deserialize(IfcParse::impl::rocks_db_file_storage* storage, const std
ptr++;
size_t v;
memcpy(&v, ptr, sizeof(size_t));
if (tt == 'e') {
t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_name));
if (tt == 'i') {
t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::entityinstance_ref));
} else if (tt == 't') {
t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_identity));
t->push(storage->assert_existance(v, IfcParse::impl::rocks_db_file_storage::typedecl_ref));
} else {
throw std::runtime_error("");
}
@@ -352,7 +357,7 @@ void rocks_db_attribute_storage::set(void* storage, const IfcParse::declaration*
impl::serialize(v, value);
rdb_storage->db->Put(
rocksdb::WriteOptions{},
(is_header ? "h|" : "i|") +
(is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) +
(is_header ? decl->name() : std::to_string(identity)) + "|" +
std::to_string(index), v);
}
src/ifcparse/IfcEntityInstanceData.h
+3 -4
View File
@@ -280,9 +280,8 @@ namespace impl {
struct AttributeValue {
uint8_t index_;
uint8_t storage_model_ = 0;
uint8_t entity_or_type_ = 0;
size_t instance_name_;
// @todo couple with db_ptr;
const IfcParse::schema_definition* schema_;
union pointer_type {
const in_memory_attribute_storage* storage_ptr;
IfcParse::impl::rocks_db_file_storage* db_ptr;
@@ -303,12 +302,12 @@ struct AttributeValue {
, storage_model_(0)
{}
AttributeValue(const IfcParse::schema_definition* schema, IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t index)
AttributeValue(IfcParse::impl::rocks_db_file_storage* db, size_t instance_name, uint8_t entity_or_type, uint8_t index)
: index_(index)
, array_(db)
, storage_model_(1)
, instance_name_(instance_name)
, schema_(schema)
, entity_or_type_(entity_or_type)
{}
operator int() const;
src/ifcparse/IfcFile.cpp
+28 -30
View File
@@ -329,6 +329,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
return IfcEntityInstanceData(std::move(storage));
}
/*
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_iterator::operator*() const {
auto it = storage_->byid_.find(*read_id_());
if (it != storage_->byid_.end()) {
@@ -341,32 +342,22 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::rocksdb_instance_i
}
}
}
*/
const IfcParse::declaration* IfcParse::impl::rocks_db_file_storage::rocksdb_types_iterator::operator*() const {
return storage_->file->schema()->declarations()[*read_id_()];
}
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(size_t number, instance_ref r) {
size_t name, identity;
std::string v;
if (r == by_identity) {
name = 0;
identity = number;
} else {
name = number;
auto it = byid_.find(name);
if (it == byid_.end()) {
throw std::runtime_error("Unable to lookup identity of name: #" + std::to_string(number));
}
identity = it->second;
}
decltype(instance_cache_)::const_iterator it = instance_cache_.find(identity);
decltype(instance_cache_)::const_iterator it = instance_cache_.find({ r, number });
if (it != instance_cache_.end()) {
return it->second;
}
std::string v;
rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, "i|" + std::to_string(identity) + "|t", &v);
// @todo should always be name, as we can/should not assign to identity
rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, (r == entityinstance_ref ? "i|" : "t|") + std::to_string(number) + "|_", &v);
if (s.ok()) {
size_t s;
memcpy(&s, v.data(), sizeof(size_t));
@@ -375,17 +366,14 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s
}
auto decl = file->schema()->declarations()[s];
bool is_entity = decl->as_entity() != nullptr;
if (is_entity != (r == by_name)) {
if (is_entity != (r == entityinstance_ref)) {
throw std::runtime_error("Incorrect reference");
}
IfcEntityInstanceData data(rocks_db_attribute_storage{});
auto inst = file->schema()->instantiate(decl, std::move(data));
inst->id_ = name;
inst->id_ = number;
inst->file_ = file;
instance_cache_.insert({ inst->identity(), inst });
if (is_entity) {
byid_.insert({ inst->id(), inst->identity() });
}
instance_cache_.insert({ {r, number}, inst });
return inst;
}
throw std::runtime_error("");
@@ -410,11 +398,13 @@ IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string&
: file(ffile)
, db(init_db(filepath))
, byguid_internal_(db, "g|")
, byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, by_name); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
, byid_(db, "d|")
, byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, entityinstance_ref); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
, instance_ids_(db, "i|")
, instance_by_name_(&instance_ids_, [this](size_t v) { return assert_existance(v, entityinstance_ref); })
, bytype_(db, "t|")
, byref_excl_(db, "v|")
// @todo by_identity is probably not correct here, this mapping is Name -> Identity, so Fn should have access to full pair?
, byidentity_(&byid_, [this](size_t v) { return assert_existance(v, by_identity); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
// , byidentity_(&byid_, [this](size_t v) { return assert_existance(v, by_identity); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
{}
IfcParse::impl::rocks_db_file_storage::~rocks_db_file_storage()
@@ -433,7 +423,8 @@ IfcParse::impl::rocks_db_file_storage::~rocks_db_file_storage()
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int id)
{
// @todo rename assert_existance() -> instance_by_id();
return assert_existance(id, by_name);
// - no cannot be done, because it needs to differentiate between entity instances and typedecls
return assert_existance(id, entityinstance_ref);
}
void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::IfcBaseClass* inst)
@@ -441,8 +432,8 @@ void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::If
auto id = inst->id();
{
// compute next prefix that does not start with v|{id}
auto prefix = "v|" + id;
// compute next prefix that does not start with v|{id}|
auto prefix = "v|" + std::to_string(id) + "|";
auto it = db->NewIterator(rocksdb::ReadOptions());
it->Seek(prefix);
while (it->Valid()) {
@@ -472,7 +463,7 @@ void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::If
// and update inverses from entity into other
{
auto prefix = "v|" + name;
auto prefix = "v|" + std::to_string(name) + "|";
auto it = db->NewIterator(rocksdb::ReadOptions());
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
@@ -498,4 +489,11 @@ IfcUtil::IfcBaseClass* IfcParse::impl::in_memory_file_storage::instance_by_id(in
throw IfcException("Instance #" + boost::lexical_cast<std::string>(id) + " not found");
}
return it->second;
}
}
IfcParse::IfcFile::~IfcFile() {
// @todo this does not make sense for rocksdb, because it would assert existance for the entire lazy model only to free the instances again
for (const auto& p : byid_) {
delete p.second;
}
}
src/ifcparse/IfcFile.h
+62 -73
View File
@@ -25,8 +25,10 @@
#include "IfcSchema.h"
#include "IfcSpfHeader.h"
#include "rocksdb_map_adapter.h"
#include "rocksdb_set_view.h"
#include "map_variant.h"
#include "map_transformer.h"
#include "set_to_map_transformer.h"
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/random_access_index.hpp>
@@ -213,35 +215,24 @@ namespace impl {
struct in_memory_file_storage {
IfcParse::IfcSpfLexer* tokens;
IfcParse::IfcSpfStream* stream;
IfcParse::IfcFile* file;
unresolved_references references_to_resolve;
typedef std::map<const IfcParse::declaration*, aggregate_of_instance::ptr> entities_by_type_t;
typedef boost::unordered_map<size_t, size_t> identity_by_id_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_t;
typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_by_guid_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> entity_instance_by_name_t;
typedef boost::unordered_map<uint32_t, IfcUtil::IfcBaseClass*> type_instance_by_name_t;
typedef std::map<std::string, IfcUtil::IfcBaseClass*> entity_instance_by_guid_t;
typedef std::tuple<int, short, short> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef std::map<inverse_attr_record, std::vector<int>> entities_by_ref_t;
typedef std::map<int, std::vector<int>> entities_by_ref_excl_t;
typedef std::map<unsigned int, aggregate_of_instance::ptr> ref_map_t;
typedef map_transformer<identity_by_id_t, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
typedef entity_by_id_t::iterator iterator;
in_memory_file_storage()
: byid_(
&idenbyid_,
[this](size_t v) { return byidentity_[v]; },
[](IfcUtil::IfcBaseClass* inst) { return inst->identity(); }
)
{}
typedef std::map<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
typedef entity_instance_by_name_t::iterator iterator;
in_memory_file_storage() : tokens(nullptr), stream(nullptr), file(nullptr) {}
in_memory_file_storage(const in_memory_file_storage&) = delete;
in_memory_file_storage(const in_memory_file_storage&&) = delete;
@@ -289,15 +280,11 @@ namespace impl {
static bool guid_map() { return guid_map_; }
static void guid_map(bool b) { guid_map_ = b; }
// this is for simple types
entity_by_iden_t byidentity_;
// entities_by_type_t bytype_;
entity_instance_by_name_t byid_;
type_instance_by_name_t tbyid_;
entities_by_type_t bytype_excl_;
// entities_by_ref_t byref_;
entities_by_ref_t byref_excl_;
entity_by_guid_t byguid_;
identity_by_id_t idenbyid_;
entity_by_id_t byid_;
entity_instance_by_guid_t byguid_;
void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
@@ -315,33 +302,26 @@ namespace impl {
void add_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
if (ty) {
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(new_entity);
}
bytype_excl_[ty]->push(new_entity);
}
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
auto it = bytype_excl_.find(ty);
if (it != bytype_excl_.end()) {
it->second->remove(new_entity);
if (it->second->size() == 0) {
bytype_excl_.erase(ty);
if (ty) {
auto it = bytype_excl_.find(ty);
if (it != bytype_excl_.end()) {
it->second->remove(new_entity);
if (it->second->size() == 0) {
bytype_excl_.erase(ty);
}
}
}
}
void add_inverse_ref(IfcUtil::IfcBaseClass* new_entity) {
auto ty = new_entity->declaration().as_entity();
if (bytype_excl_.find(ty) == bytype_excl_.end()) {
bytype_excl_[ty].reset(new aggregate_of_instance());
}
bytype_excl_[ty]->push(new_entity);
}
void remove_inverse_ref(IfcUtil::IfcBaseClass* new_entity) {
// @todo
}
void process_deletion_inverse(IfcUtil::IfcBaseClass* inst);
template <typename T>
@@ -355,19 +335,30 @@ namespace impl {
rocksdb::DB* db;
IfcParse::IfcFile* file;
enum instance_ref {
typedecl_ref,
entityinstance_ref
};
// to make sure that instance pointer are constant during file lifetime
// cache instances because we want stable pointers
// @todo this is silly, but we cannot have the same type, this should be just a pointer then on the IfcFile side?
typedef std::map<uint32_t, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t;
typedef std::map<std::pair<instance_ref, uint32_t>, IfcUtil::IfcBaseClass*> entity_by_iden_cache_t;
entity_by_iden_cache_t instance_cache_;
// lookup id->identity
typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
identity_by_id_t byid_;
// @todo all these size_ts should probably be uint32_t for consistency with in-mem storage
typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
// lookup id->identity
// typedef rocksdb_map_adapter<size_t, size_t> identity_by_id_t;
// identity_by_id_t byid_;
typedef rocksdb_set_view<size_t> instance_name_view_t;
instance_name_view_t instance_ids_;
typedef set_to_map_transformer<instance_name_view_t, std::function<IfcUtil::IfcBaseClass* (size_t)>> entity_instance_by_name_t;
entity_instance_by_name_t instance_by_name_;
// typedef map_transformer<rocksdb_map_adapter<size_t, size_t>, std::function<IfcUtil::IfcBaseClass*(size_t)>, std::function<size_t(IfcUtil::IfcBaseClass*)>> entity_by_id_t;
// storage is now Instance name -> Identity -> Pointer (cached)
entity_by_id_t byidentity_;
// entity_by_id_t byidentity_;
// index in schema to binary serialized ids
typedef rocksdb_map_adapter<size_t, std::string> instance_id_str_by_type_t;
@@ -378,23 +369,28 @@ namespace impl {
instance_id_by_guid_str_t byguid_internal_;
// guid -> id -> instance
typedef map_transformer<rocksdb_map_adapter<std::string, size_t>, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_by_guid_t;
entity_by_guid_t byguid_;
typedef map_transformer<rocksdb_map_adapter<std::string, size_t>, std::function<IfcUtil::IfcBaseClass* (size_t)>, std::function< size_t(IfcUtil::IfcBaseClass*)>> entity_instance_by_guid_t;
entity_instance_by_guid_t byguid_;
typedef std::tuple<int, int, int> inverse_attr_record;
enum INVERSE_ATTR {
INSTANCE_ID,
INSTANCE_TYPE,
ATTRIBUTE_INDEX
};
typedef rocksdb_map_adapter<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
entities_by_ref_t byref_excl_;
// @todo naming
rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file);
~rocks_db_file_storage();
bool read_schema(const IfcParse::schema_definition*& schema);
enum instance_ref {
by_name,
by_identity
};
IfcUtil::IfcBaseClass* assert_existance(size_t instanceId, instance_ref r);
// @todo this could be another map_adapter?
/*
class rocksdb_instance_iterator {
private:
rocksdb::Iterator* state_;
@@ -467,6 +463,7 @@ namespace impl {
IfcUtil::IfcBaseClass* operator*() const;
};
*/
// @todo merge iterators (template?)
class rocksdb_types_iterator {
@@ -554,7 +551,7 @@ namespace impl {
};
// @todo rocksdb_instance_iterator?
using const_iterator = rocksdb_types_iterator;
using const_iterator = entity_instance_by_name_t::iterator;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
@@ -634,11 +631,7 @@ private:
// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = ifcspf, const std::string& path = "");
~IfcFile() {
for (const auto& p : byidentity_) {
delete p.second;
}
}
~IfcFile();
file_open_status good() const { return good_; }
@@ -761,18 +754,14 @@ private:
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass*, int attribute_index);
typedef VariantMap<impl::in_memory_file_storage::entity_by_guid_t, impl::rocks_db_file_storage::entity_by_guid_t> entity_by_guid_t;
entity_by_guid_t byguid_;
typedef VariantMap<impl::in_memory_file_storage::entity_by_id_t, impl::rocks_db_file_storage::entity_by_id_t> entity_by_id_t;
typedef VariantMap<impl::in_memory_file_storage::entity_instance_by_guid_t, impl::rocks_db_file_storage::entity_instance_by_guid_t> entity_instance_by_guid_t;
entity_instance_by_guid_t byguid_;
typedef VariantMap<impl::in_memory_file_storage::entity_instance_by_name_t, impl::rocks_db_file_storage::entity_instance_by_name_t> entity_by_id_t;
entity_by_id_t byid_;
typedef VariantMap<impl::in_memory_file_storage::entity_by_iden_t, impl::rocks_db_file_storage::entity_by_iden_cache_t> entity_by_iden_t;
entity_by_iden_t byidentity_;
typedef VariantMap<impl::in_memory_file_storage::identity_by_id_t, impl::rocks_db_file_storage::identity_by_id_t> identity_by_id_t;
identity_by_id_t idenbyid_;
typedef VariantMap<impl::in_memory_file_storage::entities_by_ref_t, impl::rocks_db_file_storage::entities_by_ref_t> entities_by_ref_t;
entities_by_ref_t byref_excl_;
// @todo
entity_by_guid_t internal_guid_map() { return byguid_; };
entity_instance_by_guid_t internal_guid_map() { return byguid_; };
void add_type_ref(IfcUtil::IfcBaseClass* new_entity);
void remove_type_ref(IfcUtil::IfcBaseClass* new_entity);
src/ifcparse/IfcParse.cpp
+131 -77
View File
@@ -767,8 +767,8 @@ void IfcParse::impl::in_memory_file_storage::register_inverse(unsigned id_from,
}
void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) {
std::vector<int>& ids = byref_excl_[{inst->id(), from_entity->index_in_schema(), attribute_index}];
std::vector<int>::iterator iter = std::find(ids.begin(), ids.end(), id_from);
auto& ids = byref_excl_[{inst->id(), from_entity->index_in_schema(), attribute_index}];
auto iter = std::find(ids.begin(), ids.end(), id_from);
if (iter == ids.end()) {
// @todo inverses also need to be populated when multiple instances are added to a new file.
// throw IfcParse::IfcException("Instance not found among inverses");
@@ -780,8 +780,9 @@ void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from
namespace {
template <typename T>
std::string to_string_fixed_width(const T& t, size_t w) {
// @todo currently inactive
std::ostringstream oss;
oss << std::setfill('0') << std::setw(w) << t;
oss << /*std::setfill('0') << std::setw(w) <<*/ t;
return oss.str();
}
}
@@ -824,44 +825,47 @@ void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from,
void IfcParse::impl::rocks_db_file_storage::add_type_ref(IfcUtil::IfcBaseClass* new_entity)
{
if (!new_entity->declaration().as_entity()) {
throw std::runtime_error("Type refs are only supposed to be used for entities");
}
size_t v = new_entity->id();
size_t v;
std::string s(sizeof(size_t), ' ');
memcpy(s.data(), &v, sizeof(size_t));
// no merges yet, because the python client doesn't support them
// db->Merge(rocksdb::WriteOptions{}, "t|" + std::to_string(new_entity->declaration().index_in_schema()), s);
{
std::string current;
auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema());
db->Get(rocksdb::ReadOptions{}, key, &current);
auto new_val = current + s;
db->Put(rocksdb::WriteOptions{}, key, new_val);
if (new_entity->declaration().as_entity()) {
v = new_entity->id();
memcpy(s.data(), &v, sizeof(size_t));
// no merges yet, because the python client doesn't support them
// db->Merge(rocksdb::WriteOptions{}, "t|" + std::to_string(new_entity->declaration().index_in_schema()), s);
{
std::string current;
// @todo this uses the same key-namespace as typedecl instances, not a direct conflict, but also not very clear
auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema());
db->Get(rocksdb::ReadOptions{}, key, &current);
auto new_val = current + s;
db->Put(rocksdb::WriteOptions{}, key, new_val);
}
}
// not only mapping also register type
v = new_entity->declaration().index_in_schema();
memcpy(s.data(), &v, sizeof(size_t));
db->Put(rocksdb::WriteOptions{}, "i|" + std::to_string(new_entity->identity()) + "|t", s);
db->Put(rocksdb::WriteOptions{}, (new_entity->declaration().as_entity() ? "i|" : "t|") + std::to_string(new_entity->id() ? new_entity->id() : new_entity->identity()) + "|_", s);
}
void IfcParse::impl::rocks_db_file_storage::remove_type_ref(IfcUtil::IfcBaseClass* new_entity)
{
std::string s;
auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema());
if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) {
std::vector<size_t> vals(s.size() / sizeof(size_t));
memcpy(vals.data(), s.data(), s.size());
vals.erase(std::find(vals.begin(), vals.end(), (size_t)new_entity->id()));
s.resize(vals.size() * sizeof(size_t));
memcpy(s.data(), vals.data(), s.size());
db->Put(rocksdb::WriteOptions{}, key, s);
if (new_entity->declaration().as_entity()) {
std::string s;
auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema());
if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) {
std::vector<size_t> vals(s.size() / sizeof(size_t));
memcpy(vals.data(), s.data(), s.size());
vals.erase(std::find(vals.begin(), vals.end(), (size_t)new_entity->id()));
s.resize(vals.size() * sizeof(size_t));
memcpy(s.data(), vals.data(), s.size());
db->Put(rocksdb::WriteOptions{}, key, s);
}
}
db->Delete(rocksdb::WriteOptions{}, "i|" + std::to_string(new_entity->identity()) + "|t");
db->Delete(rocksdb::WriteOptions{}, (new_entity->declaration().as_entity() ? "i|" : "t|") + std::to_string(new_entity->id() ? new_entity->id() : new_entity->identity()) + "|_");
}
namespace {
@@ -1242,12 +1246,12 @@ void IfcUtil::IfcBaseClass::set_attribute_value(size_t i, const T& t) {
void* const storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr;
if constexpr (std::is_pointer_v<T>) {
if (t) {
data_.set_attribute_value(storage, &declaration(), identity(), i, t);
data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(), i, t);
} else {
data_.set_attribute_value(storage, &declaration(), identity(), i, Blank{});
data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(), i, Blank{});
}
} else {
data_.set_attribute_value(storage, &declaration(), identity(),i, t);
data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(),i, t);
}
}
auto new_attribute = get_attribute_value(i);
@@ -1301,16 +1305,16 @@ IfcFile::IfcFile(const std::string& path, filetype ty) {
// @todo unify these names, it's already confusing enough as it stands
byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
idenbyid_ = decltype(idenbyid_)(&std::get<impl::in_memory_file_storage>(storage_).idenbyid_);
byidentity_ = decltype(byidentity_)(&std::get<impl::in_memory_file_storage>(storage_).byidentity_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
// byidentity_ = decltype(byidentity_)(&std::get<impl::in_memory_file_storage>(storage_).byidentity_);
} else {
// @todo this can only be used for databases that already exist, because otherwise there is no way to specify the schema
storage_.emplace<2>(path, this);
std::get<impl::rocks_db_file_storage>(storage_).read_schema(schema_);
byid_ = decltype(byid_)(&std::get<impl::rocks_db_file_storage>(storage_).byidentity_);
idenbyid_ = decltype(idenbyid_)(&std::get<impl::rocks_db_file_storage>(storage_).byid_);
byidentity_ = decltype(byidentity_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_cache_);
byid_ = decltype(byid_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_by_name_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::rocks_db_file_storage>(storage_).byref_excl_);
// byidentity_ = decltype(byidentity_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_cache_);
}
ifcroot_type_ = schema_->declaration_by_name("IfcRoot");
}
@@ -1347,14 +1351,14 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
std::get<impl::in_memory_file_storage>(storage_).file = this;
byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
idenbyid_ = decltype(idenbyid_)(&std::get<impl::in_memory_file_storage>(storage_).idenbyid_);
byidentity_ = decltype(byidentity_)(&std::get<impl::in_memory_file_storage>(storage_).byidentity_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
// byidentity_ = decltype(byidentity_)(&std::get<impl::in_memory_file_storage>(storage_).byidentity_);
} else {
storage_.emplace<2>(path, this);
byid_ = decltype(byid_)(&std::get<impl::rocks_db_file_storage>(storage_).byidentity_);
idenbyid_ = decltype(idenbyid_)(&std::get<impl::rocks_db_file_storage>(storage_).byid_);
byidentity_ = decltype(byidentity_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_cache_);
byid_ = decltype(byid_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_by_name_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::rocks_db_file_storage>(storage_).byref_excl_);
// byidentity_ = decltype(byidentity_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_cache_);
}
setDefaultHeaderValues();
}
@@ -1492,8 +1496,8 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
Logger::Message(Logger::LOG_WARNING, ss.str());
}
idenbyid_[current_id] = instance->identity();
byidentity_[instance->identity()] = instance;
// byidentity_[instance->identity()] = instance;
byid_.insert({ current_id, instance });
// @nb cannot assign to byid_;
// byid_[current_id] = instance;
@@ -1548,10 +1552,10 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
if (it == byid_.end()) {
Logger::Error("Instance reference #" + std::to_string(*name) + " used by instance #" + std::to_string(ref) + " at attribute index " + std::to_string(refattr) + " not found");
} else {
byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, it->second);
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, it->second);
}
} else if (auto* inst = boost::get<IfcUtil::IfcBaseClass*>(v)) {
byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst);
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst);
}
} else if (auto* v = boost::get<std::vector<reference_or_simple_type>>(&p.second)) {
aggregate_of_instance::ptr instances(new aggregate_of_instance);
@@ -1568,7 +1572,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
instances->push(*inst);
}
}
byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances);
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances);
} else if (auto* v = boost::get<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance);
for (const auto& vi : *v) {
@@ -1587,7 +1591,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
}
instances->push(inner);
}
byidentity_[idenbyid_[p.first.name_]]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances);
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, instances);
}
}
@@ -1760,7 +1764,12 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
if (entity->declaration().as_entity() == nullptr) {
// While not a mapping that can be queried, we do need to free the instance later on
// @todo. why (over?)write this when adding from the same file?
byidentity_.insert({ new_entity->identity(), new_entity });
std::visit([new_entity](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
// @todo not freed yet
m.tbyid_.insert({ new_entity->identity(), new_entity });
}
}, storage_);
}
// If it is part of this file
@@ -1786,10 +1795,23 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
}
new_entity->file_ = this;
// A new entity instance name is generated and
// the instance is pointed to this file.
if (new_entity->declaration().as_entity() != nullptr) {
if (id == -1) {
new_entity->as<IfcUtil::IfcBaseEntity>()->set_id(FreshId());
} else {
new_entity->as<IfcUtil::IfcBaseEntity>()->set_id((unsigned int)id);
if ((unsigned)id > max_id_) {
max_id_ = (unsigned)id;
}
}
}
void* own_storage = std::visit([](const auto& m) { return (void*)&m; }, storage_);
void* other_storage = std::visit([](const auto& m) { return (void*)&m; }, other_file->storage_);
for (size_t i = 0; i < (entity->declaration().as_entity() ? entity->declaration().as_entity()->attribute_count() : 1); ++i) {
entity->data().apply_visitor(other_storage, decl, entity->identity(), [this, i, decl, new_entity, own_storage](const auto& v) {
entity->data().apply_visitor(other_storage, decl, entity->id() ? entity->id() : entity->identity(), [this, i, decl, new_entity, own_storage](const auto& v) {
using U = std::decay_t<decltype(v)>;
// only need to copy non-instance attribute values, others are assigned below after mapping
if constexpr (std::is_same_v<U, IfcUtil::IfcBaseClass*>) {
@@ -1895,19 +1917,6 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
}
}
// A new entity instance name is generated and
// the instance is pointed to this file.
if (new_entity->declaration().as_entity() != nullptr) {
if (id == -1) {
new_entity->as<IfcUtil::IfcBaseEntity>()->set_id(FreshId());
} else {
new_entity->as<IfcUtil::IfcBaseEntity>()->set_id((unsigned int)id);
if ((unsigned)id > max_id_) {
max_id_ = (unsigned)id;
}
}
}
entity_file_map_.insert(entity_entity_map_t::value_type(entity->identity(), new_entity));
}
@@ -1929,9 +1938,10 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
// The mapping by entity type is updated.
const IfcParse::declaration* ty = &new_entity->declaration();
if (ty->as_entity() != nullptr) {
// @nb happens always because this also registers the type of the instance in rocksdb
// if (ty->as_entity() != nullptr) {
add_type_ref(new_entity);
}
// }
if (ty->as_entity() != nullptr) {
int new_id = -1;
@@ -1956,17 +1966,27 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
ss << "Overwriting entity with id " << new_id;
Logger::Message(Logger::LOG_WARNING, ss.str());
}
// The mapping by entity instance name is updated.
idenbyid_.insert({ new_id, new_entity->identity() });
byidentity_.insert({ new_entity->identity(), new_entity });
// rocksdb instances are assumed to be create with file.create();
std::visit([new_entity](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
// @todo not freed yet
m.byid_.insert({ new_entity->identity(), new_entity });
}
}, storage_);
} else if (new_entity->file_ == nullptr) {
// For non-entity instances, no mappings are updated, but the file
// pointer has to be set, so that actual copies are created in subsequent
// times.
new_entity->file_ = this;
// While not a mapping that can be queried, we do need to free the instance
byidentity_.insert({ new_entity->identity(), new_entity });
// rocksdb instances are assumed to be create with file.create();
std::visit([new_entity](auto& m) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
// @todo not freed yet
m.tbyid_.insert({ new_entity->identity(), new_entity });
}
}, storage_);
}
/*
@@ -2077,7 +2097,8 @@ void IfcFile::removeEntity(IfcUtil::IfcBaseClass* entity) {
}
}
byid_.erase(byid_.find(id));
//byid_.erase(byid_.find(id));
byid_.erase(id);
const IfcParse::declaration* ty = &entity->declaration();
@@ -2183,7 +2204,7 @@ aggregate_of_instance::ptr IfcFile::instances_by_type_excl_subtypes(const IfcPar
std::vector<size_t> vals(s.size() / sizeof(size_t));
memcpy(vals.data(), s.data(), s.size());
for (auto& v : vals) {
ret->push(x.assert_existance(v, IfcParse::impl::rocks_db_file_storage::by_name));
ret->push(x.assert_existance(v, IfcParse::impl::rocks_db_file_storage::entityinstance_ref));
}
}
return ret;
@@ -2413,7 +2434,26 @@ aggregate_of_instance::ptr IfcFile::getInverse(int instance_id, const IfcParse::
}
}
} else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
// @todo
if (attribute_index == -1) {
// @todo no lower/upper_bounds() implemented yet
auto prefix = "v|" + std::to_string(instance_id) + "|";
auto it = x.db->NewIterator(rocksdb::ReadOptions());
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
std::vector<size_t> vals(it->value().size() / sizeof(size_t));
memcpy(vals.data(), it->value().data(), it->value().size());
for (auto& v : vals) {
return_value->push(instance_by_id(v));
}
}
} else {
auto it = x.byref_excl_.find({ instance_id, ent->index_in_schema(), attribute_index });
if (it != x.byref_excl_.end()) {
for (auto& i : it->second) {
return_value->push(instance_by_id(i));
}
}
}
}
}, storage_);
});
@@ -2586,12 +2626,12 @@ std::atomic_uint32_t IfcUtil::IfcBaseClass::counter_(0);
void IfcUtil::IfcBaseClass::unset_attribute_value(size_t index) {
void* storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr;
data_.set_attribute_value(storage, &declaration(), identity(), index, Blank{});
data_.set_attribute_value(storage, &declaration(), id() ? id() : identity(), index, Blank{});
}
AttributeValue IfcUtil::IfcBaseClass::get_attribute_value(size_t index) const {
void* storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr;
return data_.get_attribute_value(storage, &declaration(), identity(), index);
return data_.get_attribute_value(storage, &declaration(), id() ? id() : identity(), index);
}
void IfcUtil::IfcBaseClass::toString(std::ostream& out, bool upper) const
@@ -2606,7 +2646,7 @@ void IfcUtil::IfcBaseClass::toString(std::ostream& out, bool upper) const
out << declaration().name();
}
void* storage = file_ ? std::visit([](const auto& m) { return (void*)&m; }, file_->storage_) : nullptr;
data().toString(storage, &declaration(), identity(), out, upper);
data().toString(storage, &declaration(), id() ? id() : identity(), out, upper);
}
/*
@@ -2623,8 +2663,7 @@ AttributeValue IfcEntityInstanceData::get_attribute_value(void* storage, const I
if constexpr (std::is_same_v<std::decay_t<decltype(x)>, in_memory_attribute_storage>) {
return AttributeValue(&x, (uint8_t)index);
} else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, rocks_db_attribute_storage>) {
// @todo
return AttributeValue(decl->schema(), (IfcParse::impl::rocks_db_file_storage*) storage, identity, index);
return AttributeValue((IfcParse::impl::rocks_db_file_storage*) storage, identity, decl->as_entity() ? 1 : 0, index);
} else {
return AttributeValue{};
}
@@ -2643,6 +2682,21 @@ bool IfcParse::impl::rocks_db_file_storage::read_schema(const IfcParse::schema_d
return false;
}
IfcUtil::IfcBaseClass::IfcBaseClass(IfcEntityInstanceData&& data)
: identity_(counter_++)
, id_(0)
, file_(nullptr)
, data_(std::move(data))
{
/*
* @todo this is not allowed cannot call virtual func in constructor
if (!declaration().as_entity()) {
// @nb from v0.9 type decl instances have their own id, which may collide with instance names in the file
// but is otherwise unique
id_ = identity_;
}
*/
}
template void IFC_PARSE_API IfcUtil::IfcBaseClass::set_attribute_value<Blank>(size_t index, const Blank& value);
src/ifcparse/IfcSIPrefix.cpp
+1 -2
View File
@@ -120,8 +120,7 @@ double IfcParse::get_SI_equivalent(typename Schema::IfcNamedUnit* named_unit) {
if (component->declaration().is(Schema::IfcSIUnit::Class())) {
si_unit = component->template as<typename Schema::IfcSIUnit>();
typename Schema::IfcValue* value = factor->ValueComponent();
// @todo provide sufficient context
scale = value->data().get_attribute_value(nullptr, nullptr, 0, 0);
scale = value->as<IfcUtil::IfcBaseClass>()->get_attribute_value(0);
}
} else if (named_unit->declaration().is(Schema::IfcSIUnit::Class())) {
si_unit = named_unit->template as<typename Schema::IfcSIUnit>();
src/ifcparse/rocksdb_map_adapter.h
+86 -1
View File
@@ -25,10 +25,18 @@
#include <iterator>
#include <cstddef>
template <typename T>
struct is_std_tuple : std::false_type {};
template <typename... Ts>
struct is_std_tuple<std::tuple<Ts...>> : std::true_type {};
// Serialization and deserialization primitives
template <typename T>
struct DefaultCodec;
// @todo specialize for integral types in one go
// Specialization for size_t.
template <>
struct DefaultCodec<size_t> {
@@ -46,6 +54,38 @@ struct DefaultCodec<size_t> {
}
};
// Specialization for uint32_t.
template <>
struct DefaultCodec<uint32_t> {
std::string encode(const uint32_t& v) const {
std::string s(sizeof(v), 0);
memcpy(s.data(), &v, sizeof(v));
return s;
}
uint32_t decode(const std::string& s) const {
uint32_t v = 0;
// @todo take min of sizeof(v), len(s)
// @todo unify all serialization primitives
memcpy(&v, s.data(), sizeof(v));
return v;
}
};
// Specialization for std::vector<int>.
template <>
struct DefaultCodec<std::vector<uint32_t>> {
std::string encode(const std::vector<uint32_t>& vs) const {
std::string s(sizeof(uint32_t) * vs.size(), 0);
memcpy(s.data(), vs.data(), s.size());
return s;
}
std::vector<uint32_t> decode(const std::string& s) const {
std::vector<uint32_t> vs(s.size() / sizeof(uint32_t), 0);
memcpy(vs.data(), s.data(), s.size());
return vs;
}
};
// Specialization for std::string (identity)
template <>
struct DefaultCodec<std::string> {
@@ -67,7 +107,7 @@ std::string key_to_string(const KeyT& key) {
}
// Convert from a string to a key. For non-string types, we assume numeric keys.
template <typename KeyT>
template <typename KeyT, typename std::enable_if<!is_std_tuple<KeyT>::value, int>::type = 0>
KeyT key_from_string(const std::string& s) {
// @todo tuples
if constexpr (std::is_same_v<KeyT, std::string>) {
@@ -79,6 +119,51 @@ KeyT key_from_string(const std::string& s) {
}
}
template<typename Tuple, std::size_t... Is>
std::string tuple_to_string_impl(const Tuple& t, std::index_sequence<Is...>) {
std::ostringstream oss;
// Unpack the tuple; add a pipe before each element except the first.
((oss << (Is == 0 ? "" : "|") << std::to_string(std::get<Is>(t))), ...);
return oss.str();
}
template<typename... Ts>
std::string key_to_string(const std::tuple<Ts...>& key) {
return tuple_to_string_impl(key, std::index_sequence_for<Ts...>{});
}
// Helper: Convert a string token to the desired numeric type.
template<typename T>
T convert_string(const std::string& token) {
if constexpr (std::is_integral_v<T>) {
return static_cast<T>(std::stoll(token));
} else if constexpr (std::is_floating_point_v<T>) {
return static_cast<T>(std::stod(token));
} else {
static_assert(sizeof(T) == 0, "convert_string not implemented for this type");
}
}
// Helper: Build a tuple from a vector of string tokens.
template <typename TupleT, std::size_t... Is>
TupleT tuple_from_string_impl(const std::vector<std::string>& tokens, std::index_sequence<Is...>) {
return std::make_tuple(convert_string<std::tuple_element_t<Is, TupleT>>(tokens[Is])...);
}
template <typename TupleT, typename std::enable_if<is_std_tuple<TupleT>::value, int>::type = 0>
TupleT key_from_string(const std::string& s) {
std::vector<std::string> tokens;
std::istringstream iss(s);
std::string token;
while (std::getline(iss, token, '|')) {
tokens.push_back(token);
}
if (tokens.size() != std::tuple_size<TupleT>::value) {
throw std::runtime_error("Invalid tuple format");
}
return tuple_from_string_impl<TupleT>(tokens, std::make_index_sequence<std::tuple_size<TupleT>::value>{});
}
// rocksdb_map_adapter: a std::map-like interface on a RocksDB keyspace with a given prefix.
// The mapped_type is templated and encoded/decoded via Codec.
template <typename KeyT, typename MappedT, typename Codec = DefaultCodec<MappedT>>
src/ifcparse/rocksdb_set_view.h
+195
View File
@@ -0,0 +1,195 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <memory>
#include <string>
#include <iterator>
#include <cstddef>
#include <cstring>
#include <stdexcept>
template <typename KeyT>
class rocksdb_set_view {
public:
using key_type = KeyT;
using value_type = key_type;
private:
rocksdb::DB* db_;
std::string prefix_;
public:
// Constructor: provide a pointer to an open RocksDB instance and the key-space prefix,
// e.g. "i|"
rocksdb_set_view(rocksdb::DB* db, const std::string& prefix)
: db_(db), prefix_(prefix) {}
// --- Iterator ---
class iterator {
public:
using value_type = key_type;
using difference_type = std::ptrdiff_t;
using iterator_category = std::forward_iterator_tag;
using pointer = const value_type*;
using reference = const value_type&;
private:
rocksdb::DB* db_;
std::string prefix_;
std::unique_ptr<rocksdb::Iterator> it_;
mutable value_type cached_value_;
// Helper: extract the key (i.e. the value) from the current RocksDB key.
value_type extract_current_value() const {
std::string full_key = it_->key().ToString();
std::string remainder = full_key.substr(prefix_.size());
size_t pos = remainder.find('|');
std::string key_str = (pos != std::string::npos) ? remainder.substr(0, pos) : remainder;
return key_from_string<key_type>(key_str);
}
// Validates the current iterator state.
void check_valid() {
if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_))
it_.reset();
}
public:
iterator() : db_(nullptr), prefix_(), it_(nullptr) {}
iterator(rocksdb::DB* db, const std::string& prefix,
std::unique_ptr<rocksdb::Iterator> iter)
: db_(db), prefix_(prefix), it_(std::move(iter))
{
check_valid();
}
iterator(const iterator& other)
: db_(other.db_), prefix_(other.prefix_)
{
if (other.it_) {
std::string curr = other.it_->key().ToString();
it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
it_->Seek(curr);
if (!it_->Valid() || it_->key().ToString() != curr)
it_.reset();
}
}
iterator& operator=(const iterator& other) {
if (this != &other) {
db_ = other.db_;
prefix_ = other.prefix_;
if (other.it_) {
std::string curr = other.it_->key().ToString();
it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
it_->Seek(curr);
if (!it_->Valid() || it_->key().ToString() != curr)
it_.reset();
} else {
it_.reset();
}
}
return *this;
}
// Dereference: extract the key part from the RocksDB key.
value_type operator*() const {
return extract_current_value();
}
// Pointer access (via a cached value)
const value_type* operator->() const {
cached_value_ = **this;
return &cached_value_;
}
// Pre-increment: advance the iterator and skip over any duplicate keys.
iterator& operator++() {
if (it_) {
// Record the current key value.
value_type curr = extract_current_value();
do {
it_->Next();
} while (it_ && it_->Valid() && it_->key().starts_with(prefix_) &&
(extract_current_value() == curr));
if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_))
it_.reset();
}
return *this;
}
iterator operator++(int) {
iterator tmp(*this);
++(*this);
return tmp;
}
bool operator==(const iterator& other) const {
if (!it_ && !other.it_)
return true;
if (it_ && other.it_)
return it_->key().ToString() == other.it_->key().ToString();
return false;
}
bool operator!=(const iterator& other) const {
return !(*this == other);
}
};
// Returns an iterator to the first element in the key-space (or end() if none exist).
iterator begin() const {
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
iter->Seek(prefix_);
if (iter->Valid() && iter->key().starts_with(prefix_))
return iterator(db_, prefix_, std::move(iter));
return end();
}
// Returns an iterator representing the end of the key-space.
iterator end() const {
return iterator();
}
// Read-only find: returns an iterator to the element with the given key if it exists.
iterator find(const key_type& key) const {
std::string key_str = key_to_string(key);
// Construct the search key: prefix + key_str + separator.
std::string start_key = prefix_ + key_str + "|";
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
iter->Seek(start_key);
if (iter->Valid() && iter->key().starts_with(prefix_)) {
std::string full_key = iter->key().ToString();
std::string remainder = full_key.substr(prefix_.size());
size_t pos = remainder.find('|');
std::string found_key_str = (pos != std::string::npos) ? remainder.substr(0, pos) : remainder;
if (key_from_string<key_type>(found_key_str) == key)
return iterator(db_, prefix_, std::move(iter));
}
return end();
}
size_t erase(const key_type& key) {
// @todo
}
};
src/ifcparse/set_to_map_transformer.h
+113
View File
@@ -0,0 +1,113 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include <iterator>
#include <type_traits>
#include <utility>
#include <functional>
// map_transformer: wraps a set-like construct so that its iterator returns
// a value_type where of the underlying set element as the key, with the value
// that same key transformed via a function.
template <typename BaseSet, typename Transform>
class set_to_map_transformer {
public:
using key_type = typename BaseSet::value_type;
using transformed_mapped_type = std::invoke_result_t<Transform, key_type>;
using value_type = std::pair<key_type, transformed_mapped_type>;
using mapped_type = transformed_mapped_type;
private:
BaseSet* base_map_;
Transform transform_;
public:
set_to_map_transformer(BaseSet* map, Transform transform)
: base_map_(map), transform_(transform) {}
class iterator {
public:
using base_iterator = typename BaseSet::iterator;
using iterator_category = std::forward_iterator_tag;
using difference_type = typename std::iterator_traits<base_iterator>::difference_type;
using key_type = typename BaseSet::key_type;
using transformed_mapped_type = std::invoke_result_t<Transform, key_type>;
using value_type = std::pair<key_type, transformed_mapped_type>;
private:
base_iterator base_it_;
Transform* transform_ptr_;
mutable value_type cached_value_;
public:
iterator() : base_it_(), transform_ptr_(nullptr) {}
iterator(base_iterator base_it, Transform* transform_ptr)
: base_it_(base_it), transform_ptr_(transform_ptr) {}
// On dereference, return a pair where the key is the set value and the mapped value
// is the result of applying the transform to the underlying value.
value_type operator*() const {
auto base_val = *base_it_;
return { base_val, (*transform_ptr_)(base_val) };
}
// operator-> uses a mutable cache to return a pointer to the current value.
value_type* operator->() const {
cached_value_ = **this;
return &cached_value_;
}
iterator& operator++() {
++base_it_;
return *this;
}
iterator operator++(int) {
iterator tmp(*this);
++(*this);
return tmp;
}
bool operator==(const iterator& other) const {
return base_it_ == other.base_it_;
}
bool operator!=(const iterator& other) const {
return !(*this == other);
}
};
iterator begin() {
return iterator(base_map_->begin(), &transform_);
}
iterator end() {
return iterator(base_map_->end(), &transform_);
}
iterator find(const key_type& k) {
return iterator(base_map_->find(k), &transform_);
}
size_t erase(const key_type& k) {
// @todo
return 0;
}
};
win/build-deps.cmd
+2
View File
@@ -195,6 +195,8 @@ IF "%IFCOS_INSTALL_PYTHON%"=="TRUE" (
echo PYTHONHOME=%PYTHONHOME%>>"%~dp0\%BUILD_DEPS_CACHE_PATH%"
)
goto :rocksdb
:proj
IF EXIST "%INSTALL_DIR%\proj-9.2.1" (