Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-21 14:23:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcparse: fixed-width hex ids in RocksDB keys, batched instance erasure

Browse Source 
Review: recalculate_id_counter() scanned every i|<id>|_ record because
ids were written as decimal text, which doesn't sort numerically. Make
every numeric key segment fixed-width 16-digit lowercase hex, produced
and parsed by key_to_string()/key_from_string() in rocksdb_map_adapter.h,
with named builders (rocksdb_key::attribute, header_attribute,
type_record, inverse, inverse_prefix, type_list, upper_bound) that the
storage, entity_instance_data.cpp, read_schema() and the serializer use
instead of assembling "i|" + std::to_string(id) + ... by hand. Keys now
sort by id, an instance's records are contiguous in id order, and the
largest id is the last key under i|, which recalculate_id_counter()
seeks to. The two dormant to_string_fixed_width() helpers are gone.

This changes the on-disk layout; databases converted before this commit
have to be re-converted.

Also review: instance_cache_ eviction went through an on-erase hook on
set_to_map_transformer, a std::function call under a mutex per deleted
instance. Drop the hook; file::remove_entity() and unbatch() call
file::erase_instances_(ids), which on RocksDB is
rocks_db_file_storage::erase_instances(): one WriteBatch of DeleteRanges
and one lock for the whole batch.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HNrXDmR88wKPCYwGE21SyH
This commit is contained in:
Dion Moult
2026-09-17 10:22:01 +10:00
parent 2a2cb63b89
commit 9334308ffb
8 changed files with 134 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/entity_instance_data.cpp
+8 -24
View File
@@ -65,9 +65,7 @@ namespace {
{
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{},
(is_header ? "h|" : (entity_or_type->as_entity() ? "i|" : "t|")) +
(is_header ? entity_or_type->name() : std::to_string(instance_name_)) + "|" +
std::to_string(index_), &str);
(is_header ? rocksdb_key::header_attribute(entity_or_type->name(), index_) : rocksdb_key::attribute(entity_or_type->as_entity() != nullptr, instance_name_, index_)), &str);
::impl::deserialize(array_.db_ptr, str, val);
} else {
static_assert(
@@ -97,9 +95,7 @@ namespace {
std::string str;
const bool is_header = entity_or_type->schema() == &Header_section_schema::get_schema();
array_.db_ptr->db->Get(rocksdb::ReadOptions{},
(is_header ? "h|" : (entity_or_type->as_entity() ? "i|" : "t|")) +
(is_header ? entity_or_type->name() : std::to_string(instance_name_)) + "|" +
std::to_string(index_), &str);
(is_header ? rocksdb_key::header_attribute(entity_or_type->name(), index_) : rocksdb_key::attribute(entity_or_type->as_entity() != nullptr, instance_name_, index_)), &str);
if constexpr (std::is_same_v<T, blank>) {
if (str.size() == 0) {
return true;
@@ -125,9 +121,7 @@ namespace {
std::string str;
const bool is_header = entity_or_type->schema() == &Header_section_schema::get_schema();
if (!array_.db_ptr->db->Get(rocksdb::ReadOptions{},
(is_header ? "h|" : (entity_or_type->as_entity() ? "i|" : "t|")) +
(is_header ? entity_or_type->name() : std::to_string(instance_name_)) + "|" +
std::to_string(index_), &str).ok()) {
(is_header ? rocksdb_key::header_attribute(entity_or_type->name(), index_) : rocksdb_key::attribute(entity_or_type->as_entity() != nullptr, instance_name_, index_)), &str).ok()) {
return type_encoder::encode_type<blank>() - 'A';
}
return (size_t) str[0] - 'A';
@@ -179,9 +173,7 @@ attribute_value::operator std::string() const
std::string str;
const bool is_header = entity_or_type_->schema() == &Header_section_schema::get_schema();
array_.db_ptr->db->Get(rocksdb::ReadOptions{},
(is_header ? "h|" : (entity_or_type_->as_entity() ? "i|" : "t|")) +
(is_header ? entity_or_type_->name() : std::to_string(instance_name_)) + "|" +
std::to_string(index_), &str);
(is_header ? rocksdb_key::header_attribute(entity_or_type_->name(), index_) : rocksdb_key::attribute(entity_or_type_->as_entity() != nullptr, instance_name_, index_)), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = array_.db_ptr->file->schema()->declarations()[v]->as_enumeration_type();
@@ -203,9 +195,7 @@ attribute_value::operator enumeration_reference() const
std::string str;
const bool is_header = entity_or_type_->schema() == &Header_section_schema::get_schema();
array_.db_ptr->db->Get(rocksdb::ReadOptions{},
(is_header ? "h|" : (entity_or_type_->as_entity() ? "i|" : "t|")) +
(is_header ? entity_or_type_->name() : std::to_string(instance_name_)) + "|" +
std::to_string(index_), &str);
(is_header ? rocksdb_key::header_attribute(entity_or_type_->name(), index_) : rocksdb_key::attribute(entity_or_type_->as_entity() != nullptr, instance_name_, index_)), &str);
size_t v;
memcpy(&v, str.data() + 1, sizeof(size_t));
auto decl = array_.db_ptr->file->schema()->declarations()[v]->as_enumeration_type();
@@ -231,9 +221,7 @@ attribute_value::operator express::base () const
std::string str;
const bool is_header = entity_or_type_->schema() == &Header_section_schema::get_schema();
array_.db_ptr->db->Get(rocksdb::ReadOptions{},
(is_header ? "h|" : (entity_or_type_->as_entity() ? "i|" : "t|")) +
(is_header ? entity_or_type_->name() : std::to_string(instance_name_)) + "|" +
std::to_string(index_), &str);
(is_header ? rocksdb_key::header_attribute(entity_or_type_->name(), index_) : rocksdb_key::attribute(entity_or_type_->as_entity() != nullptr, instance_name_, index_)), &str);
size_t v;
memcpy(&v, str.data() + 2, sizeof(size_t));
if (str.size() > 1 && str[1] == 'i') {
@@ -548,9 +536,7 @@ bool rocks_db_attribute_storage::has(void* storage, const ifcopenshell::declarat
std::string v;
auto success = rdb_storage->db->Get(
rocksdb::ReadOptions{},
(is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) +
(is_header ? decl->name() : std::to_string(identity)) + "|" +
std::to_string(index), &v);
(is_header ? rocksdb_key::header_attribute(decl->name(), index) : rocksdb_key::attribute(decl->as_entity() != nullptr, identity, index)), &v);
if constexpr (std::is_same_v<std::decay_t<T>, blank>) {
if (!success.ok()) {
return true;
@@ -568,9 +554,7 @@ void rocks_db_attribute_storage::set(void* storage, const ifcopenshell::declarat
::impl::serialize(v, value);
rdb_storage->db->Put(
rdb_storage->wopts,
(is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) +
(is_header ? decl->name() : std::to_string(identity)) + "|" +
std::to_string(index), v);
(is_header ? rocksdb_key::header_attribute(decl->name(), index) : rocksdb_key::attribute(decl->as_entity() != nullptr, identity, index)), v);
}
template IFC_PARSE_API void rocks_db_attribute_storage::set<blank>(void* storage, const ifcopenshell::declaration* decl, std::size_t identity, size_t index, const blank& value);
src/ifcparse/file.cpp
+26 -13
View File
@@ -55,7 +55,7 @@ express::base ifcopenshell::impl::rocks_db_file_storage::assert_existance(size_t
std::string v;
rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, (r == entityinstance_ref ? "i|" : "t|") + std::to_string(number) + "|_", &v);
rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, rocksdb_key::type_record(r == entityinstance_ref, number), &v);
if (s.ok()) {
size_t s;
memcpy(&s, v.data(), sizeof(size_t));
@@ -155,15 +155,7 @@ ifcopenshell::impl::rocks_db_file_storage::rocks_db_file_storage(const std::stri
: db(init_db(filepath, readonly))
, file(ffile)
, instance_ids_(db.get(), "i|")
, instance_by_name_(
&instance_ids_,
[this](size_t v) { return assert_existance(v, entityinstance_ref); },
[this](size_t v) {
// The instance's keys are gone from the database; drop the
// cached handle so lookups don't keep resolving it.
std::lock_guard<std::mutex> lock(instance_cache_mutex_);
instance_cache_.erase((uint32_t)v);
})
, instance_by_name_(&instance_ids_, [this](size_t v) { return assert_existance(v, entityinstance_ref); })
, bytype_(db.get(), "t|")
, byguid_internal_(db.get(), "g|"),
byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, entityinstance_ref); }, [](const express::base& v) { return v.identity(); })
@@ -218,8 +210,8 @@ void ifcopenshell::impl::rocks_db_file_storage::process_deletion_inverse(const e
// Delete every record referencing inst: all keys under v|<id>|. The
// exclusive upper bound is the same prefix with its separator
// incremented, so no iterator is needed to find the range end.
auto prefix = "v|" + std::to_string(id) + "|";
auto upper_bound = "v|" + std::to_string(id) + std::string(1, '|' + 1);
const auto prefix = rocksdb_key::inverse_prefix(id);
const auto upper_bound = rocksdb_key::upper_bound(prefix);
rocksdb::WriteBatch batch;
batch.DeleteRange(prefix, upper_bound);
@@ -239,7 +231,7 @@ void ifcopenshell::impl::rocks_db_file_storage::process_deletion_inverse(const e
const unsigned int name = entity_attribute.id();
// Do not update inverses for simple types (which have id()==0 in IfcOpenShell).
if (name != 0) {
auto prefix = "v|" + std::to_string(name) + "|";
auto prefix = rocksdb_key::inverse_prefix(name);
auto it = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
@@ -263,6 +255,27 @@ void ifcopenshell::impl::rocks_db_file_storage::process_deletion_inverse(const e
#endif
}
void ifcopenshell::impl::rocks_db_file_storage::erase_instances(const std::vector<uint32_t>& ids)
{
#ifndef IFOPSH_WITH_ROCKSDB
(void)ids;
#endif
#ifdef IFOPSH_WITH_ROCKSDB
// One write for every instance's keys, one lock for their cached handles.
rocksdb::WriteBatch batch;
for (auto id : ids) {
const auto prefix = rocksdb_key::instance(true, id);
batch.DeleteRange(prefix, rocksdb_key::upper_bound(prefix));
}
db->Write(wopts, &batch);
std::lock_guard<std::mutex> lock(instance_cache_mutex_);
for (auto id : ids) {
instance_cache_.erase(id);
}
#endif
}
express::base ifcopenshell::impl::in_memory_file_storage::instance_by_id(int id)
{
auto it = byid_.find(id);
src/ifcparse/file.h
+1
View File
@@ -234,6 +234,7 @@ public:
batch_deletion_ids_t batch_deletion_ids_;
bool batch_mode_ = false;
void process_deletion_(const express::base& entity);
void erase_instances_(const std::vector<uint32_t>& ids);
public:
#ifdef USE_MMAP
src/ifcparse/parse.cpp
+32 -32
View File
@@ -1099,16 +1099,6 @@ void ifcopenshell::impl::in_memory_file_storage::unregister_inverse(unsigned id_
}
}
namespace {
template <typename T>
std::string to_string_fixed_width(const T& t, size_t) {
// @todo currently inactive
std::ostringstream oss;
oss << /*std::setfill('0') << std::setw(w) <<*/ t;
return oss.str();
}
}
void ifcopenshell::impl::rocks_db_file_storage::register_inverse(unsigned id_from, const ifcopenshell::entity* from_entity, int inst_id, int attribute_index) {
#ifndef IFOPSH_WITH_ROCKSDB
(void)id_from;
@@ -1122,7 +1112,7 @@ void ifcopenshell::impl::rocks_db_file_storage::register_inverse(unsigned id_fro
s.resize(sizeof(uint32_t));
memcpy(s.data(), &v, sizeof(uint32_t));
auto key = "v|" + to_string_fixed_width(inst_id, 10) + "|" + to_string_fixed_width(from_entity->index_in_schema(), 4) + "|" + to_string_fixed_width(attribute_index, 2);
auto key = rocksdb_key::inverse(inst_id, from_entity->index_in_schema(), attribute_index);
db->Merge(wopts, key, s);
/*
@@ -1147,7 +1137,7 @@ void ifcopenshell::impl::rocks_db_file_storage::unregister_inverse(unsigned id_f
#ifdef IFOPSH_WITH_ROCKSDB
static std::string s;
auto inst_id = inst.id();
auto key = "v|" + to_string_fixed_width(inst_id, 10) + "|" + to_string_fixed_width(from_entity->index_in_schema(), 4) + "|" + to_string_fixed_width(attribute_index, 2);
auto key = rocksdb_key::inverse(inst_id, from_entity->index_in_schema(), attribute_index);
if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) {
std::vector<uint32_t> vals(s.size() / sizeof(uint32_t));
memcpy(vals.data(), s.data(), s.size());
@@ -1178,12 +1168,12 @@ void ifcopenshell::impl::rocks_db_file_storage::add_type_ref(const express::base
memcpy(s.data(), &v, sizeof(size_t));
// no merges yet, because the python client doesn't support them
db->Merge(wopts, "t|" + std::to_string(new_entity.declaration().index_in_schema()), s);
db->Merge(wopts, rocksdb_key::type_list(new_entity.declaration().index_in_schema()), s);
/*{
std::string current;
// @todo this uses the same key-namespace as typedecl instances, not a direct conflict, but also not very clear
auto key = "t|" + std::to_string(new_entity.declaration().index_in_schema());
auto key = rocksdb_key::type_list(new_entity.declaration().index_in_schema());
db->Get(rocksdb::ReadOptions{}, key, &current);
auto new_val = current + s;
db->Put(wopts, key, new_val);
@@ -1193,7 +1183,7 @@ void ifcopenshell::impl::rocks_db_file_storage::add_type_ref(const express::base
// not only mapping also register type
v = new_entity.declaration().index_in_schema();
memcpy(s.data(), &v, sizeof(size_t));
db->Put(wopts, (new_entity.declaration().as_entity() ? "i|" : "t|") + std::to_string(new_entity.id() ? new_entity.id() : new_entity.identity()) + "|_", s);
db->Put(wopts, rocksdb_key::type_record(new_entity.declaration().as_entity() != nullptr, new_entity.id() ? new_entity.id() : new_entity.identity()), s);
#endif
}
@@ -1205,7 +1195,7 @@ void ifcopenshell::impl::rocks_db_file_storage::remove_type_ref(const express::b
#ifdef IFOPSH_WITH_ROCKSDB
if (new_entity.declaration().as_entity()) {
std::string s;
auto key = "t|" + std::to_string(new_entity.declaration().index_in_schema());
auto key = rocksdb_key::type_list(new_entity.declaration().index_in_schema());
if (db->Get(rocksdb::ReadOptions{}, key, &s).ok()) {
std::vector<size_t> vals(s.size() / sizeof(size_t));
memcpy(vals.data(), s.data(), s.size());
@@ -1216,7 +1206,7 @@ void ifcopenshell::impl::rocks_db_file_storage::remove_type_ref(const express::b
}
}
db->Delete(wopts, (new_entity.declaration().as_entity() ? "i|" : "t|") + std::to_string(new_entity.id() ? new_entity.id() : new_entity.identity()) + "|_");
db->Delete(wopts, rocksdb_key::type_record(new_entity.declaration().as_entity() != nullptr, new_entity.id() ? new_entity.id() : new_entity.identity()));
#endif
}
@@ -2645,15 +2635,13 @@ void file::recalculate_id_counter() {
}
#ifdef IFOPSH_WITH_ROCKSDB
else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
// Keys sort as text, so the largest id can't be found by seeking;
// scan the i|<id>|_ type records, one per entity instance.
// Ids are fixed-width hex in the keys, so the largest id owns the
// last key under the entity prefix.
const std::string prefix = "i|";
auto it = std::unique_ptr<rocksdb::Iterator>(x.db->NewIterator(rocksdb::ReadOptions()));
for (it->Seek(prefix); it->Valid() && it->key().starts_with(prefix); it->Next()) {
const auto key = it->key().ToString();
if (key.size() > 2 && key.compare(key.size() - 2, 2, "|_") == 0) {
k = std::max(k, (unsigned int)std::stoul(key.substr(2, key.size() - 4)));
}
it->SeekForPrev(rocksdb_key::upper_bound(prefix));
if (it->Valid() && it->key().starts_with(prefix)) {
k = (unsigned int)parse_hex_key(it->key().ToString().substr(prefix.size(), 16));
}
}
#endif
@@ -2956,10 +2944,24 @@ void file::remove_entity(const express::base& entity) {
batch_deletion_ids_.push_back(id);
} else {
process_deletion_(entity);
byid_.erase(entity.id());
erase_instances_({(uint32_t)id});
}
}
void file::erase_instances_(const std::vector<uint32_t>& ids) {
std::visit([this, &ids](auto& x) {
if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::in_memory_file_storage>) {
for (auto id : ids) {
byid_.erase(id);
}
} else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
x.erase_instances(ids);
} else {
throw std::runtime_error("Storage not initialized");
}
}, storage_);
}
void file::process_deletion_(const express::base& entity) {
auto references = instances_by_reference(entity.id());
@@ -3143,7 +3145,7 @@ std::vector<express::base> file::instances_by_reference(int t) {
#ifdef IFOPSH_WITH_ROCKSDB
else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
// @todo no lower/upper_bounds() implemented yet
auto prefix = "v|" + std::to_string(t) + "|";
auto prefix = rocksdb_key::inverse_prefix(t);
auto it = std::unique_ptr<rocksdb::Iterator>(x.db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
@@ -3301,7 +3303,7 @@ std::vector<int> file::get_inverse_indices_by_id(int instance_id) {
} else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
#ifdef IFOPSH_WITH_ROCKSDB
// @todo no lower/upper_bounds() implemented yet
auto prefix = "v|" + std::to_string(instance_id) + "|";
auto prefix = rocksdb_key::inverse_prefix(instance_id);
auto it = std::unique_ptr<rocksdb::Iterator>(x.db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
@@ -3374,7 +3376,7 @@ std::vector<express::entity> file::get_inverse(int instance_id, const ifcopenshe
visit_subtypes(type->as_entity(), [this, attribute_index, instance_id, &return_value, &x](const ifcopenshell::declaration* ent) {
if (attribute_index == -1) {
// @todo no lower/upper_bounds() implemented yet
auto prefix = "v|" + std::to_string(instance_id) + "|" + std::to_string(ent->index_in_schema()) + "|";
auto prefix = rocksdb_key::inverse_prefix(instance_id) + key_to_string(ent->index_in_schema()) + "|";
auto it = std::unique_ptr<rocksdb::Iterator>(x.db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
@@ -3523,9 +3525,7 @@ void ifcopenshell::file::unbatch() {
process_deletion_(instance_by_id(id));
}
// keep in memory until all deletions are processed
for (auto& id : batch_deletion_ids_) {
byid_.erase(id);
}
erase_instances_(std::vector<uint32_t>(batch_deletion_ids_.begin(), batch_deletion_ids_.end()));
batch_mode_ = false;
batch_deletion_ids_.clear();
}
@@ -3629,7 +3629,7 @@ bool ifcopenshell::impl::rocks_db_file_storage::read_schema(const ifcopenshell::
#endif
#ifdef IFOPSH_WITH_ROCKSDB
std::string value;
auto key = "h|file_schema|0";
const auto key = rocksdb_key::header_attribute("file_schema", 0);
db->Get(rocksdb::ReadOptions{}, key, &value);
std::vector<std::string> strings;
if (::impl::deserialize(this, value, strings) && strings.size() == 1) {
src/ifcparse/rocksdb_map_adapter.h
+56 -4
View File
@@ -105,12 +105,64 @@ struct DefaultCodec<std::string> {
}
};
// Numeric key segments are fixed-width lowercase hex, so keys sort as
// numbers: an instance's records are laid out in id order and the largest
// id under a prefix is its last key.
inline std::string hex_key(uint64_t value) {
static constexpr char digits[] = "0123456789abcdef";
std::string s(16, '0');
for (int i = 15; i >= 0; --i) {
s[(size_t)i] = digits[value & 0xf];
value >>= 4;
}
return s;
}
inline uint64_t parse_hex_key(const std::string& s) {
return std::stoull(s, nullptr, 16);
}
template <typename KeyT>
std::string key_to_string(const KeyT& key) {
if constexpr (std::is_same_v<KeyT, std::string>) {
return key;
} else {
return std::to_string(key);
static_assert(std::is_integral_v<KeyT>, "key_to_string expects a string or an integral key");
return hex_key((uint64_t)key);
}
}
// The keys the file storage and the serializer agree on:
// i|<id>|<attribute> t|<identity>|<attribute> h|<name>|<attribute>
// i|<id>|_ t|<identity>|_ type record
// v|<referenced id>|<entity index>|<attribute> inverse record
// t|<declaration index> instances of a type
namespace rocksdb_key {
// The exclusive end of everything under prefix.
inline std::string upper_bound(std::string prefix) {
prefix.back() = (char)(prefix.back() + 1);
return prefix;
}
inline std::string instance(bool is_entity, uint64_t id) {
return (is_entity ? "i|" : "t|") + key_to_string(id) + "|";
}
inline std::string attribute(bool is_entity, uint64_t id, uint64_t index) {
return instance(is_entity, id) + key_to_string(index);
}
inline std::string header_attribute(const std::string& name, uint64_t index) {
return "h|" + name + "|" + key_to_string(index);
}
inline std::string type_record(bool is_entity, uint64_t id) {
return instance(is_entity, id) + "_";
}
inline std::string inverse_prefix(uint64_t referenced_id) {
return "v|" + key_to_string(referenced_id) + "|";
}
inline std::string inverse(uint64_t referenced_id, uint64_t entity_index, uint64_t attribute_index) {
return inverse_prefix(referenced_id) + key_to_string(entity_index) + "|" + key_to_string(attribute_index);
}
inline std::string type_list(uint64_t declaration_index) {
return "t|" + key_to_string(declaration_index);
}
}
@@ -121,7 +173,7 @@ KeyT key_from_string(const std::string& key_string) {
if constexpr (std::is_same_v<KeyT, std::string>) {
return key_string;
} else if constexpr (std::is_integral_v<KeyT>) {
return static_cast<KeyT>(std::stoll(key_string));
return static_cast<KeyT>(parse_hex_key(key_string));
} else {
static_assert(sizeof(KeyT) == 0, "key_from_string not implemented for this type");
}
@@ -132,7 +184,7 @@ std::string tuple_to_string_impl(const Tuple& tuple_value, std::index_sequence<I
static_cast<void>(indices);
std::ostringstream oss;
// Unpack the tuple; add a pipe before each element except the first.
((oss << (Is == 0 ? "" : "|") << std::to_string(std::get<Is>(tuple_value))), ...);
((oss << (Is == 0 ? "" : "|") << key_to_string(std::get<Is>(tuple_value))), ...);
return oss.str();
}
@@ -145,7 +197,7 @@ std::string key_to_string(const std::tuple<Ts...>& key) {
template<typename T>
T convert_string(const std::string& token) {
if constexpr (std::is_integral_v<T>) {
return static_cast<T>(std::stoll(token));
return static_cast<T>(parse_hex_key(token));
} else if constexpr (std::is_floating_point_v<T>) {
return static_cast<T>(std::stod(token));
} else {
src/ifcparse/set_to_map_transformer.h
+1 -11
View File
@@ -36,17 +36,11 @@ public:
private:
BaseSet* base_map_;
Transform transform_;
std::function<void(const key_type&)> on_erase_;
public:
set_to_map_transformer(BaseSet* base_set, Transform transform)
: base_map_(base_set), transform_(transform) {}
// on_erase runs after erase(key), whether or not the base set still
// held the key, so state derived from the set can be dropped.
set_to_map_transformer(BaseSet* base_set, Transform transform, std::function<void(const key_type&)> on_erase)
: base_map_(base_set), transform_(transform), on_erase_(std::move(on_erase)) {}
class iterator {
public:
using base_iterator = typename BaseSet::iterator;
@@ -113,10 +107,6 @@ public:
}
size_t erase(const key_type& key) {
const size_t erased = base_map_->erase(key);
if (on_erase_) {
on_erase_(key);
}
return erased;
return base_map_->erase(key);
}
};
src/ifcparse/storage.h
+4
View File
@@ -682,6 +682,10 @@ namespace ifcopenshell {
// id counter is recalculated on the first create().
bool id_counter_recalculated_ = false;
// Deletes every key of the given instances and drops their cached
// handles, in one write and under one lock.
void erase_instances(const std::vector<uint32_t>& ids);
// @todo all these size_ts should probably be uint32_t for consistency with in-mem storage
// lookup id->identity
src/serializers/rocks_db_serializer.cpp
+6 -18
View File
@@ -98,16 +98,6 @@ namespace {
}
}
namespace {
template <typename T>
std::string to_string_fixed_width(const T& t, size_t w) {
// @todo currently inactive
std::ostringstream oss;
oss << /*std::setfill('0') << std::setw(w) <<*/ t;
return oss.str();
}
}
void RocksDbSerializer::write_streaming_() {
ifcopenshell::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr);
@@ -170,15 +160,13 @@ void RocksDbSerializer::write_streaming_() {
// @nb cast to int in order not be interpreted as a char when appending to string
int index = p.first.index_;
auto key = (is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) +
(is_header ? decl->name() : std::to_string(p.first.name_)) + "|" +
std::to_string(index);
auto key = (is_header ? rocksdb_key::header_attribute(decl->name(), index) : rocksdb_key::attribute(decl->as_entity() != nullptr, p.first.name_, index));
if (storage.db->Get(storage.ropts, key, &tmp) == rocksdb::Status::OK() && tmp.size() == (sizeof(size_t) + 2) && tmp[0] == ifcopenshell::type_encoder::encode_type<express::base>() && tmp[1] == 't')
{
size_t iden;
memcpy(&iden, tmp.data() + 2, sizeof(size_t));
key = "t|" + std::to_string(iden) + "|0";
key = rocksdb_key::attribute(false, iden, 0);
type_identities_wrote_as_refs.insert(iden);
}
@@ -215,7 +203,7 @@ void RocksDbSerializer::write_streaming_() {
auto write_inverse = [&](const ifcopenshell::reference_or_simple_type& v) {
if (auto* ref = std::get_if<ifcopenshell::instance_reference>(&v)) {
auto key = "v|" + to_string_fixed_width(*ref, 10) + "|" + to_string_fixed_width(decl->index_in_schema(), 4) + "|" + to_string_fixed_width(index, 2);
auto key = rocksdb_key::inverse(*ref, decl->index_in_schema(), index);
static std::string s;
uint32_t vv = name;
s.resize(sizeof(uint32_t));
@@ -247,7 +235,7 @@ void RocksDbSerializer::write_streaming_() {
storage.db->Put(
storage.wopts,
(inst.declaration().as_entity() ? "i|" : "t|") + std::to_string(inst.identity()) + "|_", s);
rocksdb_key::type_record(inst.declaration().as_entity() != nullptr, inst.identity()), s);
if (type_identities_wrote_as_refs.find(inst.identity()) != type_identities_wrote_as_refs.end()) {
// already written as reference, skip
@@ -275,13 +263,13 @@ void RocksDbSerializer::write_streaming_() {
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Put(
storage.wopts,
(decl->as_entity() ? "i|" : "t|") + std::to_string(name) + "|_", s);
rocksdb_key::type_record(decl->as_entity() != nullptr, name), s);
{
size_t v = name;
std::string s(sizeof(size_t), ' ');
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Merge(storage.wopts, "t|" + std::to_string(decl->index_in_schema()), s);
storage.db->Merge(storage.wopts, rocksdb_key::type_list(decl->index_in_schema()), s);
}
// GlobalId as numeric ref to instance name, so that the guid map in