Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-23 06:12:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcparse: fixed-width hex ids in RocksDB keys, batched instance erasure

Browse Source 
Review: recalculate_id_counter() scanned every i|<id>|_ record because
ids were written as decimal text, which doesn't sort numerically. Make
every numeric key segment fixed-width 16-digit lowercase hex, produced
and parsed by key_to_string()/key_from_string() in rocksdb_map_adapter.h,
with named builders (rocksdb_key::attribute, header_attribute,
type_record, inverse, inverse_prefix, type_list, upper_bound) that the
storage, entity_instance_data.cpp, read_schema() and the serializer use
instead of assembling "i|" + std::to_string(id) + ... by hand. Keys now
sort by id, an instance's records are contiguous in id order, and the
largest id is the last key under i|, which recalculate_id_counter()
seeks to. The two dormant to_string_fixed_width() helpers are gone.

This changes the on-disk layout; databases converted before this commit
have to be re-converted.

Also review: instance_cache_ eviction went through an on-erase hook on
set_to_map_transformer, a std::function call under a mutex per deleted
instance. Drop the hook; file::remove_entity() and unbatch() call
file::erase_instances_(ids), which on RocksDB is
rocks_db_file_storage::erase_instances(): one WriteBatch of DeleteRanges
and one lock for the whole batch.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HNrXDmR88wKPCYwGE21SyH
This commit is contained in:
Dion Moult
2026-09-17 10:22:01 +10:00
parent 2a2cb63b89
commit 9334308ffb
8 changed files with 134 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/serializers/rocks_db_serializer.cpp
+6 -18
View File
@@ -98,16 +98,6 @@ namespace {
}
}
namespace {
template <typename T>
std::string to_string_fixed_width(const T& t, size_t w) {
// @todo currently inactive
std::ostringstream oss;
oss << /*std::setfill('0') << std::setw(w) <<*/ t;
return oss.str();
}
}
void RocksDbSerializer::write_streaming_() {
ifcopenshell::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr);
@@ -170,15 +160,13 @@ void RocksDbSerializer::write_streaming_() {
// @nb cast to int in order not be interpreted as a char when appending to string
int index = p.first.index_;
auto key = (is_header ? "h|" : (decl->as_entity() ? "i|" : "t|")) +
(is_header ? decl->name() : std::to_string(p.first.name_)) + "|" +
std::to_string(index);
auto key = (is_header ? rocksdb_key::header_attribute(decl->name(), index) : rocksdb_key::attribute(decl->as_entity() != nullptr, p.first.name_, index));
if (storage.db->Get(storage.ropts, key, &tmp) == rocksdb::Status::OK() && tmp.size() == (sizeof(size_t) + 2) && tmp[0] == ifcopenshell::type_encoder::encode_type<express::base>() && tmp[1] == 't')
{
size_t iden;
memcpy(&iden, tmp.data() + 2, sizeof(size_t));
key = "t|" + std::to_string(iden) + "|0";
key = rocksdb_key::attribute(false, iden, 0);
type_identities_wrote_as_refs.insert(iden);
}
@@ -215,7 +203,7 @@ void RocksDbSerializer::write_streaming_() {
auto write_inverse = [&](const ifcopenshell::reference_or_simple_type& v) {
if (auto* ref = std::get_if<ifcopenshell::instance_reference>(&v)) {
auto key = "v|" + to_string_fixed_width(*ref, 10) + "|" + to_string_fixed_width(decl->index_in_schema(), 4) + "|" + to_string_fixed_width(index, 2);
auto key = rocksdb_key::inverse(*ref, decl->index_in_schema(), index);
static std::string s;
uint32_t vv = name;
s.resize(sizeof(uint32_t));
@@ -247,7 +235,7 @@ void RocksDbSerializer::write_streaming_() {
storage.db->Put(
storage.wopts,
(inst.declaration().as_entity() ? "i|" : "t|") + std::to_string(inst.identity()) + "|_", s);
rocksdb_key::type_record(inst.declaration().as_entity() != nullptr, inst.identity()), s);
if (type_identities_wrote_as_refs.find(inst.identity()) != type_identities_wrote_as_refs.end()) {
// already written as reference, skip
@@ -275,13 +263,13 @@ void RocksDbSerializer::write_streaming_() {
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Put(
storage.wopts,
(decl->as_entity() ? "i|" : "t|") + std::to_string(name) + "|_", s);
rocksdb_key::type_record(decl->as_entity() != nullptr, name), s);
{
size_t v = name;
std::string s(sizeof(size_t), ' ');
memcpy(s.data(), &v, sizeof(size_t));
storage.db->Merge(storage.wopts, "t|" + std::to_string(decl->index_in_schema()), s);
storage.db->Merge(storage.wopts, rocksdb_key::type_list(decl->index_in_schema()), s);
}
// GlobalId as numeric ref to instance name, so that the guid map in