Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-22 02:50:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcparse: make deleting and creating instances work on RocksDB-backed files

Browse Source 
file.remove() on a RocksDB-backed file segfaulted. Reducing it turned up
five gaps that each made editing such a file crash or silently do
nothing:

- process_deletion_inverse() decoded the v| inverse-record values as
  size_t while the serializer, register_inverse(), unregister_inverse()
  and instances_by_reference() use uint32_t, so std::find failed and
  vals.erase(end()) was undefined behaviour. It also took the DeleteRange
  end from an iterator that is invalid when the instance has no inverse
  records. Decode as uint32_t, remove every occurrence guarded on
  "found", and derive the range end from the prefix itself.
- attribute_value::size() ignored storage_model_, so every aggregate
  assignment on a RocksDB instance threw "Invalid variant index" from
  set_attribute_value(). Branch on the storage model like the sibling
  accessors and count the deserialized aggregate.
- rocks_db_file_storage::create() was a stub returning an empty handle,
  which anything creating an instance then dereferenced. Implement it
  after in_memory_file_storage::create().
- max_id_ is only initialised by the in-memory parse, so a RocksDB file
  would have handed out ids that overwrite existing instances.
  Implement the recalculate_id_counter() stub per backend and run it
  once before the first fresh_id() on RocksDB.
- byid_.erase() was a no-op: set_to_map_transformer::erase() and
  rocksdb_set_view::erase() were stubs. The deleted instance's attribute
  keys and cached handle survived, entity_names() still listed it and
  reopening the database threw. Erase deletes every key under the
  instance's prefix; the transformer forwards to it and takes an
  on-erase hook the storage uses to drop the cached handle.

root.remove_product on the first 200 products of a 61 MB model now
leaves the same surviving ids and inverse counts whether the file was
opened from SPF or converted to RocksDB.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HNrXDmR88wKPCYwGE21SyH
This commit is contained in:
Dion Moult
2026-09-16 09:03:06 +10:00
parent 336cb80394
commit 4b642c5e00
6 changed files with 161 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/file.cpp
+68 -35
View File
@@ -155,7 +155,15 @@ ifcopenshell::impl::rocks_db_file_storage::rocks_db_file_storage(const std::stri
: db(init_db(filepath, readonly))
, file(ffile)
, instance_ids_(db.get(), "i|")
, instance_by_name_(&instance_ids_, [this](size_t v) { return assert_existance(v, entityinstance_ref); })
, instance_by_name_(
&instance_ids_,
[this](size_t v) { return assert_existance(v, entityinstance_ref); },
[this](size_t v) {
// The instance's keys are gone from the database; drop the
// cached handle so lookups don't keep resolving it.
std::lock_guard<std::mutex> lock(instance_cache_mutex_);
instance_cache_.erase((uint32_t)v);
})
, bytype_(db.get(), "t|")
, byguid_internal_(db.get(), "g|"),
byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, entityinstance_ref); }, [](const express::base& v) { return v.identity(); })
@@ -207,22 +215,21 @@ void ifcopenshell::impl::rocks_db_file_storage::process_deletion_inverse(const e
auto id = inst.id();
{
// compute next prefix that does not start with v|{id}|
// Delete every record referencing inst: all keys under v|{id}|. The
// prefix with its last byte incremented is the exclusive upper bound
// ('}' follows '|'), so no iterator is needed to find the range end.
auto prefix = "v|" + std::to_string(id) + "|";
auto it = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid()) {
it->Next();
if (!it->key().starts_with(prefix)) {
break;
}
}
auto upper_bound = prefix;
upper_bound.back() = '}';
rocksdb::WriteBatch batch;
batch.DeleteRange(prefix, it->key());
batch.DeleteRange(prefix, upper_bound);
db->Write(wopts, &batch);
}
// Delete the records inst contributed through its own attributes: drop
// its id from the value lists of every instance it references. The values
// are uint32_t ids, as written by the serializer and register_inverse().
// This is based on traversal which needs instances to still be contained in the map.
// another option would be to keep byid intact for the remainder of this loop
auto entity_attributes = traverse(inst, 1);
@@ -233,26 +240,24 @@ void ifcopenshell::impl::rocks_db_file_storage::process_deletion_inverse(const e
const unsigned int name = entity_attribute.id();
// Do not update inverses for simple types (which have id()==0 in IfcOpenShell).
if (name != 0) {
// Find instances entity -> other
// and update inverses from entity into other
auto prefix = "v|" + std::to_string(name) + "|";
auto it = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
std::string s = it->value().ToString();
{
auto prefix = "v|" + std::to_string(name) + "|";
auto it = std::unique_ptr<rocksdb::Iterator>(db->NewIterator(rocksdb::ReadOptions()));
it->Seek(prefix);
while (it->Valid() && it->key().starts_with(prefix)) {
std::string s = it->value().ToString();
// Iterator are snapshotted? So don't get invalidated?
std::vector<size_t> vals(s.size() / sizeof(size_t));
memcpy(vals.data(), s.data(), s.size());
vals.erase(std::find(vals.begin(), vals.end(), (size_t)id));
s.resize(vals.size() * sizeof(size_t));
// Iterator are snapshotted? So don't get invalidated?
std::vector<uint32_t> vals(s.size() / sizeof(uint32_t));
memcpy(vals.data(), s.data(), s.size());
auto removed = std::remove(vals.begin(), vals.end(), (uint32_t)id);
if (removed != vals.end()) {
vals.erase(removed, vals.end());
s.resize(vals.size() * sizeof(uint32_t));
memcpy(s.data(), vals.data(), s.size());
db->Put(wopts, it->key(), s);
it->Next();
}
it->Next();
}
}
}
@@ -314,19 +319,47 @@ ifcopenshell::filetype ifcopenshell::guess_file_type(const std::string& fn) {
}
express::base ifcopenshell::impl::rocks_db_file_storage::create(const ifcopenshell::declaration* decl, int id) {
#ifndef IFOPSH_WITH_ROCKSDB
(void)decl;
(void)id;
return express::base{};
/*
if (decl->as_entity() || decl->as_type_declaration()) {
auto* inst = file->schema()->instantiate(decl, rocks_db_attribute_storage{});
// @todo maybe this needs to be set to file? In order to have a context (ie. rocksdb::db*) to write to?
inst->file_ = nullptr;
return file->add_entity(inst);
throw exception("RocksDB support not compiled in");
#else
// Mirrors in_memory_file_storage::create(). The instance's attributes
// live in the database (written by set_attribute_value(), read back on
// access), so the cache only has to keep the identity of the handle
// stable: assert_existance() can reload it from the type record that
// add_type_ref() writes.
uint32_t instance_name;
if (decl->as_entity() != nullptr) {
if (id == -1) {
if (!id_counter_recalculated_) {
file->recalculate_id_counter();
id_counter_recalculated_ = true;
}
instance_name = file->fresh_id();
} else {
instance_name = id;
}
} else if (decl->as_type_declaration() != nullptr) {
instance_name = 0;
} else {
throw std::runtime_error("Requires and entity or type declaration");
}
*/
auto data = ifcopenshell::make_pointer_type<instance_data>(file, decl, instance_name, rocks_db_attribute_storage{});
{
std::lock_guard<std::mutex> lock(instance_cache_mutex_);
if (instance_name) {
instance_cache_.insert({instance_name, data});
} else {
type_instance_cache_.insert({data->identity(), data});
}
}
express::base inst(data);
add_type_ref(inst);
return inst;
#endif
}
express::base ifcopenshell::impl::in_memory_file_storage::create(const ifcopenshell::declaration* decl, int id) {