From 67932a22c358d775a740da5a1d145d9bca50687a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 13 Mar 2025 12:34:54 +0100 Subject: [PATCH] Working non-streaming serializer and read-only model --- src/ifcparse/IfcFile.cpp | 6 +- src/ifcparse/IfcParse.cpp | 5 +- src/serializers/RocksDbSerializer.cpp | 84 +++++++++++++++++++++++++++ src/serializers/RocksDbSerializer.h | 28 +++++++++ 4 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 src/serializers/RocksDbSerializer.cpp create mode 100644 src/serializers/RocksDbSerializer.h diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index af585d1499..d423542399 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -356,7 +356,6 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s std::string v; - // @todo should always be name, as we can/should not assign to identity rocksdb::Status s = db->Get(rocksdb::ReadOptions{}, (r == entityinstance_ref ? "i|" : "t|") + std::to_string(number) + "|_", &v); if (s.ok()) { size_t s; @@ -375,8 +374,9 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s inst->file_ = file; instance_cache_.insert({ {r, number}, inst }); return inst; + } else { + throw IfcException("Instance #" + boost::lexical_cast(number) + " not found"); } - throw std::runtime_error(""); } namespace { @@ -476,6 +476,8 @@ void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::If s.resize(vals.size() * sizeof(size_t)); memcpy(s.data(), vals.data(), s.size()); db->Put(rocksdb::WriteOptions{}, it->key(), s); + + it->Next(); } } } diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index a09e6b7791..f3b9389403 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -2436,15 +2436,16 @@ aggregate_of_instance::ptr IfcFile::getInverse(int instance_id, const IfcParse:: } else if constexpr (std::is_same_v, impl::rocks_db_file_storage>) { if (attribute_index == -1) { // @todo no lower/upper_bounds() implemented yet - auto prefix = "v|" + std::to_string(instance_id) + "|"; + auto prefix = "v|" + std::to_string(instance_id) + "|" + std::to_string(ent->index_in_schema()) + "|"; auto it = x.db->NewIterator(rocksdb::ReadOptions()); it->Seek(prefix); while (it->Valid() && it->key().starts_with(prefix)) { - std::vector vals(it->value().size() / sizeof(size_t)); + std::vector vals(it->value().size() / sizeof(uint32_t)); memcpy(vals.data(), it->value().data(), it->value().size()); for (auto& v : vals) { return_value->push(instance_by_id(v)); } + it->Next(); } } else { auto it = x.byref_excl_.find({ instance_id, ent->index_in_schema(), attribute_index }); diff --git a/src/serializers/RocksDbSerializer.cpp b/src/serializers/RocksDbSerializer.cpp new file mode 100644 index 0000000000..c99f450f6f --- /dev/null +++ b/src/serializers/RocksDbSerializer.cpp @@ -0,0 +1,84 @@ +#ifdef WITH_ROCKSDB + +#include "RocksDbSerializer.h" + +#include + +#include "../ifcparse/IfcLogger.h" + +RocksDbSerializer::RocksDbSerializer(IfcParse::IfcFile* file, const std::string& rocksdb_filename) + : file_(file) + , rocksdb_filename_(rocksdb_filename) +{ + /*rocksdb::Options options; + options.create_if_missing = true; + options.merge_operator.reset(new ConcatenateIdMergeOperator()); + rocksdb::Status status = rocksdb::DB::Open(options, rocksdb_filename, &db_);*/ + output_file_ = new IfcParse::IfcFile(file_->schema(), IfcParse::rocksdb, rocksdb_filename_); +} + +void RocksDbSerializer::finalize() +{ + // Build a map of instances and their references/dependencies + std::map> dependencies; + std::visit([&dependencies](const auto& m) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + for (const auto& ps : m.byref_excl_) { + for (const auto& p : ps.second) { + dependencies[p].insert(std::get<0>(ps.first)); + } + } + } + }, file_->storage_); + // Add bottom-rank nodes, inv mapping does not contain them + for (const auto& p : *file_) { + dependencies[p.first]; + } + + // Do a topological sort over the nodes + std::vector deps_topo_order; + while (dependencies.size() > 0) { + std::vector no_deps; + for (auto& ps : dependencies) { + if (ps.second.size() == 0) { + no_deps.push_back(ps.first); + } + } + + if (no_deps.size() == 0) { + throw std::runtime_error("cyclic dependencies in model, unable to serialize"); + } + + for (auto& i : no_deps) { + deps_topo_order.push_back(i); + } + + // mutate mapping + for (auto& i : no_deps) { + dependencies.erase(i); + } + for (auto& p : dependencies) { + for (auto& i : no_deps) { + p.second.erase(i); + } + } + } + + // Add them in topological order, so that add() never recurses into something not previously visited + for (auto& i : deps_topo_order) { + output_file_->addEntity(file_->instance_by_id(i), i); + } + + // Copy inverses + std::visit([this](const auto& m) { + if constexpr (std::is_same_v, IfcParse::impl::in_memory_file_storage>) { + for (auto& p : m.byref_excl_) { + std::get(output_file_->storage_).byref_excl_.insert(p); + } + } + }, file_->storage_); + + delete output_file_; +} + +#endif \ No newline at end of file diff --git a/src/serializers/RocksDbSerializer.h b/src/serializers/RocksDbSerializer.h new file mode 100644 index 0000000000..e7456d28ef --- /dev/null +++ b/src/serializers/RocksDbSerializer.h @@ -0,0 +1,28 @@ +#ifdef WITH_ROCKSDB + +#include "../serializers/serializers_api.h" +#include "../ifcgeom/Serializer.h" +#include "../ifcparse/IfcFile.h" + +#include + +class SERIALIZERS_API RocksDbSerializer : public Serializer { +private: + rocksdb::DB* db_; + std::string rocksdb_filename_; + IfcParse::IfcFile* file_; + IfcParse::IfcFile* output_file_; + +public: + RocksDbSerializer(IfcParse::IfcFile* file, const std::string& rocksdb_filename); + + virtual ~RocksDbSerializer() {} + + bool ready() { return true; } + void writeHeader() {} + + void finalize(); + void setFile(IfcParse::IfcFile*) { throw IfcParse::IfcException("Should be supplied on construction"); } +}; + +#endif \ No newline at end of file