From 18b79a4360942d3b2e9cb2b11302e57b5bfe3710 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Fri, 8 May 2026 10:57:58 +0200 Subject: [PATCH] Rocksdb streaming serializer connect to IfcConvert --- src/ifcconvert/IfcConvert.cpp | 10 +- src/ifcgeom/Serializer.h | 1 + .../ifcopenshell/__init__.py | 2 +- .../ifcopenshell/ifcopenshell_wrapper.pyi | 5 + src/ifcwrap/IfcGeomWrapper.i | 117 ++++++++++++++++-- src/serializers/RocksDbSerializer.cpp | 31 +---- src/serializers/RocksDbSerializer.h | 21 ++-- src/serializers/document_rdb_plugin.cpp | 6 +- 8 files changed, 139 insertions(+), 54 deletions(-) diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index 265af91b73..0578a24ea4 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -314,7 +314,7 @@ int main(int argc, char** argv) { #endif ("input-file", new po::typed_value(0), "input IFC file") ("output-file", new po::typed_value(0), "output geometry file") - ("stream", "Use streaming conversion (currently supported with conversion to RocksDB)") + ("stream", "Use streaming conversion when supported (RocksDB uses it automatically)") ; if (supports_geometry_cache) { fileio_options.add_options() @@ -606,7 +606,8 @@ int main(int argc, char** argv) { auto run_document_serializer = [&](const ifcopenshell::serializers::document_serializer_info* document_serializer_info) { int exit_code = EXIT_FAILURE; try { - const bool use_input_filename = vmap.count("stream") && document_serializer_info->supports_input_filename; + const bool use_input_filename = document_serializer_info->supports_input_filename && + (vmap.count("stream") || !document_serializer_info->supports_ifc_file); if (!use_input_filename && !document_serializer_info->supports_ifc_file) { throw ifcopenshell::exception("Selected document serializer requires --stream"); } @@ -623,13 +624,16 @@ int main(int argc, char** argv) { time(&start); ifcopenshell::serializers::document_serializer_context context; - context.file = ifc_file; + context.file = use_input_filename ? nullptr : ifc_file; context.input_filename = ifcopenshell::path::to_utf8(input_filename); context.output_filename = ifcopenshell::path::to_utf8(document_serializer_info->writes_final_output ? output_filename : output_temp_filename); context.schema_name = ifc_file ? ifc_file->schema()->name() : document_serializer_info->schema_name; context.stream = use_input_filename; boost::shared_ptr serializer = document_serializer_registry.create(output_extension_utf8, context); + if (serializer->is_streaming() != use_input_filename) { + throw ifcopenshell::exception("Selected document serializer streaming mode does not match its registry metadata"); + } logger::status("Writing " + boost::to_upper_copy(document_serializer_info->format) + " output..."); serializer->finalize(); serializer.reset(); diff --git a/src/ifcgeom/Serializer.h b/src/ifcgeom/Serializer.h index 3db0069951..b06f7225b2 100644 --- a/src/ifcgeom/Serializer.h +++ b/src/ifcgeom/Serializer.h @@ -28,6 +28,7 @@ public: virtual ~Serializer() {} virtual bool ready() = 0; + virtual bool is_streaming() const { return false; } virtual void writeHeader() = 0; virtual void finalize() = 0; virtual void setFile(ifcopenshell::file*) = 0; diff --git a/src/ifcopenshell-python/ifcopenshell/__init__.py b/src/ifcopenshell-python/ifcopenshell/__init__.py index 68083e9a48..5eea87fb65 100644 --- a/src/ifcopenshell-python/ifcopenshell/__init__.py +++ b/src/ifcopenshell-python/ifcopenshell/__init__.py @@ -410,7 +410,7 @@ def convert_path_to_rocksdb( instances become dangling - reads that walk into them will fail. """ skip = list(skip_supertypes) if skip_supertypes else [] - ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), True, skip) + ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), skip) ser.finalize() diff --git a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi index 0d0b7a202a..1b2c66cc06 100644 --- a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi +++ b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi @@ -275,13 +275,18 @@ class GeometrySerializer: READ_BREP: Any READ_TRIANGULATION: Any def __init__(self, *args, **kwargs): ... + def finalize(self): ... def geometry_settings(self, *args): ... def isTesselated(self): ... + def is_streaming(self) -> bool: ... def object_id(self, o): ... def read(self, *args): ... + def ready(self): ... + def setFile(self, arg2): ... def setUnitNameAndMagnitude(self, name, magnitude): ... def settings(self, *args): ... def write(self, *args): ... + def writeHeader(self): ... class GltfSerializer(WriteOnlyGeometrySerializer): def __init__(self, filename, geometry_settings, settings): ... diff --git a/src/ifcwrap/IfcGeomWrapper.i b/src/ifcwrap/IfcGeomWrapper.i index 66dd8d284b..846634fcc6 100644 --- a/src/ifcwrap/IfcGeomWrapper.i +++ b/src/ifcwrap/IfcGeomWrapper.i @@ -260,13 +260,116 @@ namespace { %include "../ifcgeom/ConversionSettings.h" %include "../ifcgeom/IfcGeomElement.h" %include "../ifcgeom/IfcGeomRepresentation.h" -%include "../ifcgeom/Iterator.h" -%include "../ifcgeom/GeometrySerializer.h" -%include "../ifcgeom/taxonomy.h" -%include "../ifcgeom/function_item_evaluator.h" - -%extend ifcopenshell::geometry::taxonomy::style { - size_t instance_id() const { +%include "../ifcgeom/Iterator.h" +%include "../ifcgeom/GeometrySerializer.h" +%include "../ifcgeom/taxonomy.h" +%include "../ifcgeom/function_item_evaluator.h" + +%{ +#include "../serializers/geometry_serializer_plugin.h" + +class PythonPluginGeometrySerializer : public GeometrySerializer { +public: + PythonPluginGeometrySerializer( + const std::string& extension, + const std::string& output_filename, + const std::string& output_temp_filename, + ifcopenshell::geometry::Settings& geometry_settings, + const ifcopenshell::geometry::SerializerSettings& serializer_settings + ) + : GeometrySerializer(geometry_settings, serializer_settings) + { + ifcopenshell::serializers::geometry_serializer_context context{ + output_filename, + output_temp_filename.empty() ? output_filename : output_temp_filename, + geometry_settings, + serializer_settings + }; + + auto& registry = ifcopenshell::serializers::geometry_serializer_registry_instance(); + registry.configure(extension, context); + geometry_settings_ = context.geometry_settings; + serializer_ = registry.create(extension, context); + } + + bool ready() override { + return serializer_->ready(); + } + + bool is_streaming() const override { + return serializer_->is_streaming(); + } + + void writeHeader() override { + serializer_->writeHeader(); + } + + void finalize() override { + serializer_->finalize(); + } + + void setFile(ifcopenshell::file* file) override { + serializer_->setFile(file); + } + + bool isTesselated() const override { + return serializer_->isTesselated(); + } + + void write(const IfcGeom::TriangulationElement* element) override { + serializer_->write(element); + } + + void write(const IfcGeom::BRepElement* element) override { + serializer_->write(element); + } + + void setUnitNameAndMagnitude(const std::string& name, float magnitude) override { + serializer_->setUnitNameAndMagnitude(name, magnitude); + } + + IfcGeom::Element* read( + ifcopenshell::file& file, + const std::string& guid, + const std::string& representation_id, + read_type rt = READ_BREP + ) override { + return serializer_->read(file, guid, representation_id, rt); + } + + std::string object_id(const IfcGeom::Element* element) override { + return serializer_->object_id(element); + } + +private: + boost::shared_ptr serializer_; +}; +%} + +%extend GeometrySerializer { + bool ready() { + return $self->ready(); + } + + bool is_streaming() const { + return $self->is_streaming(); + } + + void writeHeader() { + $self->writeHeader(); + } + + void finalize() { + $self->finalize(); + } + + void setFile(ifcopenshell::file* file) { + $self->setFile(file); + } +} + +%extend ifcopenshell::geometry::taxonomy::style { + size_t instance_id() const { if (!self->instance) { return 0; } diff --git a/src/serializers/RocksDbSerializer.cpp b/src/serializers/RocksDbSerializer.cpp index bbc8e38057..c7247938f1 100644 --- a/src/serializers/RocksDbSerializer.cpp +++ b/src/serializers/RocksDbSerializer.cpp @@ -6,25 +6,8 @@ #include "../ifcparse/logger.h" -RocksDbSerializer::RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename) - : file_(file) - , rocksdb_filename_(rocksdb_filename) -{ - /*rocksdb::Options options; - options.create_if_missing = true; - options.merge_operator.reset(new ConcatenateIdMergeOperator()); - rocksdb::status status = rocksdb::DB::Open(options, rocksdb_filename, &db_);*/ - - output_file_ = new ifcopenshell::file(file->schema(), ifcopenshell::FT_ROCKSDB, rocksdb_filename_); - - // We promise never to add the same instance twice - output_file_->check_existance_before_adding = false; - // We only copy one file into an empty container so units will match - output_file_->calculate_unit_factors = false; -} - -RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector& skip_supertypes) - : file_(input_filename) +RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, const std::vector& skip_supertypes) + : input_filename_(input_filename) , rocksdb_filename_(rocksdb_filename) , skip_supertypes_(skip_supertypes) { @@ -123,13 +106,11 @@ namespace { } void RocksDbSerializer::write_streaming_() { - const auto& input_filename = std::get(file_); - ifcopenshell::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr); std::string tmp; - ifcopenshell::instance_streamer streamer(input_filename); + ifcopenshell::instance_streamer streamer(input_filename_); // We do not want to coerce attribute counts here, because we want // to store exactly what is in the file for validation purposes @@ -304,11 +285,7 @@ void RocksDbSerializer::write_streaming_() { } void RocksDbSerializer::finalize() { - if (file_.index() == 0) { - throw std::runtime_error("Non-streaming mode no longer supported"); - } else { - write_streaming_(); - } + write_streaming_(); } diff --git a/src/serializers/RocksDbSerializer.h b/src/serializers/RocksDbSerializer.h index 66caf3edb2..31a4249cbc 100644 --- a/src/serializers/RocksDbSerializer.h +++ b/src/serializers/RocksDbSerializer.h @@ -4,34 +4,29 @@ #include "../serializers/serializers_api.h" #include "../ifcgeom/Serializer.h" -#include "../ifcparse/file.h" - -#include #include #include class SERIALIZERS_API RocksDbSerializer : public Serializer { private: - rocksdb::DB* db_; + std::string input_filename_; std::string rocksdb_filename_; - std::variant file_; - ifcopenshell::file* output_file_; std::vector skip_supertypes_; void write_streaming_(); public: - RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename); - RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector& skip_supertypes = {}); + RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, const std::vector& skip_supertypes = {}); virtual ~RocksDbSerializer() {} - bool ready() { return true; } - void writeHeader() {} + bool ready() override { return true; } + bool is_streaming() const override { return true; } + void writeHeader() override {} - void finalize(); - void setFile(ifcopenshell::file*) { throw ifcopenshell::exception("Should be supplied on construction"); } + void finalize() override; + void setFile(ifcopenshell::file*) override { throw ifcopenshell::exception("Streaming serializer uses input filename supplied on construction"); } }; #endif -#endif \ No newline at end of file +#endif diff --git a/src/serializers/document_rdb_plugin.cpp b/src/serializers/document_rdb_plugin.cpp index 291e4c721f..ffcefcd1e8 100644 --- a/src/serializers/document_rdb_plugin.cpp +++ b/src/serializers/document_rdb_plugin.cpp @@ -28,10 +28,10 @@ namespace { boost::shared_ptr create_serializer(const ifcopenshell::serializers::document_serializer_context& context) { - if (!context.stream || context.input_filename.empty()) { - throw ifcopenshell::exception("RocksDB document serializer requires --stream input"); + if (context.input_filename.empty()) { + throw ifcopenshell::exception("RocksDB document serializer requires an input filename"); } - return boost::make_shared(context.input_filename, context.output_filename, true); + return boost::make_shared(context.input_filename, context.output_filename); } }