Rocksdb streaming serializer connect to IfcConvert

src/ifcconvert/IfcConvert.cpp

@@ -314,7 +314,7 @@ int main(int argc, char** argv) {
 #endif
 		("input-file", new po::typed_value<path_t, char_t>(0), "input IFC file")
 		("output-file", new po::typed_value<path_t, char_t>(0), "output geometry file")
-		("stream", "Use streaming conversion (currently supported with conversion to RocksDB)")
+		("stream", "Use streaming conversion when supported (RocksDB uses it automatically)")
 		;
 	if (supports_geometry_cache) {
 		fileio_options.add_options()
@@ -606,7 +606,8 @@ int main(int argc, char** argv) {
 	auto run_document_serializer = [&](const ifcopenshell::serializers::document_serializer_info* document_serializer_info) {
 		int exit_code = EXIT_FAILURE;
 		try {
-			const bool use_input_filename = vmap.count("stream") && document_serializer_info->supports_input_filename;
+			const bool use_input_filename = document_serializer_info->supports_input_filename &&
+				(vmap.count("stream") || !document_serializer_info->supports_ifc_file);
 			if (!use_input_filename && !document_serializer_info->supports_ifc_file) {
 				throw ifcopenshell::exception("Selected document serializer requires --stream");
 			}
@@ -623,13 +624,16 @@ int main(int argc, char** argv) {
 				time(&start);
 
 				ifcopenshell::serializers::document_serializer_context context;
-				context.file = ifc_file;
+				context.file = use_input_filename ? nullptr : ifc_file;
 				context.input_filename = ifcopenshell::path::to_utf8(input_filename);
 				context.output_filename = ifcopenshell::path::to_utf8(document_serializer_info->writes_final_output ? output_filename : output_temp_filename);
 				context.schema_name = ifc_file ? ifc_file->schema()->name() : document_serializer_info->schema_name;
 				context.stream = use_input_filename;
 
 				boost::shared_ptr<Serializer> serializer = document_serializer_registry.create(output_extension_utf8, context);
+				if (serializer->is_streaming() != use_input_filename) {
+					throw ifcopenshell::exception("Selected document serializer streaming mode does not match its registry metadata");
+				}
 				logger::status("Writing " + boost::to_upper_copy(document_serializer_info->format) + " output...");
 				serializer->finalize();
 				serializer.reset();

src/ifcgeom/Serializer.h

@@ -28,6 +28,7 @@ public:
 	virtual ~Serializer() {}
 
 	virtual bool ready() = 0;
+	virtual bool is_streaming() const { return false; }
 	virtual void writeHeader() = 0;
 	virtual void finalize() = 0;
 	virtual void setFile(ifcopenshell::file*) = 0;

src/ifcopenshell-python/ifcopenshell/__init__.py

@@ -410,7 +410,7 @@ def convert_path_to_rocksdb(
         instances become dangling - reads that walk into them will fail.
     """
     skip = list(skip_supertypes) if skip_supertypes else []
-    ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), True, skip)
+    ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), skip)
     ser.finalize()
 
 

src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi

@@ -275,13 +275,18 @@ class GeometrySerializer:
     READ_BREP: Any
     READ_TRIANGULATION: Any
     def __init__(self, *args, **kwargs): ...
+    def finalize(self): ...
     def geometry_settings(self, *args): ...
     def isTesselated(self): ...
+    def is_streaming(self) -> bool: ...
     def object_id(self, o): ...
     def read(self, *args): ...
+    def ready(self): ...
+    def setFile(self, arg2): ...
     def setUnitNameAndMagnitude(self, name, magnitude): ...
     def settings(self, *args): ...
     def write(self, *args): ...
+    def writeHeader(self): ...
 
 class GltfSerializer(WriteOnlyGeometrySerializer):
     def __init__(self, filename, geometry_settings, settings): ...

src/ifcwrap/IfcGeomWrapper.i

@@ -260,13 +260,116 @@ namespace {
 %include "../ifcgeom/ConversionSettings.h"
 %include "../ifcgeom/IfcGeomElement.h"
 %include "../ifcgeom/IfcGeomRepresentation.h"
-%include "../ifcgeom/Iterator.h"
+%include "../ifcgeom/Iterator.h"
-%include "../ifcgeom/GeometrySerializer.h"
+%include "../ifcgeom/GeometrySerializer.h"
-%include "../ifcgeom/taxonomy.h"
+%include "../ifcgeom/taxonomy.h"
-%include "../ifcgeom/function_item_evaluator.h"
+%include "../ifcgeom/function_item_evaluator.h"
-
+
-%extend ifcopenshell::geometry::taxonomy::style {
+%{
-	size_t instance_id() const {
+#include "../serializers/geometry_serializer_plugin.h"
+
+class PythonPluginGeometrySerializer : public GeometrySerializer {
+public:
+	PythonPluginGeometrySerializer(
+		const std::string& extension,
+		const std::string& output_filename,
+		const std::string& output_temp_filename,
+		ifcopenshell::geometry::Settings& geometry_settings,
+		const ifcopenshell::geometry::SerializerSettings& serializer_settings
+	)
+		: GeometrySerializer(geometry_settings, serializer_settings)
+	{
+		ifcopenshell::serializers::geometry_serializer_context context{
+			output_filename,
+			output_temp_filename.empty() ? output_filename : output_temp_filename,
+			geometry_settings,
+			serializer_settings
+		};
+
+		auto& registry = ifcopenshell::serializers::geometry_serializer_registry_instance();
+		registry.configure(extension, context);
+		geometry_settings_ = context.geometry_settings;
+		serializer_ = registry.create(extension, context);
+	}
+
+	bool ready() override {
+		return serializer_->ready();
+	}
+
+	bool is_streaming() const override {
+		return serializer_->is_streaming();
+	}
+
+	void writeHeader() override {
+		serializer_->writeHeader();
+	}
+
+	void finalize() override {
+		serializer_->finalize();
+	}
+
+	void setFile(ifcopenshell::file* file) override {
+		serializer_->setFile(file);
+	}
+
+	bool isTesselated() const override {
+		return serializer_->isTesselated();
+	}
+
+	void write(const IfcGeom::TriangulationElement* element) override {
+		serializer_->write(element);
+	}
+
+	void write(const IfcGeom::BRepElement* element) override {
+		serializer_->write(element);
+	}
+
+	void setUnitNameAndMagnitude(const std::string& name, float magnitude) override {
+		serializer_->setUnitNameAndMagnitude(name, magnitude);
+	}
+
+	IfcGeom::Element* read(
+		ifcopenshell::file& file,
+		const std::string& guid,
+		const std::string& representation_id,
+		read_type rt = READ_BREP
+	) override {
+		return serializer_->read(file, guid, representation_id, rt);
+	}
+
+	std::string object_id(const IfcGeom::Element* element) override {
+		return serializer_->object_id(element);
+	}
+
+private:
+	boost::shared_ptr<GeometrySerializer> serializer_;
+};
+%}
+
+%extend GeometrySerializer {
+	bool ready() {
+		return $self->ready();
+	}
+
+	bool is_streaming() const {
+		return $self->is_streaming();
+	}
+
+	void writeHeader() {
+		$self->writeHeader();
+	}
+
+	void finalize() {
+		$self->finalize();
+	}
+
+	void setFile(ifcopenshell::file* file) {
+		$self->setFile(file);
+	}
+}
+
+%extend ifcopenshell::geometry::taxonomy::style {
+	size_t instance_id() const {
 		if (!self->instance) {
 			return 0;
 		}

src/serializers/RocksDbSerializer.cpp

@@ -6,25 +6,8 @@
 
 #include "../ifcparse/logger.h"
 
-RocksDbSerializer::RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename)
+RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, const std::vector<std::string>& skip_supertypes)
-	: file_(file)
+	: input_filename_(input_filename)
-	, rocksdb_filename_(rocksdb_filename)
-{
-	/*rocksdb::Options options;
-	options.create_if_missing = true;
-	options.merge_operator.reset(new ConcatenateIdMergeOperator());
-	rocksdb::status status = rocksdb::DB::Open(options, rocksdb_filename, &db_);*/
-
-	output_file_ = new ifcopenshell::file(file->schema(), ifcopenshell::FT_ROCKSDB, rocksdb_filename_);
-
-	// We promise never to add the same instance twice
-	output_file_->check_existance_before_adding = false;
-	// We only copy one file into an empty container so units will match
-	output_file_->calculate_unit_factors = false;
-}
-
-RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes)
-	: file_(input_filename)
 	, rocksdb_filename_(rocksdb_filename)
 	, skip_supertypes_(skip_supertypes)
 {
@@ -123,13 +106,11 @@ namespace {
 }
 
 void RocksDbSerializer::write_streaming_() {
-	const auto& input_filename = std::get<std::string>(file_);
-
 	ifcopenshell::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr);
 
 	std::string tmp;
 
-	ifcopenshell::instance_streamer streamer(input_filename);
+	ifcopenshell::instance_streamer streamer(input_filename_);
 
 	// We do not want to coerce attribute counts here, because we want
 	// to store exactly what is in the file for validation purposes
@@ -304,11 +285,7 @@ void RocksDbSerializer::write_streaming_() {
 }
 
 void RocksDbSerializer::finalize() {
-	if (file_.index() == 0) {
+	write_streaming_();
-        throw std::runtime_error("Non-streaming mode no longer supported");
-	} else {
-		write_streaming_();
-	}
 }
 
 

src/serializers/RocksDbSerializer.h

@@ -4,34 +4,29 @@
 
 #include "../serializers/serializers_api.h"
 #include "../ifcgeom/Serializer.h"
-#include "../ifcparse/file.h"
-
-#include <rocksdb/db.h>
 
 #include <string>
 #include <vector>
 
 class SERIALIZERS_API RocksDbSerializer : public Serializer {
 private:
-	rocksdb::DB* db_;
+	std::string input_filename_;
 	std::string rocksdb_filename_;
-	std::variant<ifcopenshell::file*, std::string> file_;
-	ifcopenshell::file* output_file_;
 	std::vector<std::string> skip_supertypes_;
 
 	void write_streaming_();
 public:
-	RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename);
+	RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, const std::vector<std::string>& skip_supertypes = {});
-	RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes = {});
 
 	virtual ~RocksDbSerializer() {}
 
-	bool ready() { return true; }
+	bool ready() override { return true; }
-	void writeHeader() {}
+	bool is_streaming() const override { return true; }
+	void writeHeader() override {}
 
-	void finalize();
+	void finalize() override;
-	void setFile(ifcopenshell::file*) { throw ifcopenshell::exception("Should be supplied on construction"); }
+	void setFile(ifcopenshell::file*) override { throw ifcopenshell::exception("Streaming serializer uses input filename supplied on construction"); }
 };
 
 #endif
-#endif
+#endif

src/serializers/document_rdb_plugin.cpp

@@ -28,10 +28,10 @@
 namespace {
 
 boost::shared_ptr<Serializer> create_serializer(const ifcopenshell::serializers::document_serializer_context& context) {
-	if (!context.stream || context.input_filename.empty()) {
+	if (context.input_filename.empty()) {
-		throw ifcopenshell::exception("RocksDB document serializer requires --stream input");
+		throw ifcopenshell::exception("RocksDB document serializer requires an input filename");
 	}
-	return boost::make_shared<RocksDbSerializer>(context.input_filename, context.output_filename, true);
+	return boost::make_shared<RocksDbSerializer>(context.input_filename, context.output_filename);
 }
 
 }