Rocksdb streaming serializer connect to IfcConvert

This commit is contained in:
Thomas Krijnen
2026-05-08 10:57:58 +02:00
parent 05cd19592d
commit 18b79a4360
8 changed files with 139 additions and 54 deletions
+4 -27
View File
@@ -6,25 +6,8 @@
#include "../ifcparse/logger.h"
RocksDbSerializer::RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename)
: file_(file)
, rocksdb_filename_(rocksdb_filename)
{
/*rocksdb::Options options;
options.create_if_missing = true;
options.merge_operator.reset(new ConcatenateIdMergeOperator());
rocksdb::status status = rocksdb::DB::Open(options, rocksdb_filename, &db_);*/
output_file_ = new ifcopenshell::file(file->schema(), ifcopenshell::FT_ROCKSDB, rocksdb_filename_);
// We promise never to add the same instance twice
output_file_->check_existance_before_adding = false;
// We only copy one file into an empty container so units will match
output_file_->calculate_unit_factors = false;
}
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes)
: file_(input_filename)
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, const std::vector<std::string>& skip_supertypes)
: input_filename_(input_filename)
, rocksdb_filename_(rocksdb_filename)
, skip_supertypes_(skip_supertypes)
{
@@ -123,13 +106,11 @@ namespace {
}
void RocksDbSerializer::write_streaming_() {
const auto& input_filename = std::get<std::string>(file_);
ifcopenshell::impl::rocks_db_file_storage storage(rocksdb_filename_, nullptr);
std::string tmp;
ifcopenshell::instance_streamer streamer(input_filename);
ifcopenshell::instance_streamer streamer(input_filename_);
// We do not want to coerce attribute counts here, because we want
// to store exactly what is in the file for validation purposes
@@ -304,11 +285,7 @@ void RocksDbSerializer::write_streaming_() {
}
void RocksDbSerializer::finalize() {
if (file_.index() == 0) {
throw std::runtime_error("Non-streaming mode no longer supported");
} else {
write_streaming_();
}
write_streaming_();
}
+8 -13
View File
@@ -4,34 +4,29 @@
#include "../serializers/serializers_api.h"
#include "../ifcgeom/Serializer.h"
#include "../ifcparse/file.h"
#include <rocksdb/db.h>
#include <string>
#include <vector>
class SERIALIZERS_API RocksDbSerializer : public Serializer {
private:
rocksdb::DB* db_;
std::string input_filename_;
std::string rocksdb_filename_;
std::variant<ifcopenshell::file*, std::string> file_;
ifcopenshell::file* output_file_;
std::vector<std::string> skip_supertypes_;
void write_streaming_();
public:
RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename);
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes = {});
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, const std::vector<std::string>& skip_supertypes = {});
virtual ~RocksDbSerializer() {}
bool ready() { return true; }
void writeHeader() {}
bool ready() override { return true; }
bool is_streaming() const override { return true; }
void writeHeader() override {}
void finalize();
void setFile(ifcopenshell::file*) { throw ifcopenshell::exception("Should be supplied on construction"); }
void finalize() override;
void setFile(ifcopenshell::file*) override { throw ifcopenshell::exception("Streaming serializer uses input filename supplied on construction"); }
};
#endif
#endif
#endif
+3 -3
View File
@@ -28,10 +28,10 @@
namespace {
boost::shared_ptr<Serializer> create_serializer(const ifcopenshell::serializers::document_serializer_context& context) {
if (!context.stream || context.input_filename.empty()) {
throw ifcopenshell::exception("RocksDB document serializer requires --stream input");
if (context.input_filename.empty()) {
throw ifcopenshell::exception("RocksDB document serializer requires an input filename");
}
return boost::make_shared<RocksDbSerializer>(context.input_filename, context.output_filename, true);
return boost::make_shared<RocksDbSerializer>(context.input_filename, context.output_filename);
}
}