mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-09-12 14:33:28 +00:00
serializers: skip_supertypes filter in RocksDbSerializer streaming write
Plumbed through to the Python convert_path_to_rocksdb wrapper. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -378,15 +378,24 @@ def stream2_from_string(data: str) -> Generator[dict]:
|
|||||||
yield inst
|
yield inst
|
||||||
|
|
||||||
|
|
||||||
def convert_path_to_rocksdb(ifcspf_path: Union[Path, str], rocksdb_path: Union[Path, str]) -> None:
|
def convert_path_to_rocksdb(
|
||||||
|
ifcspf_path: Union[Path, str],
|
||||||
|
rocksdb_path: Union[Path, str],
|
||||||
|
skip_supertypes: Optional[list[str]] = None,
|
||||||
|
) -> None:
|
||||||
"""Converts an IFC-SPF file on disk to the IfcOpenShell-specific
|
"""Converts an IFC-SPF file on disk to the IfcOpenShell-specific
|
||||||
RocksDB encoding. RocksDB is an embedded key-value store that allows
|
RocksDB encoding. RocksDB is an embedded key-value store that allows
|
||||||
partial reads and is therefore more memory efficient with larger files.
|
partial reads and is therefore more memory efficient with larger files.
|
||||||
|
|
||||||
:param ifcspf_path: Input file path - needs to exist
|
:param ifcspf_path: Input file path - needs to exist
|
||||||
:param rocksdb_path: RocksDB file path (directory) - may exist, but result may then be invalid
|
:param rocksdb_path: RocksDB file path (directory) - may exist, but result may then be invalid
|
||||||
|
:param skip_supertypes: Optional list of entity type names. Any instance
|
||||||
|
whose declaration is-a (or subclasses) one of these names will be
|
||||||
|
omitted from the resulting RocksDB. Note: references to skipped
|
||||||
|
instances become dangling - reads that walk into them will fail.
|
||||||
"""
|
"""
|
||||||
ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), True)
|
skip = list(skip_supertypes) if skip_supertypes else []
|
||||||
|
ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), True, skip)
|
||||||
ser.finalize()
|
ser.finalize()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,9 +23,10 @@ RocksDbSerializer::RocksDbSerializer(ifcopenshell::file* file, const std::string
|
|||||||
output_file_->calculate_unit_factors = false;
|
output_file_->calculate_unit_factors = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream)
|
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes)
|
||||||
: file_(input_filename)
|
: file_(input_filename)
|
||||||
, rocksdb_filename_(rocksdb_filename)
|
, rocksdb_filename_(rocksdb_filename)
|
||||||
|
, skip_supertypes_(skip_supertypes)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,6 +145,21 @@ void RocksDbSerializer::write_streaming_() {
|
|||||||
|
|
||||||
const bool is_header = decl->schema() == &Header_section_schema::get_schema();
|
const bool is_header = decl->schema() == &Header_section_schema::get_schema();
|
||||||
|
|
||||||
|
if (!is_header && decl->as_entity() && !skip_supertypes_.empty()) {
|
||||||
|
bool skip = false;
|
||||||
|
for (const auto& super : skip_supertypes_) {
|
||||||
|
if (decl->is(super)) {
|
||||||
|
skip = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (skip) {
|
||||||
|
streamer.references().clear();
|
||||||
|
streamer.inverses().clear();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
std::vector<express::Base> simple_type_instances;
|
std::vector<express::Base> simple_type_instances;
|
||||||
|
|
||||||
for (size_t i = 0; i < data->storage_->size(); i++) {
|
for (size_t i = 0; i < data->storage_->size(); i++) {
|
||||||
|
|||||||
@@ -8,17 +8,21 @@
|
|||||||
|
|
||||||
#include <rocksdb/db.h>
|
#include <rocksdb/db.h>
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
class SERIALIZERS_API RocksDbSerializer : public Serializer {
|
class SERIALIZERS_API RocksDbSerializer : public Serializer {
|
||||||
private:
|
private:
|
||||||
rocksdb::DB* db_;
|
rocksdb::DB* db_;
|
||||||
std::string rocksdb_filename_;
|
std::string rocksdb_filename_;
|
||||||
std::variant<ifcopenshell::file*, std::string> file_;
|
std::variant<ifcopenshell::file*, std::string> file_;
|
||||||
ifcopenshell::file* output_file_;
|
ifcopenshell::file* output_file_;
|
||||||
|
std::vector<std::string> skip_supertypes_;
|
||||||
|
|
||||||
void write_streaming_();
|
void write_streaming_();
|
||||||
public:
|
public:
|
||||||
RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename);
|
RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename);
|
||||||
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream);
|
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes = {});
|
||||||
|
|
||||||
virtual ~RocksDbSerializer() {}
|
virtual ~RocksDbSerializer() {}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user