mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-08-10 01:41:57 +00:00
serializers: skip_supertypes filter in RocksDbSerializer streaming write
Plumbed through to the Python convert_path_to_rocksdb wrapper. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -378,15 +378,24 @@ def stream2_from_string(data: str) -> Generator[dict]:
|
||||
yield inst
|
||||
|
||||
|
||||
def convert_path_to_rocksdb(ifcspf_path: Union[Path, str], rocksdb_path: Union[Path, str]) -> None:
|
||||
def convert_path_to_rocksdb(
|
||||
ifcspf_path: Union[Path, str],
|
||||
rocksdb_path: Union[Path, str],
|
||||
skip_supertypes: Optional[list[str]] = None,
|
||||
) -> None:
|
||||
"""Converts an IFC-SPF file on disk to the IfcOpenShell-specific
|
||||
RocksDB encoding. RocksDB is an embedded key-value store that allows
|
||||
partial reads and is therefore more memory efficient with larger files.
|
||||
|
||||
:param ifcspf_path: Input file path - needs to exist
|
||||
:param rocksdb_path: RocksDB file path (directory) - may exist, but result may then be invalid
|
||||
:param skip_supertypes: Optional list of entity type names. Any instance
|
||||
whose declaration is-a (or subclasses) one of these names will be
|
||||
omitted from the resulting RocksDB. Note: references to skipped
|
||||
instances become dangling - reads that walk into them will fail.
|
||||
"""
|
||||
ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), True)
|
||||
skip = list(skip_supertypes) if skip_supertypes else []
|
||||
ser = ifcopenshell_wrapper.RocksDbSerializer(str(ifcspf_path), str(rocksdb_path), True, skip)
|
||||
ser.finalize()
|
||||
|
||||
|
||||
|
||||
@@ -23,9 +23,10 @@ RocksDbSerializer::RocksDbSerializer(ifcopenshell::file* file, const std::string
|
||||
output_file_->calculate_unit_factors = false;
|
||||
}
|
||||
|
||||
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream)
|
||||
RocksDbSerializer::RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes)
|
||||
: file_(input_filename)
|
||||
, rocksdb_filename_(rocksdb_filename)
|
||||
, skip_supertypes_(skip_supertypes)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -144,6 +145,21 @@ void RocksDbSerializer::write_streaming_() {
|
||||
|
||||
const bool is_header = decl->schema() == &Header_section_schema::get_schema();
|
||||
|
||||
if (!is_header && decl->as_entity() && !skip_supertypes_.empty()) {
|
||||
bool skip = false;
|
||||
for (const auto& super : skip_supertypes_) {
|
||||
if (decl->is(super)) {
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (skip) {
|
||||
streamer.references().clear();
|
||||
streamer.inverses().clear();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<express::Base> simple_type_instances;
|
||||
|
||||
for (size_t i = 0; i < data->storage_->size(); i++) {
|
||||
|
||||
@@ -8,17 +8,21 @@
|
||||
|
||||
#include <rocksdb/db.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class SERIALIZERS_API RocksDbSerializer : public Serializer {
|
||||
private:
|
||||
rocksdb::DB* db_;
|
||||
std::string rocksdb_filename_;
|
||||
std::variant<ifcopenshell::file*, std::string> file_;
|
||||
ifcopenshell::file* output_file_;
|
||||
std::vector<std::string> skip_supertypes_;
|
||||
|
||||
void write_streaming_();
|
||||
public:
|
||||
RocksDbSerializer(ifcopenshell::file* file, const std::string& rocksdb_filename);
|
||||
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream);
|
||||
RocksDbSerializer(const std::string& input_filename, const std::string& rocksdb_filename, bool stream, const std::vector<std::string>& skip_supertypes = {});
|
||||
|
||||
virtual ~RocksDbSerializer() {}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user