Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-06 07:51:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

RocksDB zstd encryption; cache tuning; readonly open mode

Browse Source
This commit is contained in:
Thomas Krijnen
2025-09-03 11:11:55 +02:00
parent a831884ee6
commit 59a5bf2344
15 changed files with 170 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmake/CMakeLists.txt
+38 -5
View File
@@ -84,6 +84,7 @@ option(USD_SUPPORT "Build IfcConvert with USD support (requires pixar's USD libr
option(CITYJSON_SUPPORT "Build IfcConvert with CityJSON support (requires CityJSON library)." OFF)
option(WITH_RELATIONSHIP_VALIDATION "Build IfcConvert with option to validate geometrical relationships." OFF)
option(WITH_ROCKSDB "Support a RocksDB key-value store as a file backend in IfcOpenShell" OFF)
option(WITH_ZSTD "Use Zstd compression in RocksDB writes" OFF)
option(USERSPACE_PYTHON_PREFIX "Installs IfcPython for the current user only instead of system-wide." OFF)
option(ADD_COMMIT_SHA "Add commit sha and branch in version number, warning results in many rebuilds, requires git" OFF)
@@ -315,6 +316,8 @@ endif(USD_SUPPORT)
if (WITH_ROCKSDB)
UNIFY_ENVVARS_AND_CACHE(ROCKSDB_INCLUDE_DIR)
UNIFY_ENVVARS_AND_CACHE(ROCKSDB_LIBRARY_DIR)
UNIFY_ENVVARS_AND_CACHE(ZSTD_INCLUDE_DIR)
UNIFY_ENVVARS_AND_CACHE(ZSTD_LIBRARY_DIR)
if("${ROCKSDB_INCLUDE_DIR}" STREQUAL "")
find_path(ROCKSDB_INCLUDE_DIR rocksdb/db.h
@@ -342,9 +345,39 @@ if (WITH_ROCKSDB)
message(FATAL_ERROR "Unable to find rocksdb library in: ${ROCKSDB_LIBRARY_DIR}")
endif()
# @todo IFOPSH_ prefix?
add_definitions(-DWITH_ROCKSDB)
set(SWIG_DEFINES ${SWIG_DEFINES} -DWITH_ROCKSDB)
add_definitions(-DIFOPSH_WITH_ROCKSDB)
set(SWIG_DEFINES ${SWIG_DEFINES} -DIFOPSH_WITH_ROCKSDB)
if (WITH_ZSTD)
if("${ZSTD_INCLUDE_DIR}" STREQUAL "")
find_path(ZSTD_INCLUDE_DIR zstd.h
PATHS
/usr/include
/usr/local/include
REQUIRED
)
if(ZSTD_INCLUDE_DIR)
message(STATUS "Found zstd include files in: ${ZSTD_INCLUDE_DIR}")
else()
message(FATAL_ERROR "Unable to find zstd include directory, specify ZSTD_INCLUDE_DIR manually.")
endif()
else()
set(ZSTD_INCLUDE_DIR ${ZSTD_INCLUDE_DIR} CACHE FILEPATH "zstd header files")
message(STATUS "Looking for zstd include files in: ${ZSTD_INCLUDE_DIR}")
endif()
find_library(ZSTD_LIBRARY
NAMES zstd zstd_static
PATHS ${ZSTD_LIBRARY_DIR})
if(ZSTD_LIBRARY)
message(STATUS "zstd library ${ZSTD_LIBRARY} found in: ${ZSTD_LIBRARY_DIR}")
else()
message(FATAL_ERROR "Unable to find rocksdb library in: ${ZSTD_LIBRARY_DIR}")
endif()
add_definitions(-DIFOPSH_WITH_ROCKSDB_ZSTD)
set(SWIG_DEFINES ${SWIG_DEFINES} -DIFOPSH_WITH_ROCKSDB_ZSTD)
endif()
if(WIN32)
set(RPCRT_LIBRARIES "rpcrt4.lib" "shlwapi.lib")
@@ -898,7 +931,7 @@ endif()
include_directories(${INCLUDE_DIRECTORIES} ${OCC_INCLUDE_DIR} ${OPENCOLLADA_INCLUDE_DIRS}
${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${JSON_INCLUDE_DIR} ${HDF5_INCLUDE_DIR}
${EIGEN_DIR} ${CGAL_INCLUDE_DIR} ${GMP_INCLUDE_DIR} ${MPFR_INCLUDE_DIR} ${USD_INCLUDE_DIR}
${TBB_INCLUDE_DIR} ${ROCKSDB_INCLUDE_DIR}
${TBB_INCLUDE_DIR} ${ROCKSDB_INCLUDE_DIR} ${ZSTD_INCLUDE_DIR}
)
if(NOT SCHEMA_VERSIONS)
@@ -1061,7 +1094,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
endif()
add_library(IfcParse ${IFCPARSE_FILES})
target_link_libraries(IfcParse ${ROCKSDB_LIBRARY} ${RPCRT_LIBRARIES} ${STDCPPFS})
target_link_libraries(IfcParse ${ROCKSDB_LIBRARY} ${ZSTD_LIBRARY} ${RPCRT_LIBRARIES} ${STDCPPFS})
set_target_properties(IfcParse PROPERTIES COMPILE_FLAGS -DIFC_PARSE_EXPORTS VERSION "${PROJECT_VERSION}" SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}")
if(WASM_BUILD)
src/ifcopenshell-python/ifcopenshell/__init__.py
+2 -2
View File
@@ -132,7 +132,7 @@ def open(
path: Union[os.PathLike, str], format: Optional[str] = None, *, should_stream: bool
) -> Union[file, sqlite, stream]: ...
def open(
path: Union[os.PathLike, str], format: Optional[str] = None, should_stream: bool = False
path: Union[os.PathLike, str], format: Optional[str] = None, should_stream: bool = False, readonly: bool = False
) -> Union[file, sqlite, stream]:
"""Loads an IFC dataset from a filepath
@@ -179,7 +179,7 @@ def open(
return sqlite(path)
if should_stream:
return stream(path)
f = ifcopenshell_wrapper.open(str(path.absolute()))
f = ifcopenshell_wrapper.open(str(path.absolute()), readonly)
return file(f)
src/ifcparse/IfcEntityInstanceData.cpp
+7 -7
View File
@@ -36,7 +36,7 @@ namespace {
if (storage_model_ == 0) {
return array_.storage_ptr->get<T>(index_);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
T val;
if constexpr (
@@ -59,7 +59,7 @@ namespace {
if (storage_model_ == 0) {
return array_.storage_ptr->has<T>(index_);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
@@ -78,7 +78,7 @@ namespace {
if (storage_model_ == 0) {
return array_.storage_ptr->index(index_);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
std::string str;
if (!array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (is_entity ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str).ok()) {
@@ -121,7 +121,7 @@ AttributeValue::operator std::string() const
if (storage_model_ == 0) {
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_).value();
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
@@ -141,7 +141,7 @@ AttributeValue::operator EnumerationReference() const
if (storage_model_ == 0) {
return dispatch_get_<EnumerationReference>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
@@ -164,7 +164,7 @@ AttributeValue::operator IfcUtil::IfcBaseClass* () const
if (storage_model_ == 0) {
return dispatch_get_<IfcUtil::IfcBaseClass*>(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
std::string str;
array_.db_ptr->db->Get(rocksdb::ReadOptions{}, (entity_or_type_ == 1 ? "i|" : "t|") + std::to_string(instance_name_) + "|" + std::to_string(index_), &str);
@@ -237,7 +237,7 @@ IfcUtil::ArgumentType AttributeValue::type() const
return static_cast<IfcUtil::ArgumentType>(dispatch_index_(array_, storage_model_, instance_name_, entity_or_type_ == 1 ? true : false, index_));
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
bool impl::serialize(std::string& val, const IfcUtil::IfcBaseClass* t)
{
src/ifcparse/IfcEntityInstanceData.h
+6 -6
View File
@@ -25,7 +25,7 @@
#include "aggregate_of_instance.h"
#include "IfcSchema.h"
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
#pragma push_macro("Handle")
#undef Handle
@@ -165,7 +165,7 @@ namespace IfcParse {
}
}
#if WITH_ROCKSDB
#if IFOPSH_WITH_ROCKSDB
namespace impl {
@@ -393,7 +393,7 @@ struct AttributeValue {
struct rocks_db_attribute_storage {
public:
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
// @todo void* is obviously very ugly here
template<typename T>
void set(void* storage, const IfcParse::declaration*, std::size_t identity, std::size_t index, const T& value);
@@ -449,7 +449,7 @@ class IFC_PARSE_API IfcEntityInstanceData {
if (storage_) {
storage_->set(index, value);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
rocks_db_attribute_storage{}.set(storage, decl, identity, index, value);
}
@@ -461,7 +461,7 @@ class IFC_PARSE_API IfcEntityInstanceData {
if (storage_) {
return storage_->has<T>(index);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
return rocks_db_attribute_storage{}.has<T>(storage, decl, identity, index);
}
@@ -473,7 +473,7 @@ class IFC_PARSE_API IfcEntityInstanceData {
if (storage_) {
return storage_->apply_visitor(std::forward<Visitor>(visitor), index);
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else {
return rocks_db_attribute_storage{}.apply_visitor(storage, decl, identity, index, std::forward<Visitor>(visitor));
}
src/ifcparse/IfcFile.cpp
+40 -11
View File
@@ -1,6 +1,8 @@
#include "IfcFile.h"
#include "IfcLogger.h"
#include <rocksdb/table.h>
IfcParse::parse_context::~parse_context() {
for (auto& t : tokens_) {
std::visit([](auto& v) {
@@ -346,7 +348,7 @@ IfcParse::impl::rocks_db_file_storage::rocksdb_types_iterator::value_type const&
}
IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(size_t number, instance_ref r) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (r == IfcParse::impl::rocks_db_file_storage::entityinstance_ref) {
auto it = instance_cache_.find(number);
if (it != instance_cache_.end()) {
@@ -397,26 +399,53 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::assert_existance(s
}
namespace {
rocksdb::DB* init_db(const std::string& filepath) {
rocksdb::DB* init_db(const std::string& filepath, bool readonly) {
rocksdb::DB* db = nullptr;
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
rocksdb::Options options;
// options.disable_auto_compactions = true;
options.create_if_missing = true;
options.merge_operator.reset(new ConcatenateIdMergeOperator());
rocksdb::Status status = rocksdb::DB::Open(options, filepath, &db);
if (!status.ok()) {
throw std::runtime_error(status.ToString());
options.compression = rocksdb::kZSTD;
rocksdb::BlockBasedTableOptions tbo;
/*
tbo.block_size = 16 * 1024;
tbo.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10 /*bits/key/, false));
tbo.partition_filters = true;
tbo.index_type = rocksdb::BlockBasedTableOptions::kHashSearch;
tbo.cache_index_and_filter_blocks = true;
tbo.cache_index_and_filter_blocks_with_high_priority = true;
tbo.pin_top_level_index_and_filter = true;
*/
auto block_cache = rocksdb::NewLRUCache(1ULL << 30);
tbo.block_cache = block_cache;
// rocksdb::CreateDBStatistics();
options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(tbo));
rocksdb::Status status;
if (readonly) {
status = rocksdb::DB::OpenForReadOnly(options, filepath, &db);
} else {
status = rocksdb::DB::Open(options, filepath, &db);
}
#endif // WITH_ROCKSDB#
if (!status.ok()) {
return nullptr;
}
#endif // IFOPSH_WITH_ROCKSDB#
return db;
}
}
// @todo naming
IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile)
IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* ffile, bool readonly)
: file(ffile)
, db(init_db(filepath))
, db(init_db(filepath, readonly))
, byguid_internal_(db, "g|")
, byguid_(&byguid_internal_, [this](size_t v) { return assert_existance(v, entityinstance_ref); }, [](IfcUtil::IfcBaseClass* v) { return v->identity(); })
, instance_ids_(db, "i|")
@@ -431,7 +460,7 @@ IfcParse::impl::rocks_db_file_storage::rocks_db_file_storage(const std::string&
IfcParse::impl::rocks_db_file_storage::~rocks_db_file_storage()
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
rocksdb::FlushOptions flush_options;
flush_options.allow_write_stall = true;
flush_options.wait = true; // Wait until flush completes.
@@ -457,7 +486,7 @@ IfcUtil::IfcBaseClass* IfcParse::impl::rocks_db_file_storage::instance_by_id(int
void IfcParse::impl::rocks_db_file_storage::process_deletion_inverse(IfcUtil::IfcBaseClass* inst)
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
auto id = inst->id();
{
src/ifcparse/IfcFile.h
+2 -2
View File
@@ -35,7 +35,7 @@
#include <iterator>
#include <map>
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
#include <rocksdb/merge_operator.h>
namespace {
@@ -232,7 +232,7 @@ private:
#ifdef USE_MMAP
IfcFile(const std::string& path, bool mmap = false);
#else
IfcFile(const std::string& path, filetype ty=FT_AUTODETECT);
IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false);
#endif
IfcFile(std::istream& stream, int length);
IfcFile(void* data, int length);
src/ifcparse/IfcParse.cpp
+23 -14
View File
@@ -791,7 +791,7 @@ namespace {
}
void IfcParse::impl::rocks_db_file_storage::register_inverse(unsigned id_from, const IfcParse::entity* from_entity, int inst_id, int attribute_index) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
static std::string s;
uint32_t v = id_from;
s.resize(sizeof(uint32_t));
@@ -813,7 +813,7 @@ void IfcParse::impl::rocks_db_file_storage::register_inverse(unsigned id_from, c
}
void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from, const IfcParse::entity* from_entity, IfcUtil::IfcBaseClass* inst, int attribute_index) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
static std::string s;
auto inst_id = inst->id();
auto key = "v|" + to_string_fixed_width(inst_id, 10) + "|" + to_string_fixed_width(from_entity->index_in_schema(), 4) + "|" + to_string_fixed_width(attribute_index, 2);
@@ -835,7 +835,7 @@ void IfcParse::impl::rocks_db_file_storage::unregister_inverse(unsigned id_from,
void IfcParse::impl::rocks_db_file_storage::add_type_ref(IfcUtil::IfcBaseClass* new_entity)
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
size_t v;
std::string s(sizeof(size_t), ' ');
@@ -865,7 +865,7 @@ void IfcParse::impl::rocks_db_file_storage::add_type_ref(IfcUtil::IfcBaseClass*
void IfcParse::impl::rocks_db_file_storage::remove_type_ref(IfcUtil::IfcBaseClass* new_entity)
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (new_entity->declaration().as_entity()) {
std::string s;
auto key = "t|" + std::to_string(new_entity->declaration().index_in_schema());
@@ -1349,7 +1349,7 @@ IfcFile::IfcFile(const std::string& fn, bool mmap) {
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s);
}
#else
IfcFile::IfcFile(const std::string& path, filetype ty)
IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
: schema_(nullptr)
, max_id_(0)
{
@@ -1370,13 +1370,22 @@ IfcFile::IfcFile(const std::string& path, filetype ty)
}
// byidentity_ = decltype(byidentity_)(&std::get<impl::in_memory_file_storage>(storage_).byidentity_);
} else if (ty == FT_ROCKSDB) {
// @todo this can only be used for databases that already exist, because otherwise there is no way to specify the schema
storage_.emplace<2>(path, this);
std::get<impl::rocks_db_file_storage>(storage_).read_schema(schema_);
// This would make some difference, but in the greater light of things, not really significant
// LateBoundEntity is also still large per instance
// instantiate_typed_instances = false;
byid_ = decltype(byid_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_by_name_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::rocks_db_file_storage>(storage_).byref_excl_);
byguid_ = decltype(byguid_)(&std::get<impl::rocks_db_file_storage>(storage_).byguid_);
// @todo this can only be used for databases that already exist, because otherwise there is no way to specify the schema
storage_.emplace<2>(path, this, readonly);
if (std::get<impl::rocks_db_file_storage>(storage_).db == nullptr) {
storage_.emplace<0>();
good_ = file_open_status::READ_ERROR;
} else {
std::get<impl::rocks_db_file_storage>(storage_).read_schema(schema_);
byid_ = decltype(byid_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_by_name_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::rocks_db_file_storage>(storage_).byref_excl_);
byguid_ = decltype(byguid_)(&std::get<impl::rocks_db_file_storage>(storage_).byguid_);
}
// byidentity_ = decltype(byidentity_)(&std::get<impl::rocks_db_file_storage>(storage_).instance_cache_);
} else {
storage_.emplace<0>();
@@ -2320,7 +2329,7 @@ aggregate_of_instance::ptr IfcFile::instances_by_reference(int t) {
}
}
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
// @todo no lower/upper_bounds() implemented yet
auto prefix = "v|" + std::to_string(t) + "|";
@@ -2544,7 +2553,7 @@ aggregate_of_instance::ptr IfcFile::getInverse(int instance_id, const IfcParse::
}
}
}
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
else if constexpr (std::is_same_v<std::decay_t<decltype(x)>, impl::rocks_db_file_storage>) {
if (attribute_index == -1) {
// @todo no lower/upper_bounds() implemented yet
@@ -2789,7 +2798,7 @@ AttributeValue IfcEntityInstanceData::get_attribute_value(void* storage, const I
}
bool IfcParse::impl::rocks_db_file_storage::read_schema(const IfcParse::schema_definition*& schema) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string value;
auto key = "h|file_schema|0";
db->Get(rocksdb::ReadOptions{}, key, &value);
src/ifcparse/rocksdb_map_adapter.h
+11 -11
View File
@@ -20,7 +20,7 @@
#ifndef ROCKSDB_MAP_ADAPTER_H
#define ROCKSDB_MAP_ADAPTER_H
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#endif
@@ -205,7 +205,7 @@ public:
mutable value_type cached_value_;
void check_valid() {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_)) {
it_.reset();
}
@@ -225,7 +225,7 @@ public:
iterator(const iterator& other)
: db_(other.db_), prefix_(other.prefix_), codec_(other.codec_)
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (other.it_) {
std::string curr = other.it_->key().ToString();
it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
@@ -237,7 +237,7 @@ public:
}
iterator& operator=(const iterator& other) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (this != &other) {
db_ = other.db_;
prefix_ = other.prefix_;
@@ -257,7 +257,7 @@ public:
}
value_type operator*() const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string full_key = it_->key().ToString();
std::string key_without_prefix = full_key.substr(prefix_.size());
std::string value_str = it_->value().ToString();
@@ -274,7 +274,7 @@ public:
}
iterator& operator++() {
#if WITH_ROCKSDB
#if IFOPSH_WITH_ROCKSDB
if (it_) {
it_->Next();
check_valid();
@@ -290,7 +290,7 @@ public:
}
bool operator==(const iterator& other) const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (!it_ && !other.it_) return true;
if (it_ && other.it_)
return it_->key().ToString() == other.it_->key().ToString();
@@ -304,7 +304,7 @@ public:
};
iterator begin() const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
iter->Seek(prefix_);
if (iter->Valid() && iter->key().starts_with(prefix_)) {
@@ -319,7 +319,7 @@ public:
}
iterator find(const key_type& key) const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string key_str = key_to_string(key);
std::string full_key = prefix_ + key_str;
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
@@ -331,7 +331,7 @@ public:
}
size_t erase(const key_type& key) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string key_str = key_to_string(key);
std::string full_key = prefix_ + key_str;
rocksdb::Status s = db_->Delete(rocksdb::WriteOptions{}, full_key);
@@ -342,7 +342,7 @@ public:
}
std::pair<iterator, bool> insert(const value_type& val) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string key_str = key_to_string(val.first);
std::string full_key = prefix_ + key_str;
std::string existing;
src/ifcparse/rocksdb_set_view.h
+9 -9
View File
@@ -20,7 +20,7 @@
#ifndef ROCKSDB_SET_VIEW_H
#define ROCKSDB_SET_VIEW_H
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#endif
@@ -66,7 +66,7 @@ public:
// Helper: extract the key (i.e. the value) from the current RocksDB key.
value_type extract_current_value() const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string full_key = it_->key().ToString();
std::string remainder = full_key.substr(prefix_.size());
size_t pos = remainder.find('|');
@@ -79,7 +79,7 @@ public:
// Validates the current iterator state.
void check_valid() {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (!it_ || !it_->Valid() || !it_->key().starts_with(prefix_))
it_.reset();
#endif
@@ -98,7 +98,7 @@ public:
iterator(const iterator& other)
: db_(other.db_), prefix_(other.prefix_)
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (other.it_) {
std::string curr = other.it_->key().ToString();
it_.reset(db_->NewIterator(rocksdb::ReadOptions{}));
@@ -110,7 +110,7 @@ public:
}
iterator& operator=(const iterator& other) {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (this != &other) {
db_ = other.db_;
prefix_ = other.prefix_;
@@ -141,7 +141,7 @@ public:
// Pre-increment: advance the iterator and skip over any duplicate keys.
iterator& operator++() {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (it_) {
// Record the current key value.
value_type curr = extract_current_value();
@@ -163,7 +163,7 @@ public:
}
bool operator==(const iterator& other) const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (!it_ && !other.it_)
return true;
if (it_ && other.it_)
@@ -179,7 +179,7 @@ public:
// Returns an iterator to the first element in the key-space (or end() if none exist).
iterator begin() const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
auto iter = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions{}));
iter->Seek(prefix_);
if (iter->Valid() && iter->key().starts_with(prefix_))
@@ -195,7 +195,7 @@ public:
// Read-only find: returns an iterator to the element with the given key if it exists.
iterator find(const key_type& key) const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
std::string key_str = key_to_string(key);
// Construct the search key: prefix + key_str + separator.
std::string start_key = prefix_ + key_str + "|";
src/ifcparse/storage.h
+5 -5
View File
@@ -1,7 +1,7 @@
#ifndef STORAGE_H
#define STORAGE_H
#ifndef WITH_ROCKSDB
#ifndef IFOPSH_WITH_ROCKSDB
namespace rocksdb {
class DB {};
@@ -357,7 +357,7 @@ namespace IfcParse {
entities_by_ref_t byref_excl_;
// @todo naming
rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file);
rocks_db_file_storage(const std::string& filepath, IfcParse::IfcFile* file, bool readonly=false);
~rocks_db_file_storage();
bool read_schema(const IfcParse::schema_definition*& schema);
@@ -449,7 +449,7 @@ namespace IfcParse {
static constexpr char prefix_[] = "t|";
boost::optional<size_t> read_id_() const {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
auto sv = state_->key().ToStringView();
auto ii = sv.find("|", 2);
if (ii != decltype(sv)::npos) {
@@ -479,7 +479,7 @@ namespace IfcParse {
rocksdb_types_iterator(const rocks_db_file_storage* fs)
: storage_(fs)
{
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
state_ = fs->db->NewIterator(rocksdb::ReadOptions());
state_->Seek(prefix_);
if (!state_->Valid() || !state_->key().starts_with(prefix_)) {
@@ -490,7 +490,7 @@ namespace IfcParse {
}
rocksdb_types_iterator& operator++() {
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
if (!state_) {
return *this;
}
src/ifcwrap/IfcParseWrapper.i
+2 -2
View File
@@ -633,10 +633,10 @@ static IfcUtil::ArgumentType helper_fn_attribute_type(const IfcUtil::IfcBaseClas
%newobject parse_ifcxml;
%inline %{
IfcParse::IfcFile* open(const std::string& fn) {
IfcParse::IfcFile* open(const std::string& fn, bool readonly=false) {
IfcParse::IfcFile* f;
Py_BEGIN_ALLOW_THREADS;
f = new IfcParse::IfcFile(fn);
f = new IfcParse::IfcFile(fn, IfcParse::FT_AUTODETECT, readonly);
Py_END_ALLOW_THREADS;
return f;
}
src/serializers/RocksDbSerializer.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
#include "RocksDbSerializer.h"
src/serializers/RocksDbSerializer.h
+1 -1
View File
@@ -1,6 +1,6 @@
#ifndef ROCKSDBSERIALIZER_H
#define ROCKSDBSERIALIZER_H
#ifdef WITH_ROCKSDB
#ifdef IFOPSH_WITH_ROCKSDB
#include "../serializers/serializers_api.h"
#include "../ifcgeom/Serializer.h"
win/build-deps.cmd
+20 -2
View File
@@ -608,23 +608,41 @@ IF EXIST "%INSTALL_DIR%\%DEPENDENCY_NAME%" (
)
call :GitCloneAndCheckoutRevision https://gitlab.com/libeigen/eigen.git "%DEPENDENCY_DIR%" 3.3.9
:zstd
set DEPENDENCY_NAME=zstd
set DEPENDENCY_DIR=%DEPS_DIR%\%DEPENDENCY_NAME%
call :GitCloneAndCheckoutRevision https://github.com/facebook/zstd "%DEPENDENCY_DIR%" v1.5.7
IF NOT %ERRORLEVEL%==0 GOTO :Error
cd "%DEPENDENCY_DIR%"\build\cmake
call :RunCMake -DCMAKE_INSTALL_PREFIX="%INSTALL_DIR%\zstd" -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF
IF NOT %ERRORLEVEL%==0 GOTO :Error
call :BuildSolution "%DEPENDENCY_DIR%\build\cmake\%BUILD_DIR%\zstd.sln" %BUILD_CFG%
IF NOT %ERRORLEVEL%==0 GOTO :Error
call :InstallCMakeProject "%DEPENDENCY_DIR%\build\cmake\%BUILD_DIR%" %BUILD_CFG%
IF NOT %ERRORLEVEL%==0 GOTO :Error
:rocksdb
set DEPENDENCY_NAME=rocksdb
set DEPENDENCY_DIR=%DEPS_DIR%\%DEPENDENCY_NAME%
cd %DEPS_DIR%
call :GitCloneAndCheckoutRevision https://github.com/facebook/rocksdb "%DEPENDENCY_DIR%" v9.11.2
IF NOT %ERRORLEVEL%==0 GOTO :Error
cd "%DEPENDENCY_DIR%"
:: see rocksdb\thirdparty.inc
set ZSTD_INCLUDE=%INSTALL_DIR%\zstd\include
set ZSTD_LIB_DEBUG=%INSTALL_DIR%\zstd\lib\zstd_static.lib
set ZSTD_LIB_RELEASE=%INSTALL_DIR%\zstd\lib\zstd_static.lib
call :RunCMake -DCMAKE_INSTALL_PREFIX="%INSTALL_DIR%\rocksdb" ^
-DROCKSDB_INSTALL_ON_WINDOWS=On ^
-DFAIL_ON_WARNINGS=Off ^
-DWITH_TESTS=OFF
-DWITH_TESTS=OFF ^
-DWITH_ZSTD=On
IF NOT %ERRORLEVEL%==0 GOTO :Error
call :BuildSolution "%DEPENDENCY_DIR%\%BUILD_DIR%\rocksdb.sln" %BUILD_CFG%
IF NOT %ERRORLEVEL%==0 GOTO :Error
call :InstallCMakeProject "%DEPENDENCY_DIR%\%BUILD_DIR%" %BUILD_CFG%
IF NOT %ERRORLEVEL%==0 GOTO :Error
:: :tbb
:: set DEPENDENCY_NAME=tbb
:: set DEPENDENCY_DIR=%DEPS_DIR%\tbb
win/run-cmake.bat
+3 -1
View File
@@ -156,16 +156,18 @@ echo.
set CMAKELISTS_DIR=..\cmake
:: Delete CMakeCache.txt if command-line options were provided for this batch script.
if not (%1)==() if exist CMakeCache.txt. del /Q CMakeCache.txt
call cecho.cmd 0 13 "Running CMake for %PROJECT_NAME%."
echo "Running CMake for %PROJECT_NAME%."
IF NOT "%VS_TOOLSET_HOST%"=="" (
cmake.exe %CMAKELISTS_DIR% -G %GENERATOR% -A %VS_PLATFORM% -T %VS_TOOLSET_HOST% ^
-DCMAKE_INSTALL_PREFIX="%CMAKE_INSTALL_PREFIX%" -DBoost_NO_BOOST_CMAKE=ON ^
-DWITH_ROCKSDB=On -DWITH_ZSTD=On ^
-DCMAKE_PREFIX_PATH="%HDF5_INSTALL_DIR%;%OPENCOLLADA_INSTALL_DIR%;%SWIG_INSTALL_DIR%" ^
-DADD_COMMIT_SHA=%ADD_COMMIT_SHA% %ARGUMENTS%
) ELSE (
cmake.exe %CMAKELISTS_DIR% -G %GENERATOR% -A %VS_PLATFORM% ^
-DCMAKE_INSTALL_PREFIX="%CMAKE_INSTALL_PREFIX%" -DBoost_NO_BOOST_CMAKE=ON ^
-DWITH_ROCKSDB=On -DWITH_ZSTD=On ^
-DCMAKE_PREFIX_PATH="%HDF5_INSTALL_DIR%;%OPENCOLLADA_INSTALL_DIR%;%SWIG_INSTALL_DIR%" ^
-DADD_COMMIT_SHA=%ADD_COMMIT_SHA% %ARGUMENTS%
)