Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-08 17:01:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Follow up after 4d688170e0

Browse Source
This commit is contained in:
Thomas Krijnen
2025-10-22 21:42:15 +02:00
parent d5d942bb2e
commit a6d20c0cc4
9 changed files with 102 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcopenshell-python/ifcopenshell/__init__.py
+29 -5
View File
@@ -299,20 +299,44 @@ def guess_format(path: Path) -> Literal[".ifc", ".ifcZIP", ".ifcXML", ".ifcJSON"
return None
def stream2(path: Union[Path, str]):
def stream2(path: Union[Path, str], mmap: bool = False, page_size: int = 0):
"""Streams the content of a file path from disk, yielding each instance
as a dictionary.
Args:
path (Union[Path, str]): input file path
mmap (bool): open the file contents using memory mapping
page_size (int): open file in python and feed chunks to the parser
Yields:
dict: entity instance dictionaries
"""
streamer = ifcopenshell_wrapper.InstanceStreamer(str(path))
while streamer:
if inst := streamer.read_instance_py():
yield inst
if page_size:
import builtins
f = builtins.open(path, encoding="ascii")
strm = ifcopenshell_wrapper.InstanceStreamer()
strm.pushPage(f.read(page_size))
finished = False
while True:
while strm.hasSemicolon():
if inst := strm.readInstancePy():
yield inst
else:
finished = True
break
if finished:
break
else:
if data := f.read(page_size):
strm.pushPage(data)
else:
break
else:
streamer = ifcopenshell_wrapper.InstanceStreamer(str(path), mmap)
while streamer:
if inst := streamer.readInstancePy():
yield inst
def stream2_from_string(data: str):
src/ifcopenshell-python/test/test_streaming_rocksdb_and_simpletyperefs.py
+5
View File
@@ -32,6 +32,11 @@ def test_stream():
"value": ({"ref": 136}, {"ref": 138}),
}
def test_chunked_stream():
assert list(ifcopenshell.stream2(fn)) == list(ifcopenshell.stream2(fn, page_size=1024))
def test_mmaped_stream():
assert list(ifcopenshell.stream2(fn)) == list(ifcopenshell.stream2(fn, mmap=True))
def test_file():
f = ifcopenshell.open(fn)
src/ifcparse/FileReader.cpp
+26 -28
View File
@@ -11,11 +11,8 @@
#include <vector>
#include <deque>
// Boost iostreams mmap
#include <boost/iostreams/device/mapped_file.hpp>
namespace {
#if defined(_WIN32)
@@ -32,8 +29,6 @@ namespace {
} // namespace
// ===================== Concrete backends =====================
using namespace IfcParse;
struct FullBufferImpl final : FileReader::Impl {
@@ -64,7 +59,7 @@ struct PagedFileImpl final : FileReader::Impl {
// LRU cache
size_t capacity_ = 8;
mutable std::list<size_t> lru_; // most recent at front
mutable std::list<size_t> lru_;
struct Entry {
FileReader::Page page;
std::list<size_t>::iterator it;
@@ -90,16 +85,14 @@ struct PagedFileImpl final : FileReader::Impl {
char get(size_t pos) const override {
if (pos >= file_size_) throw std::out_of_range("get out of range");
const size_t pidx = pos / page_size_;
const FileReader::Page& p = fetch_page_(pidx);
const FileReader::Page& p = fetchPage_(pidx);
const size_t off = pos % page_size_;
if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes");
// Opportunistic read-ahead for sequential scans
// if (off + 1 == p.data.size()) (void)try_prefetch_(pidx + 1);
return p.data[off];
}
private:
const FileReader::Page& fetch_page_(size_t idx) const {
const FileReader::Page& fetchPage_(size_t idx) const {
auto it = map_.find(idx);
if (it != map_.end()) {
touch_(it);
@@ -116,7 +109,8 @@ private:
const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
if (nread != avail) throw std::runtime_error("Short fread on page");
}
pg.data.resize(avail); // trim to actual size
// trim to actual size
pg.data.resize(avail);
// Insert into LRU
if (map_.size() >= capacity_) evict_();
@@ -127,16 +121,6 @@ private:
return emplaced_it->second.page;
}
/*
bool try_prefetch_(size_t idx) const {
if (idx * page_size_ >= file_size_) return false;
if (map_.find(idx) != map_.end()) return true;
if (map_.size() + 1 > capacity_) return false;
(void)fetch_page_(idx);
return true;
}
*/
void touch_(typename std::unordered_map<size_t, Entry>::iterator it) const {
lru_.erase(it->second.it);
lru_.push_front(it->first);
@@ -189,7 +173,7 @@ struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentia
}
// Drop fully-consumed pages so pos is guaranteed to be within the first page
void drop_consumed_up_to(size_t pos) {
void dropPages(size_t pos) override {
while (!pages_.empty()) {
if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
discarded_page_bytes_ += pages_.front().data.size();
@@ -202,11 +186,17 @@ struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentia
char get(size_t pos) const override {
auto self = const_cast<PushedSequentialImpl*>(this);
/*
// We do not do this automatically because all variable width tokens:
// ENUM/STRING/BINARY/KEYWORD are stored as file offsets until a full
// entity instance is finalized.
if (this->shared_from_this().use_count() == 2) {
// only drop pages when there is only one active client.
// NB this->shared_from_this() increases count by 1
self->drop_consumed_up_to(pos);
}
*/
const size_t avail_end = size();
if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet");
@@ -226,14 +216,12 @@ struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentia
throw std::out_of_range("pushed backend: internal inconsistency");
}
void push_next_page(const std::string& data) override {
void pushNextPage(const std::string& data) override {
FileReader::Page p; p.data.assign(data.data(), data.data() + data.size());
pages_.push_back(std::move(p));
}
};
// ===================== FileReader public API =====================
IfcParse::FileReader::FileReader(const std::string& fn)
: cursor_(0)
{
@@ -255,7 +243,7 @@ IfcParse::FileReader::FileReader(const caller_fed_tag&)
IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&)
{
impl_ = std::make_shared<PushedSequentialImpl>();
impl_->push_next_page(content);
impl_->pushNextPage(content);
}
IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
@@ -289,9 +277,19 @@ void FileReader::increment(size_t n) {
cursor_ += n;
}
void IfcParse::FileReader::push_next_page(const std::string& data)
void IfcParse::FileReader::pushNextPage(const std::string& data)
{
impl_->push_next_page(data);
impl_->pushNextPage(data);
}
void IfcParse::FileReader::dropPages()
{
impl_->dropPages(0);
}
void IfcParse::FileReader::dropPages(size_t up_to_pos)
{
impl_->dropPages(up_to_pos);
}
bool IfcParse::FileReader::eof() const
src/ifcparse/FileReader.h
+11 -58
View File
@@ -29,61 +29,6 @@
#include "ifc_parse_api.h"
/*
#include <string>
#ifdef USE_MMAP
#include <boost/iostreams/device/mapped_file.hpp>
#endif
namespace IfcParse {
/// The FileReader class represents a ISO 10303-21 IFC-SPF file in memory.
/// The file is interpreted as a sequence of tokens which are lazily
/// interpreted only when requested.
class IFC_PARSE_API FileReader {
private:
#ifdef USE_MMAP
boost::iostreams::mapped_file_source mfs;
#endif
FILE* stream_;
const char* buffer_;
size_t ptr_;
size_t len_;
size_t buf_size_;
size_t ptr_offset_ = 0;
public:
bool valid;
bool eof;
size_t size;
FileReader(const std::string& path, bool mmap = false, size_t buf_size = 0);
FileReader(std::istream& stream, int length);
FileReader(void* data, int length);
~FileReader();
/// Returns the character at the cursor
char peek();
/// Returns the character at specified offset
char Read(size_t offset);
/// Increment the file cursor and reads new page if necessary
void increment();
void Close();
/// Moves the file cursor to an arbitrary offset in the file
void seek(size_t offset);
/// Returns the cursor position
size_t Tell() const;
bool is_eof_at(size_t) const;
void increment_at(size_t&);
char peek_at(size_t);
operator bool() const { return valid && !eof; }
};
} // namespace IfcParse
#endif
*/
#include <cstddef>
#include <memory>
#include <optional>
@@ -147,8 +92,13 @@ public:
/// \brief Push the next sequential page (pushed backend only).
/// \param data Contents of the page.
/// \throws std::logic_error if the current backend is not pushed mode, or if a next page is already queued.
void push_next_page(const std::string& data);
/// \throws std::logic_error if the current backend is not in pushed mode
void pushNextPage(const std::string& data);
/// \brief Drops pages up to cursor position or provided offset. Does nothing when current backend is not in pushed mode
/// \param up_to_pos Pages with an end offset before up_to_pos are dropped from memory
void dropPages();
void dropPages(size_t up_to_pos);
/// \brief Returns true if the cursor is at or beyond the end of available data.
/// For the pushed backend, EOF means all pushed bytes have been consumed.
@@ -165,9 +115,12 @@ public:
virtual size_t size() const = 0;
virtual char get(size_t pos) const = 0;
/// \brief Backend may support pushing pages; default throws.
virtual void push_next_page(const std::string&) {
virtual void pushNextPage(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
}
virtual void dropPages(size_t) {
// empty on purpose
}
};
private:
src/ifcparse/IfcFile.cpp
+4 -5
View File
@@ -651,9 +651,7 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
}
}
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> IfcParse::InstanceStreamer::read_instance() {
// std::cout << "global: " << stream_->tell() << std::endl;
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> IfcParse::InstanceStreamer::readInstance() {
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> return_value;
if (header_ && yielded_header_instances_ < 3) {
@@ -741,11 +739,12 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstance
break;
}
// std::cout << next_token.startPos << " " << TokenFunc::toString(next_token) << std::endl;
token_stream_.push_back(next_token);
}
// Free pages in front of cursor when variable-width tokens are materialized into entity instance data objects
(stream_ ? stream_ : (lexer_)->stream)->dropPages();
return return_value;
}
src/ifcparse/IfcFile.h
+4 -4
View File
@@ -126,13 +126,13 @@ public:
return storage_.byref_excl_;
}
std::vector<std::unique_ptr<IfcUtil::IfcBaseClass>> steal_instances() {
std::vector<std::unique_ptr<IfcUtil::IfcBaseClass>> stealInstances() {
return storage_.steal_instances();
}
bool has_semicolon() const;
bool hasSemicolon() const;
void push_page(const std::string& page);
void pushPage(const std::string& page);
InstanceStreamer();
@@ -150,7 +150,7 @@ public:
delete header_;
}
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> read_instance();
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> readInstance();
};
/// This class provides access to the entity instances in an IFC file
src/ifcparse/IfcParse.cpp
+18 -17
View File
@@ -336,11 +336,6 @@ Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, size_t start, const std::str
token.type = Token_KEYWORD;
}
// todo STRING/BINARY/ENUM/KEYWORD not stored
// tokenStr is gone after this.
// Orrrr we could only free pages when after a full instance is formed because after that no
// references to prior token ranges exist anymore.
return token;
}
@@ -1262,7 +1257,7 @@ IfcFile::IfcFile(std::istream& stream, int length)
std::string string_data;
string_data.resize(length);
stream.read(string_data.data(), length);
s.push_next_page(string_data);
s.pushNextPage(string_data);
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
@@ -1335,27 +1330,33 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
setDefaultHeaderValues();
}
bool IfcParse::InstanceStreamer::has_semicolon() const
bool IfcParse::InstanceStreamer::hasSemicolon() const
{
auto local_stream = stream_->clone();
auto local_lexer = IfcSpfLexer(&local_stream);
Token t = local_lexer.Next();
Token t;
try {
t = local_lexer.Next();
} catch (const std::out_of_range&) {
return false;
}
while (t.type != Token_NONE) {
if (TokenFunc::isOperator(t, ';')) {
return true;
}
// probably the issue is that this moves the cursor past the page boundary on the local stream which also affects the global one
// 'freeze' the stream so that no pages are cleared
t = local_lexer.Next();
try {
t = local_lexer.Next();
} catch (const std::out_of_range&) {
// This most likely happens when a page boundary is contained within a string
break;
}
}
std::cout << "local: " << local_stream.tell() << " global: " << stream_->tell() << std::endl;
return false;
}
void IfcParse::InstanceStreamer::push_page(const std::string& page)
void IfcParse::InstanceStreamer::pushPage(const std::string& page)
{
std::cout << "global: " << stream_->tell() << std::endl;
stream_->push_next_page(page);
stream_->pushNextPage(page);
if (good_ == file_open_status::NO_HEADER) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
@@ -1503,7 +1504,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
while (streamer) {
auto inst = streamer.read_instance();
auto inst = streamer.readInstance();
if (!inst) {
// No more instances to read
@@ -1558,7 +1559,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
byref_excl_ = streamer.inverses();
// Move the storage of simple type instances so that they are retained during the lifetime of the file
read_simple_type_instances = streamer.steal_instances();
read_simple_type_instances = streamer.stealInstances();
Logger::Status("\rDone scanning file ");
src/ifcwrap/IfcParseWrapper.i
+4 -4
View File
@@ -36,8 +36,8 @@ private:
%ignore IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer);
%ignore IfcParse::InstanceStreamer::read_instance;
%ignore IfcParse::InstanceStreamer::steal_instances;
%ignore IfcParse::InstanceStreamer::readInstance;
%ignore IfcParse::InstanceStreamer::stealInstances;
%ignore in_memory_file_storage;
%ignore rocks_db_file_storage;
@@ -1003,7 +1003,7 @@ private:
%}
%extend IfcParse::InstanceStreamer {
PyObject* read_instance_py() {
PyObject* readInstancePy() {
auto simply_type_to_dictionary = [&](IfcUtil::IfcBaseClass* t) -> PyObject* {
const auto& nm = t->declaration().name();
auto ifc_val = t->get_attribute_value(0);
@@ -1071,7 +1071,7 @@ private:
Py_INCREF(Py_None);
return Py_None;
}
auto inst = self->read_instance();
auto inst = self->readInstance();
if (!inst) {
Py_INCREF(Py_None);
return Py_None;
src/serializers/RocksDbSerializer.cpp
+1 -1
View File
@@ -135,7 +135,7 @@ void RocksDbSerializer::write_streaming_() {
streamer.coerce_attribute_count = false;
while (streamer) {
auto inst = streamer.read_instance();
auto inst = streamer.readInstance();
if (inst) {
// name can be zero in case of header instances
auto name = std::get<0>(*inst);