From 4d688170e07ddf19a46210ae92e29d9a76215f39 Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Mon, 13 Oct 2025 20:47:43 +0200 Subject: [PATCH] Storage rework WIP --- src/ifcparse/FileReader.cpp | 312 ++++++++++++++++++ src/ifcparse/FileReader.h | 180 ++++++++++ src/ifcparse/IfcCharacterDecoder.cpp | 73 +--- src/ifcparse/IfcCharacterDecoder.h | 8 +- src/ifcparse/IfcEntityInstanceData.h | 2 +- src/ifcparse/IfcException.h | 4 +- src/ifcparse/IfcFile.cpp | 28 +- src/ifcparse/IfcFile.h | 53 ++- src/ifcparse/IfcParse.cpp | 475 +++++++++++---------------- src/ifcparse/IfcParse.h | 18 +- src/ifcparse/IfcSpfHeader.cpp | 2 +- src/ifcparse/IfcSpfStream.h | 84 ----- src/ifcparse/IfcUtil.cpp | 4 +- src/ifcparse/aggregate_of_instance.h | 6 +- src/ifcparse/storage.h | 14 +- 15 files changed, 788 insertions(+), 475 deletions(-) create mode 100644 src/ifcparse/FileReader.cpp create mode 100644 src/ifcparse/FileReader.h delete mode 100644 src/ifcparse/IfcSpfStream.h diff --git a/src/ifcparse/FileReader.cpp b/src/ifcparse/FileReader.cpp new file mode 100644 index 0000000000..07537c7a90 --- /dev/null +++ b/src/ifcparse/FileReader.cpp @@ -0,0 +1,312 @@ +#include "FileReader.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Boost iostreams mmap +#include + + + +namespace { + +#if defined(_WIN32) + inline void file_seek_abs(FILE* f, std::uint64_t off) { + if (_fseeki64(f, static_cast(off), SEEK_SET) != 0) + throw std::runtime_error("fseek failed"); + } +#else + inline void file_seek_abs(FILE* f, std::uint64_t off) { + if (fseeko(f, static_cast(off), SEEK_SET) != 0) + throw std::runtime_error("fseeko failed"); + } +#endif + +} // namespace + +// ===================== Concrete backends ===================== + +using namespace IfcParse; + +struct FullBufferImpl final : FileReader::Impl { + std::vector buf_; + explicit FullBufferImpl(const std::string& fn) { + std::ifstream ifs(fn, std::ios::binary); + if (!ifs) throw std::runtime_error("Failed to open: " + fn); + ifs.seekg(0, std::ios::end); + const std::streamsize sz = ifs.tellg(); + ifs.seekg(0, std::ios::beg); + buf_.resize(static_cast(sz)); + if (sz > 0 && !ifs.read(buf_.data(), sz)) { + throw std::runtime_error("Failed to read file into buffer"); + } + } + size_t size() const override { return buf_.size(); } + char get(size_t pos) const override { + if (pos >= buf_.size()) throw std::out_of_range("get out of range"); + return buf_[pos]; + } +}; + +struct PagedFileImpl final : FileReader::Impl { + std::string fn_; + FILE* fp_ = nullptr; + size_t file_size_ = 0; + size_t page_size_ = 4096; + + // LRU cache + size_t capacity_ = 8; + mutable std::list lru_; // most recent at front + struct Entry { + FileReader::Page page; + std::list::iterator it; + }; + mutable std::unordered_map map_; + + PagedFileImpl(const std::string& fn, size_t page_size, size_t cap) + : fn_(fn), page_size_(std::max(512, page_size)), capacity_(std::max(2, cap)) { + namespace fs = std::filesystem; + if (!fs::exists(fn_)) throw std::runtime_error("File not found: " + fn_); + file_size_ = static_cast(fs::file_size(fn_)); + fp_ = std::fopen(fn_.c_str(), "rb"); + if (!fp_) throw std::runtime_error("Failed to fopen: " + fn_); + } + + ~PagedFileImpl() override { + if (fp_) std::fclose(fp_); + fp_ = nullptr; + } + + size_t size() const override { return file_size_; } + + char get(size_t pos) const override { + if (pos >= file_size_) throw std::out_of_range("get out of range"); + const size_t pidx = pos / page_size_; + const FileReader::Page& p = fetch_page_(pidx); + const size_t off = pos % page_size_; + if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes"); + // Opportunistic read-ahead for sequential scans + // if (off + 1 == p.data.size()) (void)try_prefetch_(pidx + 1); + return p.data[off]; + } + +private: + const FileReader::Page& fetch_page_(size_t idx) const { + auto it = map_.find(idx); + if (it != map_.end()) { + touch_(it); + return it->second.page; + } + // Load page from disk using persistent FILE* + FileReader::Page pg; + pg.data.resize(page_size_); + const size_t begin = idx * page_size_; + const size_t avail = std::min(page_size_, file_size_ - begin); + + file_seek_abs(fp_, begin); + if (avail > 0) { + const size_t nread = std::fread(pg.data.data(), 1, avail, fp_); + if (nread != avail) throw std::runtime_error("Short fread on page"); + } + pg.data.resize(avail); // trim to actual size + + // Insert into LRU + if (map_.size() >= capacity_) evict_(); + lru_.push_front(idx); + auto lit = lru_.begin(); + auto [emplaced_it, ok] = map_.emplace(idx, Entry{ std::move(pg), lit }); + (void)ok; + return emplaced_it->second.page; + } + + /* + bool try_prefetch_(size_t idx) const { + if (idx * page_size_ >= file_size_) return false; + if (map_.find(idx) != map_.end()) return true; + if (map_.size() + 1 > capacity_) return false; + (void)fetch_page_(idx); + return true; + } + */ + + void touch_(typename std::unordered_map::iterator it) const { + lru_.erase(it->second.it); + lru_.push_front(it->first); + it->second.it = lru_.begin(); + } + + void evict_() const { + if (lru_.empty()) return; + const size_t victim = lru_.back(); + lru_.pop_back(); + map_.erase(victim); + } +}; + +struct MMapImpl final : FileReader::Impl { + boost::iostreams::mapped_file_source map_; + size_t size_ = 0; + + explicit MMapImpl(const std::string& fn) { + namespace fs = std::filesystem; + if (!fs::exists(fn)) throw std::runtime_error("File not found: " + fn); + size_ = static_cast(fs::file_size(fn)); + if (size_ == 0) return; // empty file: map_ stays closed + map_.open(fn); + if (!map_.is_open()) throw std::runtime_error("Failed to open mapped_file_source"); + size_ = static_cast(map_.size()); + } + + size_t size() const override { return size_; } + + char get(size_t pos) const override { + if (pos >= size_) throw std::out_of_range("get out of range"); + return map_.data()[pos]; + } +}; + +/// User-pushed sequential backend with an arbitrary-length queue of future pages. +/// We keep a deque of pages; when reads move forward, we drop fully-consumed +/// pages from the front to release memory. +struct PushedSequentialImpl final : std::enable_shared_from_this, FileReader::Impl { + // Deque of pages, front is earliest in file. + std::deque pages_; + // total bytes in dropped pages + size_t discarded_page_bytes_ = 0; + + size_t size() const override { + size_t n = discarded_page_bytes_; + for (auto& pg : pages_) n += pg.data.size(); + return n; + } + + // Drop fully-consumed pages so pos is guaranteed to be within the first page + void drop_consumed_up_to(size_t pos) { + while (!pages_.empty()) { + if (pos - discarded_page_bytes_ >= pages_.front().data.size()) { + discarded_page_bytes_ += pages_.front().data.size(); + pages_.pop_front(); + } else { + break; + } + } + } + + char get(size_t pos) const override { + auto self = const_cast(this); + if (this->shared_from_this().use_count() == 2) { + // only drop pages when there is only one active client. + // NB this->shared_from_this() increases count by 1 + self->drop_consumed_up_to(pos); + } + + const size_t avail_end = size(); + if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet"); + + pos -= discarded_page_bytes_; + + size_t page_start = 0; + for (const auto& pg : pages_) { + if (pos < page_start + pg.data.size()) { + const size_t off = pos - page_start; + return pg.data[off]; + } else { + page_start += pg.data.size(); + } + } + + throw std::out_of_range("pushed backend: internal inconsistency"); + } + + void push_next_page(const std::string& data) override { + FileReader::Page p; p.data.assign(data.data(), data.data() + data.size()); + pages_.push_back(std::move(p)); + } +}; + +// ===================== FileReader public API ===================== + +IfcParse::FileReader::FileReader(const std::string& fn) + : cursor_(0) +{ + impl_ = std::make_shared(fn); +} + +IfcParse::FileReader::FileReader(const std::string& fn, const mmap_tag&) + : cursor_(0) +{ + impl_ = std::make_shared(fn); +} + +IfcParse::FileReader::FileReader(const caller_fed_tag&) + : cursor_(0) +{ + impl_ = std::make_shared(); +} + +IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&) +{ + impl_ = std::make_shared(); + impl_->push_next_page(content); +} + +IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity) + : cursor_(0) +{ + impl_ = std::make_shared(fn, page_size, page_capacity); +} + +FileReader FileReader::clone() const { + FileReader c(*this); + c.cursor_ = this->cursor_; + return c; +} + +void FileReader::seek(size_t pos) { + if (pos > impl_->size()) throw std::out_of_range("seek out of range"); + cursor_ = pos; +} + +size_t FileReader::tell() const { return cursor_; } + +size_t FileReader::size() const { return impl_->size(); } + +char FileReader::peek() const { + if (cursor_ >= impl_->size()) throw std::out_of_range("peek at EOF"); + return impl_->get(cursor_); +} + +void FileReader::increment(size_t n) { + if (cursor_ + n > impl_->size()) throw std::out_of_range("increment past EOF"); + cursor_ += n; +} + +void IfcParse::FileReader::push_next_page(const std::string& data) +{ + impl_->push_next_page(data); +} + +bool IfcParse::FileReader::eof() const +{ + return cursor_ >= impl_->size(); +} + +char IfcParse::FileReader::read() +{ + auto c = peek(); + increment(1); + return c; +} + +char IfcParse::FileReader::get(size_t offset) const +{ + return impl_->get(offset); +} diff --git a/src/ifcparse/FileReader.h b/src/ifcparse/FileReader.h new file mode 100644 index 0000000000..aee95488d5 --- /dev/null +++ b/src/ifcparse/FileReader.h @@ -0,0 +1,180 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +/********************************************************************************* + * * + * Reads a file and provides functions to access its * + * contents randomly and character by character * + * * + ********************************************************************************/ + +#ifndef IFCSPFSTREAM_H +#define IFCSPFSTREAM_H + +#include "ifc_parse_api.h" + +/* +#include + +#ifdef USE_MMAP +#include +#endif + +namespace IfcParse { +/// The FileReader class represents a ISO 10303-21 IFC-SPF file in memory. +/// The file is interpreted as a sequence of tokens which are lazily +/// interpreted only when requested. +class IFC_PARSE_API FileReader { + private: +#ifdef USE_MMAP + boost::iostreams::mapped_file_source mfs; +#endif + FILE* stream_; + const char* buffer_; + size_t ptr_; + size_t len_; + size_t buf_size_; + size_t ptr_offset_ = 0; + + public: + bool valid; + bool eof; + size_t size; + + FileReader(const std::string& path, bool mmap = false, size_t buf_size = 0); + FileReader(std::istream& stream, int length); + FileReader(void* data, int length); + ~FileReader(); + /// Returns the character at the cursor + char peek(); + /// Returns the character at specified offset + char Read(size_t offset); + /// Increment the file cursor and reads new page if necessary + void increment(); + void Close(); + /// Moves the file cursor to an arbitrary offset in the file + void seek(size_t offset); + /// Returns the cursor position + size_t Tell() const; + + bool is_eof_at(size_t) const; + void increment_at(size_t&); + char peek_at(size_t); + + operator bool() const { return valid && !eof; } +}; +} // namespace IfcParse + +#endif +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace IfcParse { + +/// \brief Read-only file accessor that supports four backends: +/// - full-buffer in RAM +/// - paged (LRU-cached with capacity) +/// - memory-mapped via boost::iostreams +/// - user-pushed sequential pages (caller feeds pages intended for streaming reads in WASM) +class IFC_PARSE_API FileReader { +public: + struct Page { std::vector data; }; + + /// \brief Tag to choose memory-mapped backend via boost::iostreams::mapped_file_source. + struct mmap_tag {}; + /// \brief Tag to choose user-pushed sequential pages. + struct caller_fed_tag {}; + + /// \brief Construct a FileReader. + /// \param fn File path. + /// \param maybe_mmaped_or_chunked + /// - std::nullopt: full-buffer mode (entire file loaded into memory) + /// - size_t: paged mode with this page size in bytes (e.g., 4096) + /// - mmap_tag: memory-mapped mode using boost::iostreams::mapped_file_source + /// \param page_cache_capacity Number of pages to keep in the LRU when in paged mode. + FileReader(const std::string& fn); + + FileReader(const std::string& fn, const mmap_tag&); + + FileReader(const caller_fed_tag&); + FileReader(const std::string& content, const caller_fed_tag&); + + FileReader(const std::string& fn, size_t page_size, size_t page_capacity); + + /// \brief Copy-construct a new reader sharing the underlying storage but with its own cursor. + FileReader clone() const; + + /// \brief Seek to an absolute byte position. + /// \throws std::out_of_range if pos > size(). + void seek(size_t pos); + + /// \brief Return the current cursor position. + size_t tell() const; + + /// \brief Total file size in bytes. + size_t size() const; + + /// \brief Peek the byte at the current cursor. + /// \throws std::out_of_range at EOF. + char peek() const; + + /// \brief Advance the cursor by n bytes (default 1). + /// \throws std::out_of_range if advancing crosses EOF. + void increment(size_t n = 1); + + /// \brief Push the next sequential page (pushed backend only). + /// \param data Contents of the page. + /// \throws std::logic_error if the current backend is not pushed mode, or if a next page is already queued. + void push_next_page(const std::string& data); + + /// \brief Returns true if the cursor is at or beyond the end of available data. + /// For the pushed backend, EOF means all pushed bytes have been consumed. + bool eof() const; + + /// \brief Equivalent of peek() followed by increment(1) + char read(); + + /// \brief Equivalent of peek() followed by increment(1) + char get(size_t offset) const; + + struct Impl { + virtual ~Impl() = default; + virtual size_t size() const = 0; + virtual char get(size_t pos) const = 0; + /// \brief Backend may support pushing pages; default throws. + virtual void push_next_page(const std::string&) { + throw std::logic_error("push_next_page: backend does not support pushed mode"); + } + }; + +private: + std::shared_ptr impl_; + size_t cursor_ = 0; +}; + +} // namespace IfcParse + +#endif diff --git a/src/ifcparse/IfcCharacterDecoder.cpp b/src/ifcparse/IfcCharacterDecoder.cpp index 7e62910808..71685dcedc 100644 --- a/src/ifcparse/IfcCharacterDecoder.cpp +++ b/src/ifcparse/IfcCharacterDecoder.cpp @@ -29,7 +29,7 @@ #include "IfcCharacterDecoder.h" #include "IfcException.h" -#include "IfcSpfStream.h" +#include "FileReader.h" #include "IfcLogger.h" #include @@ -77,7 +77,7 @@ using namespace IfcParse; -IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::IfcSpfStream* stream) { +IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::FileReader* stream) { stream_ = stream; codepage_ = 0; } @@ -86,49 +86,9 @@ IfcCharacterDecoder::~IfcCharacterDecoder() { } namespace { -unsigned int reference_helper = 0; + std::string read_string(IfcParse::FileReader& stream_, IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) { + std::u32string builder_; -class pure_impure_helper { - private: - bool pure_; - IfcParse::IfcSpfStream* stream_; - unsigned int& pointer_; - std::u32string builder_; - - char peek() { - if (pure_) { - return stream_->peek_at(pointer_); - } - return stream_->Peek(); - } - - unsigned int tell() { - if (pure_) { - return pointer_; - } - return stream_->Tell(); - } - - void increment() { - if (pure_) { - stream_->increment_at(pointer_); - } else { - stream_->Inc(); - } - } - - public: - pure_impure_helper(IfcParse::IfcSpfStream* stream) - : pure_(false), - stream_(stream), - pointer_(reference_helper) {} - - pure_impure_helper(IfcParse::IfcSpfStream* stream, unsigned int& pointer) - : pure_(true), - stream_(stream), - pointer_(pointer) {} - - std::string get(IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) { unsigned int parse_state = 0; builder_.clear(); builder_.push_back('\''); @@ -137,7 +97,7 @@ class pure_impure_helper { unsigned int hex = 0; unsigned int hex_count = 0; - while ((current_char = peek()) != 0) { + while ((current_char = stream_.peek()) != 0) { if (EXPECTS_CHARACTER(parse_state)) { builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80)); parse_state = 0; @@ -178,7 +138,7 @@ class pure_impure_helper { } else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) { if (IS_LOWERCASE_HEX(current_char)) { Logger::Warning("Lowercase hexadecimal character '" + std::string(1, current_char) + - "' found at offset " + std::to_string(pointer_) + + "' found at offset " + std::to_string(stream_.tell()) + ". It is recommended to use uppercase for hexadecimal."); } hex <<= 4; @@ -202,12 +162,12 @@ class pure_impure_helper { if (parse_state == APOSTROPHE && current_char != '\'') { break; } - throw IfcInvalidTokenException(tell(), current_char); + throw IfcInvalidTokenException(stream_.tell(), current_char); } else { parse_state = hex = hex_count = 0; builder_.push_back(current_char); } - increment(); + stream_.increment(); } builder_.push_back('\''); @@ -248,22 +208,25 @@ class pure_impure_helper { } throw IfcParse::IfcException("Invalid conversion mode"); } -}; } // namespace IfcCharacterDecoder::operator std::string() { - return pure_impure_helper(stream_).get(mode, substitution_character); + return read_string(*stream_, mode, substitution_character); } -std::string IfcCharacterDecoder::get(unsigned int& ptr) { - return pure_impure_helper(stream_, ptr).get(mode, substitution_character); +std::string IfcCharacterDecoder::get(size_t& ptr) { + auto local_stream = *stream_; + local_stream.seek(ptr); + auto s = read_string(local_stream, mode, substitution_character); + ptr = local_stream.tell(); + return s; } void IfcCharacterDecoder::skip() { unsigned int parse_state = 0; char current_char; unsigned int hex_count = 0; - while ((current_char = stream_->Peek()) != 0) { + while ((current_char = stream_->peek()) != 0) { if (EXPECTS_CHARACTER(parse_state)) { parse_state = 0; } else if (current_char == '\'' && (parse_state == 0U)) { @@ -318,11 +281,11 @@ void IfcCharacterDecoder::skip() { if (parse_state == APOSTROPHE && current_char != '\'') { break; } - throw IfcInvalidTokenException(stream_->Tell(), current_char); + throw IfcInvalidTokenException(stream_->tell(), current_char); } else { parse_state = hex_count = 0; } - stream_->Inc(); + stream_->increment(); } } diff --git a/src/ifcparse/IfcCharacterDecoder.h b/src/ifcparse/IfcCharacterDecoder.h index 6a6d318aa1..e978a5b4c2 100644 --- a/src/ifcparse/IfcCharacterDecoder.h +++ b/src/ifcparse/IfcCharacterDecoder.h @@ -27,7 +27,7 @@ #ifndef IFCCHARACTERDECODER_H #define IFCCHARACTERDECODER_H -#include "IfcSpfStream.h" +#include "FileReader.h" #include @@ -41,7 +41,7 @@ namespace IfcParse { class IFC_PARSE_API IfcCharacterDecoder { private: - IfcParse::IfcSpfStream* stream_; + IfcParse::FileReader* stream_; int codepage_; public: @@ -52,7 +52,7 @@ class IFC_PARSE_API IfcCharacterDecoder { }; static ConversionMode mode; static char substitution_character; - IfcCharacterDecoder(IfcParse::IfcSpfStream* stream); + IfcCharacterDecoder(IfcParse::FileReader* stream); ~IfcCharacterDecoder(); // Only advances the underlying token stream read pointer // to the next token. @@ -62,7 +62,7 @@ class IFC_PARSE_API IfcCharacterDecoder { operator std::string(); // Gets a decoded string representation at the offset provided, // does not mutate the underlying token stream read pointer. - std::string get(unsigned int&); + std::string get(size_t&); }; } // namespace IfcParse diff --git a/src/ifcparse/IfcEntityInstanceData.h b/src/ifcparse/IfcEntityInstanceData.h index bf1f4789f6..65bdfdf0a2 100644 --- a/src/ifcparse/IfcEntityInstanceData.h +++ b/src/ifcparse/IfcEntityInstanceData.h @@ -155,7 +155,7 @@ struct TypeEncoder_t> { using TypeEncoder = TypeEncoder_t; struct IFC_PARSE_API MutableAttributeValue { - int name_; + uint32_t name_; uint8_t index_; }; diff --git a/src/ifcparse/IfcException.h b/src/ifcparse/IfcException.h index 6689ebb7d4..f30a5e9f3f 100644 --- a/src/ifcparse/IfcException.h +++ b/src/ifcparse/IfcException.h @@ -57,7 +57,7 @@ class IFC_PARSE_API IfcAttributeOutOfRangeException : public IfcException { class IFC_PARSE_API IfcInvalidTokenException : public IfcException { public: IfcInvalidTokenException( - int token_start, + size_t token_start, const std::string& token_string, const std::string& expected_type) : IfcException( @@ -65,7 +65,7 @@ class IFC_PARSE_API IfcInvalidTokenException : public IfcException { boost::lexical_cast(token_start) + " invalid " + expected_type) {} IfcInvalidTokenException( - int token_start, + size_t token_start, char character) : IfcException( std::string("Unexpected '") + std::string(1, character) + "' at offset " + diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index 25cd414c4c..b887cbbfdf 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -56,7 +56,7 @@ namespace { constexpr bool is_type_in_variant_v = is_type_in_variant::value; template - void dispatch_token(int instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) { + void dispatch_token(boost::optional instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) { if (t.type == IfcParse::Token_BINARY) { fn(IfcParse::TokenFunc::asBinary(t)); } else if (IfcParse::TokenFunc::isBool(t)) { @@ -89,7 +89,7 @@ namespace { } template - void construct_(int instance_id, int attribute_id, IfcParse::parse_context& p, const IfcParse::aggregation_type* aggr, Fn fn) { + void construct_(boost::optional instance_id, int attribute_id, IfcParse::parse_context& p, const IfcParse::aggregation_type* aggr, Fn fn) { if (p.tokens_.empty()) { // @todo instead of ugly if-else we could also default initialize the respective // variant types below. @@ -234,7 +234,7 @@ namespace { } } -IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size, int resolve_reference_index, bool coerce_attribute_count) { +IfcEntityInstanceData IfcParse::parse_context::construct(boost::optional name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size, int resolve_reference_index, bool coerce_attribute_count) { std::vector parameter_types; std::unique_ptr transient_named_type; @@ -259,7 +259,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re expected_size && *expected_size != tokens_.size()) { size_t expected = expected_size ? *expected_size : parameter_types.size(); - Logger::Warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + " for instance #" + std::to_string(name > 0 ? name : 0)); + Logger::Warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + (name ? std::string(" for instance #" + std::to_string(*name)) : std::string(""))); } if (tokens_.empty()) { @@ -289,12 +289,12 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re if constexpr (std::is_same_v, IfcParse::Token>) { dispatch_token(name, index, v, param_type && param_type->as_named_type() ? param_type->as_named_type()->declared_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](auto v) { if constexpr (std::is_same_v, IfcParse::reference_or_simple_type>) { - if (name > 0) { + if (name) { references_to_resolve.push_back(std::make_pair( // @todo previously this was storage but apparently the // pointer is not constant with the moving and temporary nature // maybe it ought to be and in that case a pointer is more direct - MutableAttributeValue{ name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index }, + MutableAttributeValue{ (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index }, v )); } @@ -311,12 +311,12 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re } construct_<0>(name, index, *v, pt ? pt->as_aggregation_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](const auto& v) { if constexpr (std::is_same_v, std::vector>) { - if (name > 0) { - references_to_resolve.push_back({ {name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v }); + if (name) { + references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v }); } } else if constexpr (std::is_same_v, std::vector>>) { - if (name > 0) { - references_to_resolve.push_back({ {name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v }); + if (name) { + references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v }); } } else { storage.set(index, v); @@ -652,6 +652,8 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) { } std::optional> IfcParse::InstanceStreamer::read_instance() { + // std::cout << "global: " << stream_->tell() << std::endl; + std::optional> return_value; if (header_ && yielded_header_instances_ < 3) { @@ -679,7 +681,7 @@ std::optionalstream->eof && !current_id) { + while (good_ && !lexer_->stream->eof() && !current_id) { if (token_stream_[0].type == IfcParse::Token_IDENTIFIER && token_stream_[1].type == IfcParse::Token_OPERATOR && token_stream_[1].value_char == '=' && @@ -734,11 +736,13 @@ std::optionalstream->eof && next_token.type == Token_NONE) { + if (!lexer_->stream->eof() && next_token.type == Token_NONE) { good_ = file_open_status::INVALID_SYNTAX; break; } + // std::cout << next_token.startPos << " " << TokenFunc::toString(next_token) << std::endl; + token_stream_.push_back(next_token); } diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index b14cea367d..a77cd9c476 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -88,12 +88,11 @@ IFC_PARSE_API filetype guess_file_type(const std::string& fn); class IFC_PARSE_API InstanceStreamer { private: - IfcSpfStream* stream_; + FileReader* stream_; IfcSpfLexer* lexer_; IfcSpfHeader* header_; boost::circular_buffer token_stream_; const IfcParse::schema_definition* schema_; - const IfcParse::declaration* ifcroot_type_; IfcParse::impl::in_memory_file_storage storage_; IfcParse::file_open_status good_ = IfcParse::file_open_status::SUCCESS; int progress_; @@ -104,7 +103,7 @@ public: bool coerce_attribute_count = true; operator bool() const { - return good_ && !lexer_->stream->eof; + return good_ && !lexer_->stream->eof(); } IfcParse::file_open_status status() const { @@ -131,7 +130,13 @@ public: return storage_.steal_instances(); } - InstanceStreamer(const std::string& fn); + bool has_semicolon() const; + + void push_page(const std::string& page); + + InstanceStreamer(); + + InstanceStreamer(const std::string& fn, bool mmap=false); InstanceStreamer(void* data, int length); @@ -201,14 +206,44 @@ private: public: #ifdef USE_MMAP - IfcFile(const std::string& path, bool mmap = false); -#else - IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false); + /// + /// Constructs an IfcFile object from a file path, optionally using memory-mapped I/O, only supports IFC-SPF files. + /// + /// UTF-8 file path to an IFC-SPF file + /// Whether to use memory-mapped I/O + IfcFile(const std::string& path, bool mmap); #endif + /// + /// Constructs an IfcFile object from a file path, supports IFC-SPF and the IfcOpenShell-specific RocksDB format. + /// + /// UTF-8 file path to an IFC-SPF file or RocksDB database directory + /// File type of the path + /// Whether to open in read-only mode, only supported on RocksDB databases + IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false); + + /// + /// Constructs an IfcFile object from a stream containing IFC-SPF data. + /// IfcFile(std::istream& stream, int length); + + /// + /// Constructs an IfcFile object from a memory buffer containing IFC-SPF data. + /// IfcFile(void* data, int length); - IfcFile(IfcParse::IfcSpfStream* stream); - // @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called + + /// + /// Constructs an IfcFile object from a given IFC SPF stream. + /// + /// A pointer to an IfcParse::FileReader object representing the input IFC SPF data stream. + IfcFile(IfcParse::FileReader* stream); + + /// + /// Constructs an IfcFile object with the specified schema, file type, and file path. + /// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called + /// + /// Pointer to the schema definition to use. Defaults to the IFC4 schema if not specified. + /// The file type to use for the file. Defaults to FT_AUTODETECT. + /// The file system path to the IFC file. Defaults to an empty string. IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = FT_AUTODETECT, const std::string& path = ""); ~IfcFile(); diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index bbf80c3b65..8459f5a0d6 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -26,7 +26,7 @@ #include "IfcLogger.h" #include "IfcSchema.h" #include "IfcSIPrefix.h" -#include "IfcSpfStream.h" +#include "FileReader.h" #include "utils.h" #include @@ -102,167 +102,7 @@ void init_locale() { #endif -// -// Opens the file and gets the filesize -// -#ifdef USE_MMAP -IfcSpfStream::IfcSpfStream(const std::string& path, bool mmap) -#else -IfcSpfStream::IfcSpfStream(const std::string& path) -#endif - : stream_(0), - buffer_(0), - valid(false), - eof(false) { -#ifdef _MSC_VER - std::wstring fn_ws = IfcUtil::path::from_utf8(path); - const wchar_t* fn_wide = fn_ws.c_str(); - -#ifdef USE_MMAP - if (mmap) { - mfs = boost::iostreams::mapped_file_source(boost::filesystem::wpath(fn_wide)); - } else { -#endif - stream_ = _wfopen(fn_wide, L"rb"); -#ifdef USE_MMAP - } -#endif - -#else - -#ifdef USE_MMAP - if (mmap) { - mfs = boost::iostreams::mapped_file_source(path); - } else { -#endif - stream_ = fopen(path.c_str(), "rb"); -#ifdef USE_MMAP - } -#endif - -#endif - -#ifdef USE_MMAP - if (mmap) { - if (!mfs.is_open()) { - return; - } - - valid = true; - buffer_ = mfs.data(); - ptr_ = 0; - len_ = mfs.size(); - } else { -#endif - if (stream_ == NULL) { - return; - } - - valid = true; - fseek(stream_, 0, SEEK_END); - size = (unsigned int)ftell(stream_); - rewind(stream_); - char* buffer_rw = new char[size]; - len_ = (unsigned int)fread(buffer_rw, 1, size, stream_); - buffer_ = buffer_rw; - eof = len_ == 0; - ptr_ = 0; - fclose(stream_); - stream_ = nullptr; -#ifdef USE_MMAP - } -#endif -} - -IfcSpfStream::IfcSpfStream(std::istream& stream, int length) - : stream_(0), - buffer_(0) { - eof = false; - size = length; - char* buffer_rw = new char[size]; - stream.read(buffer_rw, size); - buffer_ = buffer_rw; - valid = stream.gcount() == size; - ptr_ = 0; - len_ = length; -} - -IfcSpfStream::IfcSpfStream(void* data, int length) - : stream_(0), - buffer_(0) { - eof = false; - size = length; - buffer_ = (char*)data; - valid = true; - ptr_ = 0; - len_ = length; -} - -IfcSpfStream::~IfcSpfStream() { - Close(); -} - -void IfcSpfStream::Close() { -#ifdef USE_MMAP - if (mfs.is_open()) { - mfs.close(); - return; - } -#endif - delete[] buffer_; - if (stream_ != nullptr) { - fclose(stream_); - } -} - -// -// Seeks an arbitrary position in the file -// -void IfcSpfStream::Seek(unsigned int offset) { - ptr_ = offset; - if (ptr_ >= len_) { - throw IfcException("Reading outside of file limits"); - } - eof = false; -} - -// -// Returns the character at the cursor -// -char IfcSpfStream::Peek() { - return buffer_[ptr_]; -} - -// -// Returns the character at specified offset -// -char IfcSpfStream::Read(unsigned int offset) { - return buffer_[offset]; -} - -// -// Returns the cursor position -// -unsigned int IfcSpfStream::Tell() const { - return ptr_; -} - -// -// Increments cursor and reads new chunk if necessary -// -void IfcSpfStream::Inc() { - if (++ptr_ == len_) { - eof = true; - return; - } - const char current = IfcSpfStream::Peek(); - if (current == '\n' || current == '\r') { - // NB this is recursive. It might as well be a loop. - IfcSpfStream::Inc(); - } -} - -IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_) { +IfcSpfLexer::IfcSpfLexer(IfcParse::FileReader* stream_) { stream = stream_; decoder_ = new IfcCharacterDecoder(stream_); } @@ -271,12 +111,12 @@ IfcSpfLexer::~IfcSpfLexer() { delete decoder_; } -unsigned int IfcSpfLexer::skipWhitespace() const { - unsigned int index = 0; - while (!stream->eof) { - char character = stream->Peek(); +size_t IfcSpfLexer::skipWhitespace() const { + size_t index = 0; + while (!stream->eof()) { + char character = stream->peek(); if ((character == ' ' || character == '\r' || character == '\n' || character == '\t')) { - stream->Inc(); + stream->increment(); ++index; } else { break; @@ -285,22 +125,22 @@ unsigned int IfcSpfLexer::skipWhitespace() const { return index; } -unsigned int IfcSpfLexer::skipComment() const { - char character = stream->Peek(); +size_t IfcSpfLexer::skipComment() const { + char character = stream->peek(); if (character != '/') { return 0; } - stream->Inc(); - character = stream->Peek(); + stream->increment(); + character = stream->peek(); if (character != '*') { - stream->Seek(stream->Tell() - 1); + stream->seek(stream->tell() - 1); return 0; } - unsigned int index = 2; + size_t index = 2; char intermediate = 0; - while (!stream->eof) { - character = stream->Peek(); - stream->Inc(); + while (!stream->eof()) { + character = stream->peek(); + stream->increment(); ++index; if (character == '/' && intermediate == '*') { break; @@ -315,19 +155,20 @@ unsigned int IfcSpfLexer::skipComment() const { // Token IfcSpfLexer::Next() { - if (stream->eof) { + if (stream->eof()) { return Token{}; } while ((skipWhitespace() != 0U) || (skipComment() != 0U)) { } - if (stream->eof) { + if (stream->eof()) { return Token{}; } - unsigned int pos = stream->Tell(); - char character = stream->Peek(); + auto& str = GetTempString(); + auto pos = stream->tell(); + char character = stream->read(); // If the cursor is at [()=,;$*] we know token consists of single char if (character == '(' || @@ -336,69 +177,45 @@ Token IfcSpfLexer::Next() { character == ',' || character == ';' || character == '$' || - character == '*') { - stream->Inc(); - return OperatorTokenPtr(this, pos, pos + 1); + character == '*') + { + return OperatorTokenPtr(this, pos, character); } - int len = 0; - - while (!stream->eof) { - - // Read character and increment pointer if not starting a new token - character = stream->Peek(); - if ((len != 0) && (character == '(' || - character == ')' || - character == '=' || - character == ',' || - character == ';' || - character == '/')) { - break; - } - stream->Inc(); - len++; - + if (character == '\'') { // If a string is encountered defer processing to the IfcCharacterDecoder - if (character == '\'') { - decoder_->skip(); + str = *decoder_; + } else { + str.assign(&character, 1); + + while (!stream->eof()) { + // Read character and increment pointer if not starting a new token + character = stream->peek(); + if (character == '(' || + character == ')' || + character == '=' || + character == ',' || + character == ';' || + character == '/') { + break; + } + str.push_back(character); + stream->increment(); } } - Token t; - if (len != 0) { - t = GeneralTokenPtr(this, pos, stream->Tell()); - } else { - t = Token{}; - } - // std::wcout << "token: " << pos << " " << TokenFunc::asStringRef(t).c_str() << std::endl; - return t; -} - -bool IfcSpfStream::is_eof_at(unsigned int local_ptr) const { - return local_ptr >= len_; -} - -void IfcSpfStream::increment_at(unsigned int& local_ptr) { - if (++local_ptr == len_) { - return; - } - const char current = IfcSpfStream::peek_at(local_ptr); - if (current == '\n' || current == '\r') { - IfcSpfStream::increment_at(local_ptr); - } -} - -char IfcSpfStream::peek_at(unsigned int local_ptr) { - return buffer_[local_ptr]; + return GeneralTokenPtr(this, pos, str); } // // Reads a std::string from the file at specified offset // Omits whitespace and comments // -void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) { +void IfcSpfLexer::TokenString(size_t offset, std::string& buffer) { buffer.clear(); - while (!stream->is_eof_at(offset)) { - char character = stream->peek_at(offset); + auto local_stream = *this->stream; + local_stream.seek(offset); + while (!local_stream.eof()) { + char character = local_stream.peek(); if (!buffer.empty() && (character == '(' || character == ')' || character == '=' || @@ -407,7 +224,7 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) { character == '/')) { break; } - stream->increment_at(offset); + local_stream.increment(); if (character == ' ' || character == '\r' || character == '\n' || @@ -416,6 +233,7 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) { } if (character == '\'') { // todo, make decoder use local offset ptr + auto offset = local_stream.tell(); buffer = decoder_->get(offset); break; } @@ -424,10 +242,11 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) { } //Note: according to STEP standard, there may be newlines in tokens -inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned end, std::string& oDestination) { +/* +inline void RemoveTokenSeparators(FileReader* stream, size_t start, size_t end, std::string& oDestination) { oDestination.clear(); for (unsigned i = start; i < end; i++) { - char character = stream->Read(i); + char character = stream->get(i); if (character == ' ' || character == '\r' || character == '\n' || @@ -437,6 +256,7 @@ inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned oDestination += character; } } +*/ bool ParseInt(const char* pStart, int& val) { char* pEnd; @@ -481,22 +301,17 @@ bool ParseBool(const char* pStart, int& val) { return true; } -Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) { - char first = lexer->stream->Read(start); - Token token(lexer, start, end, Token_OPERATOR); - token.value_char = first; +Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, size_t start, char data) { + Token token(lexer, start, Token_OPERATOR); + token.value_char = data; return token; } -Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) { - Token token(lexer, start, end, Token_NONE); - - //extract token into temp buffer (remove eol-s, no encoding changes) - std::string& tokenStr = lexer->GetTempString(); - RemoveTokenSeparators(lexer->stream, start, end, tokenStr); +Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, size_t start, const std::string& tokenStr) { + Token token(lexer, start, Token_NONE); //determine type of the token - char first = lexer->stream->Read(start); + const char& first = tokenStr.front(); if (first == '#') { token.type = Token_IDENTIFIER; if (!ParseInt(tokenStr.c_str() + 1, token.value_int)) { @@ -521,6 +336,11 @@ Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end token.type = Token_KEYWORD; } + // todo STRING/BINARY/ENUM/KEYWORD not stored + // tokenStr is gone after this. + // Orrrr we could only free pages when after a full instance is formed because after that no + // references to prior token ranges exist anymore. + return token; } @@ -676,7 +496,25 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) { std::string TokenFunc::toString(const Token& token) { std::string result; - token.lexer->TokenString(token.startPos, result); + if (token.type == Token_OPERATOR) { + result.push_back(token.value_char); + } else if (token.type == Token_INT) { + result = std::to_string(token.value_int); + } else if (token.type == Token_BOOL) { + if (token.value_int == 1) { + result = ".T."; + } else if (token.value_int == 0) { + result = ".F."; + } else { + result = ".U."; + } + } else if (token.type == Token_FLOAT) { + std::ostringstream oss; + oss << std::setprecision(15) << token.value_double; + result = oss.str(); + } else { + token.lexer->TokenString(token.startPos, result); + } return result; } @@ -684,7 +522,7 @@ std::string TokenFunc::toString(const Token& token) { // Reads the arguments from a list of token // Aditionally, registers the ids (i.e. #[\d]+) in the inverse map // -void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { +void IfcParse::impl::in_memory_file_storage::load(boost::optional entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { Token next = tokens->Next(); /* @@ -708,8 +546,8 @@ void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name, load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); } else { return_value++; - if (TokenFunc::isIdentifier(next) && entity) { - register_inverse(entity_instance_name, entity, next.value_int, attribute_index == -1 ? attribute_index_within_data : attribute_index); + if (TokenFunc::isIdentifier(next) && entity && entity_instance_name) { + register_inverse(*entity_instance_name, entity, next.value_int, attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); } if (TokenFunc::isKeyword(next)) { @@ -758,10 +596,10 @@ IfcEntityInstanceData IfcParse::impl::in_memory_file_storage::read(unsigned int } void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const { - unsigned int old_offset = tokens->stream->Tell(); + auto old_offset = tokens->stream->tell(); Token semilocon = tokens->Next(); if (!TokenFunc::isOperator(semilocon, ';')) { - tokens->stream->Seek(old_offset); + tokens->stream->seek(old_offset); } } @@ -783,7 +621,7 @@ void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from namespace { template - std::string to_string_fixed_width(const T& t, size_t w) { + std::string to_string_fixed_width(const T& t, size_t) { // @todo currently inactive std::ostringstream oss; oss << /*std::setfill('0') << std::setw(w) <<*/ t; @@ -1345,11 +1183,27 @@ IfcUtil::IfcBaseClass::set_attribute_value(const std::string& s, const T& t) { // #ifdef USE_MMAP IfcFile::IfcFile(const std::string& fn, bool mmap) { - IfcSpfStream s(fn, mmap); - storage_ = impl::in_memory_file_storage{}; - std::get(storage_).read_from_stream(&s); + std::unique_ptr s; + if (mmap) { + s = std::make_unique(fn, FileReader::mmap_tag{}); + } else { + s = std::make_unique(fn); + } + + storage_.emplace<1>(this); + std::get(storage_).read_from_stream(&*s, schema_, max_id_); + + if ((good_ = std::get(storage_).good_)) { + // @todo unify these names, it's already confusing enough as it stands + byid_ = decltype(byid_)(&std::get(storage_).byid_); + byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); + byguid_ = decltype(byguid_)(&std::get(storage_).byguid_); + } + + ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; } -#else +#endif + IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly) : schema_(nullptr) , max_id_(0) @@ -1360,11 +1214,11 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly) ty = guess_file_type(path); } if (ty == FT_IFCSPF) { - IfcSpfStream s(path); + FileReader s(path); storage_.emplace<1>(this); std::get(storage_).read_from_stream(&s, schema_, max_id_); - if (good_ = std::get(storage_).good_) { + if ((good_ = std::get(storage_).good_)) { // @todo unify these names, it's already confusing enough as it stands byid_ = decltype(byid_)(&std::get(storage_).byid_); byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); @@ -1398,13 +1252,18 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly) } ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; } -#endif IfcFile::IfcFile(std::istream& stream, int length) : schema_(nullptr) , max_id_(0) { - IfcSpfStream s(stream, length); + FileReader s(FileReader::caller_fed_tag{}); + + std::string string_data; + string_data.resize(length); + stream.read(string_data.data(), length); + s.push_next_page(string_data); + storage_.emplace<1>(this); std::get(storage_).read_from_stream(&s, schema_, max_id_); good_ = std::get(storage_).good_; @@ -1419,7 +1278,8 @@ IfcFile::IfcFile(void* data, int length) : schema_(nullptr) , max_id_(0) { - IfcSpfStream s(data, length); + FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{}); + storage_.emplace<1>(this); std::get(storage_).read_from_stream(&s, schema_, max_id_); good_ = std::get(storage_).good_; @@ -1430,7 +1290,7 @@ IfcFile::IfcFile(void* data, int length) byguid_ = decltype(byguid_)(&std::get(storage_).byguid_); } -IfcFile::IfcFile(IfcParse::IfcSpfStream* s) +IfcFile::IfcFile(IfcParse::FileReader* s) : schema_(nullptr) , max_id_(0) { @@ -1475,18 +1335,65 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s setDefaultHeaderValues(); } -IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn) - : stream_(new IfcSpfStream(fn)) +bool IfcParse::InstanceStreamer::has_semicolon() const +{ + auto local_stream = stream_->clone(); + auto local_lexer = IfcSpfLexer(&local_stream); + Token t = local_lexer.Next(); + while (t.type != Token_NONE) { + if (TokenFunc::isOperator(t, ';')) { + return true; + } + // probably the issue is that this moves the cursor past the page boundary on the local stream which also affects the global one + // 'freeze' the stream so that no pages are cleared + t = local_lexer.Next(); + } + std::cout << "local: " << local_stream.tell() << " global: " << stream_->tell() << std::endl; + return false; +} + +void IfcParse::InstanceStreamer::push_page(const std::string& page) +{ + std::cout << "global: " << stream_->tell() << std::endl; + stream_->push_next_page(page); + if (good_ == file_open_status::NO_HEADER) { + header_ = new IfcParse::IfcSpfHeader(lexer_); + if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) { + try { + schema_ = IfcParse::schema_by_name(header_->file_schema()->schema_identifiers().front()); + good_ = file_open_status::SUCCESS; + } catch (const IfcParse::IfcException&) { + } + } + storage_.file = nullptr; + storage_.schema = schema_; + storage_.tokens = lexer_; + storage_.references_to_resolve = &references_to_resolve_; + } +} + +IfcParse::InstanceStreamer::InstanceStreamer() + : stream_(new FileReader(FileReader::caller_fed_tag{})) + , lexer_(new IfcSpfLexer(stream_)) + , token_stream_(3, Token{}) + , schema_(nullptr) + , progress_(0) +{ + init_locale(); + good_ = file_open_status::NO_HEADER; +} + +IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap) + : stream_(mmap ? new FileReader(fn, FileReader::mmap_tag{}) : new FileReader(fn)) , lexer_(new IfcSpfLexer(stream_)) , token_stream_(3, Token{}) , schema_(nullptr) - , ifcroot_type_(nullptr) , progress_(0) { init_locale(); good_ = file_open_status::NO_HEADER; - if (*stream_) { + if (stream_->size() && !stream_->eof()) { header_ = new IfcParse::IfcSpfHeader(lexer_); if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) { try { @@ -1503,17 +1410,16 @@ IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn) } IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length) - : stream_(new IfcSpfStream(data, length)) + : stream_(new FileReader(std::string((char*) data, length), FileReader::caller_fed_tag{})) , lexer_(new IfcSpfLexer(stream_)) , token_stream_(3, Token{}) , schema_(nullptr) - , ifcroot_type_(nullptr) , progress_(0) { init_locale(); good_ = file_open_status::NO_HEADER; - if (*stream_) { + if (stream_->size() && !stream_->eof()) { header_ = new IfcParse::IfcSpfHeader(lexer_); if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) { try { @@ -1535,7 +1441,6 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* , header_(nullptr) , token_stream_(3, Token{}) , schema_(schema) - , ifcroot_type_(schema->declaration_by_name("IfcRoot")) , progress_(0) { init_locale(); @@ -1546,14 +1451,14 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* storage_.references_to_resolve = &references_to_resolve_; } -void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfStream* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) { +void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) { // Initialize a "C" locale for locale-independent // number parsing. See comment above on line 41. init_locale(); tokens = nullptr; - if (!s->valid) { + if (!s->size() || s->eof()) { // @todo set good on parent file good_ = file_open_status::READ_ERROR; return; @@ -1608,7 +1513,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst))); instance->file_ = file; - instance->id_ = current_id; + instance->id_ = (uint32_t) current_id; if (instance->declaration().is(*ifcroot_type_)) { try { @@ -1641,7 +1546,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt } // byidentity_[instance->identity()] = instance; - byid_.insert({ current_id, instance }); + byid_.insert({(uint32_t) current_id, instance }); // @nb cannot assign to byid_; // byid_[current_id] = instance; @@ -1693,10 +1598,10 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt } else if (auto* inst = std::get_if(v)) { byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst); } - } else if (auto* v = std::get_if>(&p.second)) { + } else if (auto* vv = std::get_if>(&p.second)) { aggregate_of_instance::ptr instances(new aggregate_of_instance); - instances->reserve(v->size()); - for (const auto& vi : *v) { + instances->reserve(vv->size()); + for (const auto& vi : *vv) { if (auto* name = std::get_if(&vi)) { auto it = byid_.find(*name); if (it == byid_.end()) { @@ -1726,9 +1631,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt } else { Logger::Error("Duplicate definition for instance reference"); } - } else if (auto* v = std::get_if>>(&p.second)) { + } else if (auto* vvv = std::get_if>>(&p.second)) { aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance); - for (const auto& vi : *v) { + for (const auto& vi : *vvv) { std::vector inner; for (const auto& vii : vi) { if (auto* name = std::get_if(&vii)) { @@ -2003,15 +1908,15 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) auto attr = entity->get_attribute_value(i); IfcUtil::ArgumentType attr_type = attr.type(); - IfcParse::declaration* decl = 0; + IfcParse::declaration* potentially_length_measure_decl = 0; if (entity->declaration().as_entity() != nullptr) { - decl = 0; + potentially_length_measure_decl = 0; const parameter_type* pt = entity->declaration().as_entity()->attribute_by_index(i)->type_of_attribute(); while (pt->as_aggregation_type() != nullptr) { pt = pt->as_aggregation_type()->type_of_element(); } if (pt->as_named_type() != nullptr) { - decl = pt->as_named_type()->declared_type(); + potentially_length_measure_decl = pt->as_named_type()->declared_type(); } } @@ -2050,7 +1955,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id) } new_entity->set_attribute_value(i, new_instances); - } else if ((decl != nullptr) && decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) { + } else if ((potentially_length_measure_decl != nullptr) && potentially_length_measure_decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) { if (boost::math::isnan(conversion_factor)) { std::pair this_file_unit = {nullptr, 1.0}; std::pair other_file_unit = {nullptr, 1.0}; @@ -2283,8 +2188,6 @@ void IfcFile::process_deletion_(IfcUtil::IfcBaseClass* entity) { byid_.erase(entity->id()); - const IfcParse::declaration* ty = &entity->declaration(); - remove_type_ref(entity); // entity_file_map is in place to prevent duplicate definitions with usage of add(). @@ -2891,7 +2794,7 @@ AttributeValue IfcEntityInstanceData::get_attribute_value(void* storage, const I if (storage_) { return AttributeValue(storage_, (uint8_t)index); } else { - return AttributeValue((IfcParse::impl::rocks_db_file_storage*)storage, identity, decl, index); + return AttributeValue((IfcParse::impl::rocks_db_file_storage*)storage, identity, decl, (uint8_t) index); } } diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index 9430866ae7..a034fde218 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -32,7 +32,7 @@ #include "ifc_parse_api.h" #include "IfcBaseClass.h" #include "IfcCharacterDecoder.h" -#include "IfcSpfStream.h" +#include "FileReader.h" #include "macros.h" #include "storage.h" @@ -103,27 +103,27 @@ class IFC_PARSE_API TokenFunc { // Functions for creating Tokens from an arbitary file offset // The first 4 bits are reserved for Tokens of type ()=,;$* // -Token OperatorTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end); -Token GeneralTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end); +Token OperatorTokenPtr(IfcSpfLexer* tokens, size_t start, char data); +Token GeneralTokenPtr(IfcSpfLexer* tokens, size_t start, const std::string& data); -/// A stream of tokens to be read from a IfcSpfStream. +/// A stream of tokens to be read from a FileReader. class IFC_PARSE_API IfcSpfLexer { private: IfcCharacterDecoder* decoder_; - unsigned int skipWhitespace() const; - unsigned int skipComment() const; + size_t skipWhitespace() const; + size_t skipComment() const; public: std::string& GetTempString() const { static my_thread_local std::string string; return string; } - IfcSpfStream* stream; + FileReader* stream; // IfcFile* file; - IfcSpfLexer(IfcSpfStream* stream); + IfcSpfLexer(FileReader* stream); Token Next(); ~IfcSpfLexer(); - void TokenString(unsigned int offset, std::string& result); + void TokenString(size_t offset, std::string& result); }; IFC_PARSE_API aggregate_of_instance::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level = -1); diff --git a/src/ifcparse/IfcSpfHeader.cpp b/src/ifcparse/IfcSpfHeader.cpp index 7e264631d6..c293216046 100644 --- a/src/ifcparse/IfcSpfHeader.cpp +++ b/src/ifcparse/IfcSpfHeader.cpp @@ -35,7 +35,7 @@ namespace { parse_context pc; storage->tokens->Next(); storage->load(-1, nullptr, pc, -1); - return pc.construct(-1, *storage->references_to_resolve, nullptr, s, -1); + return pc.construct(boost::none, *storage->references_to_resolve, nullptr, s, -1); } else { // std::unreachable(); return IfcEntityInstanceData(in_memory_attribute_storage(10)); diff --git a/src/ifcparse/IfcSpfStream.h b/src/ifcparse/IfcSpfStream.h deleted file mode 100644 index 7d42a9cd52..0000000000 --- a/src/ifcparse/IfcSpfStream.h +++ /dev/null @@ -1,84 +0,0 @@ -/******************************************************************************** - * * - * This file is part of IfcOpenShell. * - * * - * IfcOpenShell is free software: you can redistribute it and/or modify * - * it under the terms of the Lesser GNU General Public License as published by * - * the Free Software Foundation, either version 3.0 of the License, or * - * (at your option) any later version. * - * * - * IfcOpenShell is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * Lesser GNU General Public License for more details. * - * * - * You should have received a copy of the Lesser GNU General Public License * - * along with this program. If not, see . * - * * - ********************************************************************************/ - -/********************************************************************************* - * * - * Reads a file and provides functions to access its * - * contents randomly and character by character * - * * - ********************************************************************************/ - -#ifndef IFCSPFSTREAM_H -#define IFCSPFSTREAM_H - -#include "ifc_parse_api.h" - -#include - -#ifdef USE_MMAP -#include -#endif - -namespace IfcParse { -/// The IfcSpfStream class represents a ISO 10303-21 IFC-SPF file in memory. -/// The file is interpreted as a sequence of tokens which are lazily -/// interpreted only when requested. -class IFC_PARSE_API IfcSpfStream { - private: -#ifdef USE_MMAP - boost::iostreams::mapped_file_source mfs; -#endif - FILE* stream_; - const char* buffer_; - unsigned int ptr_; - unsigned int len_; - - public: - bool valid; - bool eof; - unsigned int size; -#ifdef USE_MMAP - IfcSpfStream(const std::string& path, bool mmap = false); -#else - IfcSpfStream(const std::string& path); -#endif - IfcSpfStream(std::istream& stream, int length); - IfcSpfStream(void* data, int length); - ~IfcSpfStream(); - /// Returns the character at the cursor - char Peek(); - /// Returns the character at specified offset - char Read(unsigned int offset); - /// Increment the file cursor and reads new page if necessary - void Inc(); - void Close(); - /// Moves the file cursor to an arbitrary offset in the file - void Seek(unsigned int offset); - /// Returns the cursor position - unsigned int Tell() const; - - bool is_eof_at(unsigned int) const; - void increment_at(unsigned int&); - char peek_at(unsigned int); - - operator bool() const { return valid && !eof; } -}; -} // namespace IfcParse - -#endif diff --git a/src/ifcparse/IfcUtil.cpp b/src/ifcparse/IfcUtil.cpp index 7aab44d391..a47083262f 100644 --- a/src/ifcparse/IfcUtil.cpp +++ b/src/ifcparse/IfcUtil.cpp @@ -76,8 +76,8 @@ void aggregate_of_instance::push(const aggregate_of_instance::ptr& instance) { } } } -unsigned int aggregate_of_instance::size() const { return (unsigned int)list_.size(); } -void aggregate_of_instance::reserve(unsigned capacity) { list_.reserve((size_t)capacity); } +size_t aggregate_of_instance::size() const { return list_.size(); } +void aggregate_of_instance::reserve(size_t capacity) { list_.reserve(capacity); } aggregate_of_instance::it aggregate_of_instance::begin() { return list_.begin(); } aggregate_of_instance::it aggregate_of_instance::end() { return list_.end(); } IfcUtil::IfcBaseClass* aggregate_of_instance::operator[](int i) { diff --git a/src/ifcparse/aggregate_of_instance.h b/src/ifcparse/aggregate_of_instance.h index aff12f426a..bcce94ce6e 100644 --- a/src/ifcparse/aggregate_of_instance.h +++ b/src/ifcparse/aggregate_of_instance.h @@ -48,8 +48,8 @@ class IFC_PARSE_API aggregate_of_instance { it begin(); it end(); IfcUtil::IfcBaseClass* operator[](int index); - unsigned int size() const; - void reserve(unsigned capacity); + size_t size() const; + void reserve(size_t capacity); bool contains(IfcUtil::IfcBaseClass*) const; template @@ -81,7 +81,7 @@ class aggregate_of { } it begin() { return list_.begin(); } it end() { return list_.end(); } - unsigned int size() const { return (unsigned int)list_.size(); } + size_t size() const { return list_.size(); } aggregate_of_instance::ptr generalize() { aggregate_of_instance::ptr result(new aggregate_of_instance()); for (it i = begin(); i != end(); ++i) { diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h index db0e1f95b6..5729d54a39 100644 --- a/src/ifcparse/storage.h +++ b/src/ifcparse/storage.h @@ -125,7 +125,7 @@ namespace IfcParse { class IfcFile; class IfcSpfLexer; - class IfcSpfStream; + class FileReader; enum TokenType { Token_NONE, @@ -142,7 +142,7 @@ namespace IfcParse { struct Token { IfcSpfLexer* lexer; //TODO: remove it from here - unsigned startPos; + size_t startPos; TokenType type; union { char value_char; //types: OPERATOR @@ -154,7 +154,7 @@ namespace IfcParse { startPos(0), type(Token_NONE) { } - Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type) + Token(IfcSpfLexer* _lexer, size_t _startPos, TokenType _type) : lexer(_lexer), startPos(_startPos), type(_type) { @@ -184,7 +184,7 @@ namespace IfcParse { void push(IfcUtil::IfcBaseClass* inst); - IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size, int resolve_reference_index, bool coerce_attribute_count=true); + IfcEntityInstanceData construct(boost::optional name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional expected_size, int resolve_reference_index, bool coerce_attribute_count=true); }; namespace impl { @@ -195,7 +195,7 @@ namespace IfcParse { } IfcParse::IfcSpfLexer* tokens; - // IfcParse::IfcSpfStream* stream; + // IfcParse::FileReader* stream; // Either one of these needs to be set IfcParse::IfcFile* file; @@ -260,7 +260,7 @@ namespace IfcParse { entities_by_ref_t byref_excl_; entity_instance_by_guid_t byguid_; - void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); + void load(boost::optional entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); void try_read_semicolon() const; void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); @@ -268,7 +268,7 @@ namespace IfcParse { // @todo is this still used IfcEntityInstanceData read(unsigned int index); - void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id); + void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id); file_open_status good_ = file_open_status::SUCCESS;