From 603cedc487850f1123d5873addd5d125fcc2c30a Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Fri, 27 Mar 2026 20:45:13 +0100 Subject: [PATCH] Try some things: (a) fewer allocations - parse context pool; lexer string pool (b) SWAR process multiple chars at once in keywords/enums/strs/stc. --- src/ifcconvert/IfcConvert.cpp | 4 +- src/ifcparse/FileReader.cpp | 543 ++++++++++++---------- src/ifcparse/FileReader.h | 304 ++++++++---- src/ifcparse/IfcCharacterDecoder.cpp | 127 ++++- src/ifcparse/IfcCharacterDecoder.h | 11 +- src/ifcparse/IfcFile.cpp | 131 +----- src/ifcparse/IfcFile.h | 36 +- src/ifcparse/IfcParse.cpp | 666 ++++++++++++++++++++------- src/ifcparse/IfcParse.h | 7 +- src/ifcparse/IfcSchema.h | 20 +- src/ifcparse/IfcSpfHeader.cpp | 156 ++----- src/ifcparse/IfcSpfHeader.h | 26 +- src/ifcparse/storage.h | 77 +++- src/ifcwrap/IfcParseWrapper.i | 18 +- 14 files changed, 1266 insertions(+), 860 deletions(-) diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index b84a33d8f4..72b0311c22 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -522,9 +522,9 @@ int main(int argc, char** argv) { #ifdef HAVE_ICU if (!unicode_mode.empty()) { if (unicode_mode == "utf8") { - IfcParse::IfcCharacterDecoder::mode = IfcParse::IfcCharacterDecoder::UTF8; + IfcParse::RuntimeIfcCharacterDecoder::mode = IfcParse::RuntimeIfcCharacterDecoder::UTF8; } else if (unicode_mode == "escape") { - IfcParse::IfcCharacterDecoder::mode = IfcParse::IfcCharacterDecoder::JSON; + IfcParse::RuntimeIfcCharacterDecoder::mode = IfcParse::RuntimeIfcCharacterDecoder::ESCAPE; } else { cerr_ << "[Error] Invalid value for --unicode" << std::endl; print_options(serializer_options); diff --git a/src/ifcparse/FileReader.cpp b/src/ifcparse/FileReader.cpp index e16d26e641..f5fa47f619 100644 --- a/src/ifcparse/FileReader.cpp +++ b/src/ifcparse/FileReader.cpp @@ -1,18 +1,13 @@ #include "FileReader.h" #include +#include #include #include #include -#include #include -#include -#include -#include -#include #ifdef USE_MMAP -#include #include #endif @@ -21,306 +16,350 @@ namespace { #if defined(_WIN32) - inline void file_seek_abs(FILE* f, std::uint64_t off) { - if (_fseeki64(f, static_cast(off), SEEK_SET) != 0) - throw std::runtime_error("fseek failed"); +inline void file_seek_abs(FILE* f, std::uint64_t off) { + if (_fseeki64(f, static_cast(off), SEEK_SET) != 0) { + throw std::runtime_error("fseek failed"); } +} #else - inline void file_seek_abs(FILE* f, std::uint64_t off) { - if (fseeko(f, static_cast(off), SEEK_SET) != 0) - throw std::runtime_error("fseeko failed"); +inline void file_seek_abs(FILE* f, std::uint64_t off) { + if (fseeko(f, static_cast(off), SEEK_SET) != 0) { + throw std::runtime_error("fseeko failed"); } +} #endif } // namespace using namespace IfcParse; -struct FullBufferImpl final : FileReader::Impl { - std::vector buf_; - explicit FullBufferImpl(const std::string& fn) { +namespace { + +template +T gather_value(Fn&& fn) { + char bytes[sizeof(T)]; + for (size_t i = 0; i < sizeof(bytes); ++i) { + bytes[i] = fn(i); + } + T value; + std::memcpy(&value, bytes, sizeof(value)); + return value; +} + +} // namespace + +FullBufferImpl::FullBufferImpl(const std::string& fn) { #ifdef _MSC_VER - std::wstring fn_ws = IfcUtil::path::from_utf8(fn); - const wchar_t* fn_wide = fn_ws.c_str(); - auto stream = _wfopen(fn_wide, L"rb"); + std::wstring fn_ws = IfcUtil::path::from_utf8(fn); + const wchar_t* fn_wide = fn_ws.c_str(); + auto stream = _wfopen(fn_wide, L"rb"); #else - auto stream = fopen(fn.c_str(), "rb"); + auto stream = fopen(fn.c_str(), "rb"); #endif - fseek(stream, 0, SEEK_END); - buf_.resize((size_t)ftell(stream)); - rewind(stream); - buf_.resize((size_t) fread(buf_.data(), 1, buf_.capacity(), stream)); - fclose(stream); + if (!stream) { + throw std::runtime_error("Failed to open file"); } - size_t size() const override { return buf_.size(); } - char get(size_t pos) const override { - if (pos >= buf_.size()) throw std::out_of_range("get out of range"); - return buf_[pos]; + fseek(stream, 0, SEEK_END); + size_ = (size_t)ftell(stream); + buf_.resize(size_); + rewind(stream); + buf_.resize((size_t)fread(buf_.data(), 1, buf_.capacity(), stream)); + fclose(stream); +} + +size_t FullBufferImpl::size() const { return size_; } + +char FullBufferImpl::get(size_t pos) const { + if (pos >= buf_.size()) { + throw std::out_of_range("get out of range"); } -}; + return buf_[pos]; +} -struct PagedFileImpl final : FileReader::Impl { - std::string fn_; - FILE* fp_ = nullptr; - size_t file_size_ = 0; - size_t page_size_ = 4096; +uint64_t FullBufferImpl::get_u64(size_t pos) const { + if (pos + sizeof(uint64_t) > buf_.size()) { + throw std::out_of_range("get_u64 out of range"); + } + uint64_t value; + std::memcpy(&value, buf_.data() + pos, sizeof(value)); + return value; +} - // LRU cache - size_t capacity_ = 8; - mutable std::list lru_; - struct Entry { - FileReader::Page page; - std::list::iterator it; - }; - mutable std::unordered_map map_; +uint32_t FullBufferImpl::get_u32(size_t pos) const { + if (pos + sizeof(uint32_t) > buf_.size()) { + throw std::out_of_range("get_u32 out of range"); + } + uint32_t value; + std::memcpy(&value, buf_.data() + pos, sizeof(value)); + return value; +} - PagedFileImpl(const std::string& fn, size_t page_size, size_t cap) - : fn_(fn), page_size_(std::max(512, page_size)), capacity_(std::max(2, cap)) - { +void FullBufferImpl::pushNextPage(const std::string&) { + throw std::logic_error("push_next_page: backend does not support pushed mode"); +} + +void FullBufferImpl::dropPages(size_t) { +} + +PagedFileImpl::PagedFileImpl(const std::string& fn, size_t page_size, size_t cap) + : fn_(fn) + , page_size_(std::max(512, page_size)) + , capacity_(std::max(2, cap)) { #ifdef _MSC_VER - std::wstring fn_ws = IfcUtil::path::from_utf8(fn); - const wchar_t* fn_wide = fn_ws.c_str(); - fp_ = _wfopen(fn_wide, L"rb"); + std::wstring fn_ws = IfcUtil::path::from_utf8(fn); + const wchar_t* fn_wide = fn_ws.c_str(); + fp_ = _wfopen(fn_wide, L"rb"); #else - fp_ = fopen(fn.c_str(), "rb"); + fp_ = fopen(fn.c_str(), "rb"); #endif - fseek(fp_, 0, SEEK_END); - file_size_ = (size_t)ftell(fp_); - rewind(fp_); + if (!fp_) { + throw std::runtime_error("Failed to open file"); + } + fseek(fp_, 0, SEEK_END); + file_size_ = (size_t)ftell(fp_); + rewind(fp_); +} + +PagedFileImpl::~PagedFileImpl() { + if (fp_) { + std::fclose(fp_); + } + fp_ = nullptr; +} + +size_t PagedFileImpl::size() const { return file_size_; } + +char PagedFileImpl::get(size_t pos) const { + if (pos >= file_size_) { + throw std::out_of_range("get out of range"); + } + const size_t pidx = pos / page_size_; + const FileReaderPage& p = fetchPage_(pidx); + const size_t off = pos % page_size_; + if (off >= p.data.size()) { + throw std::out_of_range("offset beyond valid page bytes"); + } + return p.data[off]; +} + +uint64_t PagedFileImpl::get_u64(size_t pos) const { + if (pos + sizeof(uint64_t) > file_size_) { + throw std::out_of_range("get_u64 out of range"); } - ~PagedFileImpl() override { - if (fp_) std::fclose(fp_); - fp_ = nullptr; + const size_t pidx = pos / page_size_; + const FileReaderPage& p = fetchPage_(pidx); + const size_t off = pos % page_size_; + if (off + sizeof(uint64_t) <= p.data.size()) { + uint64_t value; + std::memcpy(&value, p.data.data() + off, sizeof(value)); + return value; } - size_t size() const override { return file_size_; } + return gather_value([&](size_t i) { return get(pos + i); }); +} - char get(size_t pos) const override { - if (pos >= file_size_) throw std::out_of_range("get out of range"); - const size_t pidx = pos / page_size_; - const FileReader::Page& p = fetchPage_(pidx); - const size_t off = pos % page_size_; - if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes"); - return p.data[off]; +uint32_t PagedFileImpl::get_u32(size_t pos) const { + if (pos + sizeof(uint32_t) > file_size_) { + throw std::out_of_range("get_u32 out of range"); } -private: - const FileReader::Page& fetchPage_(size_t idx) const { - auto it = map_.find(idx); - if (it != map_.end()) { - touch_(it); - return it->second.page; + const size_t pidx = pos / page_size_; + const FileReaderPage& p = fetchPage_(pidx); + const size_t off = pos % page_size_; + if (off + sizeof(uint32_t) <= p.data.size()) { + uint32_t value; + std::memcpy(&value, p.data.data() + off, sizeof(value)); + return value; + } + + return gather_value([&](size_t i) { return get(pos + i); }); +} + +void PagedFileImpl::pushNextPage(const std::string&) { + throw std::logic_error("push_next_page: backend does not support pushed mode"); +} + +void PagedFileImpl::dropPages(size_t) { +} + +const FileReaderPage& PagedFileImpl::fetchPage_(size_t idx) const { + auto it = map_.find(idx); + if (it != map_.end()) { + touch_(it); + return it->second.page; + } + + FileReaderPage pg; + pg.data.resize(page_size_); + const size_t begin = idx * page_size_; + const size_t avail = std::min(page_size_, file_size_ - begin); + + file_seek_abs(fp_, begin); + if (avail > 0) { + const size_t nread = std::fread(pg.data.data(), 1, avail, fp_); + if (nread != avail) { + throw std::runtime_error("Short fread on page"); } - // Load page from disk using persistent FILE* - FileReader::Page pg; - pg.data.resize(page_size_); - const size_t begin = idx * page_size_; - const size_t avail = std::min(page_size_, file_size_ - begin); - - file_seek_abs(fp_, begin); - if (avail > 0) { - const size_t nread = std::fread(pg.data.data(), 1, avail, fp_); - if (nread != avail) throw std::runtime_error("Short fread on page"); - } - // trim to actual size - pg.data.resize(avail); - - // Insert into LRU - if (map_.size() >= capacity_) evict_(); - lru_.push_front(idx); - auto lit = lru_.begin(); - auto [emplaced_it, ok] = map_.emplace(idx, Entry{ std::move(pg), lit }); - (void)ok; - return emplaced_it->second.page; } + pg.data.resize(avail); - void touch_(typename std::unordered_map::iterator it) const { - lru_.erase(it->second.it); - lru_.push_front(it->first); - it->second.it = lru_.begin(); + if (map_.size() >= capacity_) { + evict_(); } + lru_.push_front(idx); + auto lit = lru_.begin(); + auto [emplaced_it, ok] = map_.emplace(idx, Entry{std::move(pg), lit}); + (void)ok; + return emplaced_it->second.page; +} - void evict_() const { - if (lru_.empty()) return; - const size_t victim = lru_.back(); - lru_.pop_back(); - map_.erase(victim); +void PagedFileImpl::touch_(std::unordered_map::iterator it) const { + lru_.erase(it->second.it); + lru_.push_front(it->first); + it->second.it = lru_.begin(); +} + +void PagedFileImpl::evict_() const { + if (lru_.empty()) { + return; } -}; + const size_t victim = lru_.back(); + lru_.pop_back(); + map_.erase(victim); +} #ifdef USE_MMAP -struct MMapImpl final : FileReader::Impl { - boost::iostreams::mapped_file_source map_; - size_t size_ = 0; - - explicit MMapImpl(const std::string& fn) { - map_.open(boost::filesystem::path(IfcUtil::path::from_utf8(fn))); - if (!map_.is_open()) throw std::runtime_error("Failed to open mapped_file_source"); - size_ = static_cast(map_.size()); +MMapImpl::MMapImpl(const std::string& fn) { + map_.open(boost::filesystem::path(IfcUtil::path::from_utf8(fn))); + if (!map_.is_open()) { + throw std::runtime_error("Failed to open mapped_file_source"); } + size_ = static_cast(map_.size()); +} - size_t size() const override { return size_; } +size_t MMapImpl::size() const { return size_; } - char get(size_t pos) const override { - if (pos >= size_) throw std::out_of_range("get out of range"); - return map_.data()[pos]; +char MMapImpl::get(size_t pos) const { + if (pos >= size_) { + throw std::out_of_range("get out of range"); } -}; + return map_.data()[pos]; +} + +uint64_t MMapImpl::get_u64(size_t pos) const { + if (pos + sizeof(uint64_t) > size_) { + throw std::out_of_range("get_u64 out of range"); + } + uint64_t value; + std::memcpy(&value, map_.data() + pos, sizeof(value)); + return value; +} + +uint32_t MMapImpl::get_u32(size_t pos) const { + if (pos + sizeof(uint32_t) > size_) { + throw std::out_of_range("get_u32 out of range"); + } + uint32_t value; + std::memcpy(&value, map_.data() + pos, sizeof(value)); + return value; +} + +void MMapImpl::pushNextPage(const std::string&) { + throw std::logic_error("push_next_page: backend does not support pushed mode"); +} + +void MMapImpl::dropPages(size_t) { +} #endif -/// User-pushed sequential backend with an arbitrary-length queue of future pages. -/// We keep a deque of pages; when reads move forward, we drop fully-consumed -/// pages from the front to release memory. -struct PushedSequentialImpl final : std::enable_shared_from_this, FileReader::Impl { - // Deque of pages, front is earliest in file. - std::deque pages_; - // total bytes in dropped pages - size_t discarded_page_bytes_ = 0; +size_t PushedSequentialImpl::size() const { + size_t n = discarded_page_bytes_; + for (const auto& pg : pages_) { + n += pg.data.size(); + } + return n; +} - size_t size() const override { - size_t n = discarded_page_bytes_; - for (auto& pg : pages_) n += pg.data.size(); - return n; +void PushedSequentialImpl::dropPages(size_t pos) { + while (!pages_.empty()) { + if (pos - discarded_page_bytes_ >= pages_.front().data.size()) { + discarded_page_bytes_ += pages_.front().data.size(); + pages_.pop_front(); + } else { + break; + } + } +} + +char PushedSequentialImpl::get(size_t pos) const { + const size_t avail_end = size(); + if (pos >= avail_end) { + throw std::out_of_range("pushed backend: position not committed yet"); } - // Drop fully-consumed pages so pos is guaranteed to be within the first page - void dropPages(size_t pos) override { - while (!pages_.empty()) { - if (pos - discarded_page_bytes_ >= pages_.front().data.size()) { - discarded_page_bytes_ += pages_.front().data.size(); - pages_.pop_front(); - } else { - break; + pos -= discarded_page_bytes_; + + size_t page_start = 0; + for (const auto& pg : pages_) { + if (pos < page_start + pg.data.size()) { + const size_t off = pos - page_start; + return pg.data[off]; + } + page_start += pg.data.size(); + } + + throw std::out_of_range("pushed backend: internal inconsistency"); +} + +uint64_t PushedSequentialImpl::get_u64(size_t pos) const { + if (pos + sizeof(uint64_t) > size()) { + throw std::out_of_range("get_u64 out of range"); + } + + size_t relative_pos = pos - discarded_page_bytes_; + size_t page_start = 0; + for (const auto& pg : pages_) { + if (relative_pos < page_start + pg.data.size()) { + const size_t off = relative_pos - page_start; + if (off + sizeof(uint64_t) <= pg.data.size()) { + uint64_t value; + std::memcpy(&value, pg.data.data() + off, sizeof(value)); + return value; } + break; } + page_start += pg.data.size(); } - char get(size_t pos) const override { - /* - auto self = const_cast(this); - // We do not do this automatically because all variable width tokens: - // ENUM/STRING/BINARY/KEYWORD are stored as file offsets until a full - // entity instance is finalized. - if (this->shared_from_this().use_count() == 2) { - // only drop pages when there is only one active client. - // NB this->shared_from_this() increases count by 1 - self->drop_consumed_up_to(pos); - } - */ + return gather_value([&](size_t i) { return get(pos + i); }); +} - const size_t avail_end = size(); - if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet"); +uint32_t PushedSequentialImpl::get_u32(size_t pos) const { + if (pos + sizeof(uint32_t) > size()) { + throw std::out_of_range("get_u32 out of range"); + } - pos -= discarded_page_bytes_; - - size_t page_start = 0; - for (const auto& pg : pages_) { - if (pos < page_start + pg.data.size()) { - const size_t off = pos - page_start; - return pg.data[off]; - } else { - page_start += pg.data.size(); + size_t relative_pos = pos - discarded_page_bytes_; + size_t page_start = 0; + for (const auto& pg : pages_) { + if (relative_pos < page_start + pg.data.size()) { + const size_t off = relative_pos - page_start; + if (off + sizeof(uint32_t) <= pg.data.size()) { + uint32_t value; + std::memcpy(&value, pg.data.data() + off, sizeof(value)); + return value; } + break; } - - throw std::out_of_range("pushed backend: internal inconsistency"); + page_start += pg.data.size(); } - void pushNextPage(const std::string& data) override { - FileReader::Page p; p.data.assign(data.data(), data.data() + data.size()); - pages_.push_back(std::move(p)); - } -}; - -IfcParse::FileReader::FileReader(const std::string& fn) - : cursor_(0) -{ - impl_ = std::make_shared(fn); + return gather_value([&](size_t i) { return get(pos + i); }); } -IfcParse::FileReader::FileReader(const std::string& fn, const mmap_tag&) - : cursor_(0) -{ -#ifdef USE_MMAP - impl_ = std::make_shared(fn); -#else - (void)fn; - throw std::runtime_error("IfcParse::FileReader: mmap_tag specified but library not compiled with USE_MMAP"); -#endif -} - -IfcParse::FileReader::FileReader(const caller_fed_tag&) - : cursor_(0) -{ - impl_ = std::make_shared(); -} - -IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&) -{ - impl_ = std::make_shared(); - impl_->pushNextPage(content); -} - -IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity) - : cursor_(0) -{ - impl_ = std::make_shared(fn, page_size, page_capacity); -} - -FileReader FileReader::clone() const { - FileReader c(*this); - c.cursor_ = this->cursor_; - return c; -} - -void FileReader::seek(size_t pos) { - if (pos > impl_->size()) throw std::out_of_range("seek out of range"); - cursor_ = pos; -} - -size_t FileReader::tell() const { return cursor_; } - -size_t FileReader::size() const { return impl_->size(); } - -char FileReader::peek() const { - if (cursor_ >= impl_->size()) throw std::out_of_range("peek at EOF"); - return impl_->get(cursor_); -} - -void FileReader::increment(size_t n) { - if (cursor_ + n > impl_->size()) throw std::out_of_range("increment past EOF"); - cursor_ += n; -} - -void IfcParse::FileReader::pushNextPage(const std::string& data) -{ - impl_->pushNextPage(data); -} - -void IfcParse::FileReader::dropPages() -{ - impl_->dropPages(0); -} - -void IfcParse::FileReader::dropPages(size_t up_to_pos) -{ - impl_->dropPages(up_to_pos); -} - -bool IfcParse::FileReader::eof() const -{ - return cursor_ >= impl_->size(); -} - -char IfcParse::FileReader::read() -{ - auto c = peek(); - increment(1); - return c; -} - -char IfcParse::FileReader::get(size_t offset) const -{ - return impl_->get(offset); +void PushedSequentialImpl::pushNextPage(const std::string& data) { + FileReaderPage p; + p.data.assign(data.data(), data.data() + data.size()); + pages_.push_back(std::move(p)); } diff --git a/src/ifcparse/FileReader.h b/src/ifcparse/FileReader.h index 164c5826a6..0bbd9601a6 100644 --- a/src/ifcparse/FileReader.h +++ b/src/ifcparse/FileReader.h @@ -1,4 +1,4 @@ -/******************************************************************************** +/******************************************************************************** * * * This file is part of IfcOpenShell. * * * @@ -30,104 +30,244 @@ #include "ifc_parse_api.h" #include +#include +#include +#include +#include #include -#include -#include -#include #include +#include +#include +#include +#include #include +#ifdef USE_MMAP +#include +#endif + namespace IfcParse { -/// \brief Read-only file accessor that supports four backends: -/// - full-buffer in RAM -/// - paged (LRU-cached with capacity) -/// - memory-mapped via boost::iostreams -/// - user-pushed sequential pages (caller feeds pages intended for streaming reads in WASM) -class IFC_PARSE_API FileReader { +struct FileReaderPage { + std::vector data; +}; + +struct caller_fed_tag {}; + +template +inline constexpr bool file_reader_dependent_false_v = false; + +class IFC_PARSE_API FullBufferImpl; +class IFC_PARSE_API PagedFileImpl; +#ifdef USE_MMAP +class IFC_PARSE_API MMapImpl; +#endif +class IFC_PARSE_API PushedSequentialImpl; + +template +class FileReader { public: - struct Page { std::vector data; }; + using impl_type = Impl; + using Page = FileReaderPage; - /// \brief Tag to choose memory-mapped backend via boost::iostreams::mapped_file_source. - struct mmap_tag {}; - /// \brief Tag to choose user-pushed sequential pages. - struct caller_fed_tag {}; + FileReader() = default; - /// \brief Construct a FileReader. - /// \param fn File path. - /// \param maybe_mmaped_or_chunked - /// - std::nullopt: full-buffer mode (entire file loaded into memory) - /// - size_t: paged mode with this page size in bytes (e.g., 4096) - /// - mmap_tag: memory-mapped mode using boost::iostreams::mapped_file_source - /// \param page_cache_capacity Number of pages to keep in the LRU when in paged mode. - FileReader(const std::string& fn); - - FileReader(const std::string& fn, const mmap_tag&); - - FileReader(const caller_fed_tag&); - FileReader(const std::string& content, const caller_fed_tag&); - - FileReader(const std::string& fn, size_t page_size, size_t page_capacity); - - /// \brief Copy-construct a new reader sharing the underlying storage but with its own cursor. - FileReader clone() const; - - /// \brief Seek to an absolute byte position. - /// \throws std::out_of_range if pos > size(). - void seek(size_t pos); - - /// \brief Return the current cursor position. - size_t tell() const; - - /// \brief Total file size in bytes. - size_t size() const; - - /// \brief Peek the byte at the current cursor. - /// \throws std::out_of_range at EOF. - char peek() const; - - /// \brief Advance the cursor by n bytes (default 1). - /// \throws std::out_of_range if advancing crosses EOF. - void increment(size_t n = 1); - - /// \brief Push the next sequential page (pushed backend only). - /// \param data Contents of the page. - /// \throws std::logic_error if the current backend is not in pushed mode - void pushNextPage(const std::string& data); - - /// \brief Drops pages up to cursor position or provided offset. Does nothing when current backend is not in pushed mode - /// \param up_to_pos Pages with an end offset before up_to_pos are dropped from memory - void dropPages(); - void dropPages(size_t up_to_pos); - - /// \brief Returns true if the cursor is at or beyond the end of available data. - /// For the pushed backend, EOF means all pushed bytes have been consumed. - bool eof() const; - - /// \brief Equivalent of peek() followed by increment(1) - char read(); - - /// \brief Equivalent of peek() followed by increment(1) - char get(size_t offset) const; - - struct Impl { - virtual ~Impl() = default; - virtual size_t size() const = 0; - virtual char get(size_t pos) const = 0; - /// \brief Backend may support pushing pages; default throws. - virtual void pushNextPage(const std::string&) { - throw std::logic_error("push_next_page: backend does not support pushed mode"); + explicit FileReader(const std::string& fn) + : cursor_(0) { + if constexpr (std::is_same_v +#ifdef USE_MMAP + || std::is_same_v +#endif + ) { + impl_ = std::make_shared(fn); + } else { + static_assert(file_reader_dependent_false_v, "This FileReader constructor is not supported for the selected backend"); } - virtual void dropPages(size_t) { - // empty on purpose + } + + explicit FileReader(const caller_fed_tag&) + : cursor_(0) { + if constexpr (std::is_same_v) { + impl_ = std::make_shared(); + } else { + static_assert(file_reader_dependent_false_v, "This FileReader constructor is not supported for the selected backend"); } - }; + } + + FileReader(const std::string& content, const caller_fed_tag&) + : FileReader(caller_fed_tag{}) { + if constexpr (std::is_same_v) { + impl_->pushNextPage(content); + } else { + static_assert(file_reader_dependent_false_v, "This FileReader constructor is not supported for the selected backend"); + } + } + + FileReader(const std::string& fn, size_t page_size, size_t page_capacity) + : cursor_(0) { + if constexpr (std::is_same_v) { + impl_ = std::make_shared(fn, page_size, page_capacity); + } else { + static_assert(file_reader_dependent_false_v, "This FileReader constructor is not supported for the selected backend"); + } + } + + FileReader clone() const { + FileReader c(*this); + c.cursor_ = cursor_; + return c; + } + + void seek(size_t pos) { + if (pos > size()) { + throw std::out_of_range("seek out of range"); + } + cursor_ = pos; + } + + size_t tell() const { return cursor_; } + + size_t size() const { return impl_->size(); } + size_t remaining() const { return size() - cursor_; } + + char peek() const { + if (cursor_ >= size()) { + throw std::out_of_range("peek at EOF"); + } + return impl_->get(cursor_); + } + + uint64_t peek_u64() const { + if (remaining() < sizeof(uint64_t)) { + throw std::out_of_range("peek_u64 at EOF"); + } + return impl_->get_u64(cursor_); + } + + uint32_t peek_u32() const { + if (remaining() < sizeof(uint32_t)) { + throw std::out_of_range("peek_u32 at EOF"); + } + return impl_->get_u32(cursor_); + } + + void increment(size_t n = 1) { + if (cursor_ + n > size()) { + throw std::out_of_range("increment past EOF"); + } + cursor_ += n; + } + + void pushNextPage(const std::string& data) { + impl_->pushNextPage(data); + } + + void dropPages() { + impl_->dropPages(0); + } + + void dropPages(size_t up_to_pos) { + impl_->dropPages(up_to_pos); + } + + bool eof() const { + return cursor_ >= size(); + } + + char read() { + auto c = peek(); + increment(1); + return c; + } + + char get(size_t offset) const { + return impl_->get(offset); + } private: std::shared_ptr impl_; size_t cursor_ = 0; }; +class IFC_PARSE_API FullBufferImpl { +public: + explicit FullBufferImpl(const std::string& fn); + + size_t size() const; + char get(size_t pos) const; + uint32_t get_u32(size_t pos) const; + uint64_t get_u64(size_t pos) const; + void pushNextPage(const std::string& data); + void dropPages(size_t pos); + +private: + std::vector buf_; + size_t size_; +}; + +class IFC_PARSE_API PagedFileImpl { +public: + struct Entry { + FileReaderPage page; + std::list::iterator it; + }; + + PagedFileImpl(const std::string& fn, size_t page_size, size_t cap); + ~PagedFileImpl(); + + size_t size() const; + char get(size_t pos) const; + uint32_t get_u32(size_t pos) const; + uint64_t get_u64(size_t pos) const; + void pushNextPage(const std::string& data); + void dropPages(size_t pos); + +private: + const FileReaderPage& fetchPage_(size_t idx) const; + void touch_(std::unordered_map::iterator it) const; + void evict_() const; + + std::string fn_; + FILE* fp_ = nullptr; + size_t file_size_ = 0; + size_t page_size_ = 4096; + size_t capacity_ = 8; + mutable std::list lru_; + mutable std::unordered_map map_; +}; + +#ifdef USE_MMAP +class IFC_PARSE_API MMapImpl { +public: + explicit MMapImpl(const std::string& fn); + + size_t size() const; + char get(size_t pos) const; + uint32_t get_u32(size_t pos) const; + uint64_t get_u64(size_t pos) const; + void pushNextPage(const std::string& data); + void dropPages(size_t pos); + +private: + boost::iostreams::mapped_file_source map_; + size_t size_ = 0; +}; +#endif + +class IFC_PARSE_API PushedSequentialImpl { +public: + size_t size() const; + char get(size_t pos) const; + uint32_t get_u32(size_t pos) const; + uint64_t get_u64(size_t pos) const; + void pushNextPage(const std::string& data); + void dropPages(size_t pos); + +private: + std::deque pages_; + size_t discarded_page_bytes_ = 0; +}; + } // namespace IfcParse #endif diff --git a/src/ifcparse/IfcCharacterDecoder.cpp b/src/ifcparse/IfcCharacterDecoder.cpp index 1e9ce9f6b8..7a5168eef2 100644 --- a/src/ifcparse/IfcCharacterDecoder.cpp +++ b/src/ifcparse/IfcCharacterDecoder.cpp @@ -77,18 +77,81 @@ using namespace IfcParse; -IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::FileReader* stream) { - stream_ = stream; - codepage_ = 0; +namespace SWAR { + +constexpr uint32_t ONES32 = 0x01010101u; +constexpr uint32_t HIGHS32 = 0x80808080u; +constexpr uint64_t ONES64 = 0x0101010101010101ull; +constexpr uint64_t HIGHS64 = 0x8080808080808080ull; + +constexpr uint32_t splat32(unsigned char c) { + return ONES32 * c; } -IfcCharacterDecoder::~IfcCharacterDecoder() { +constexpr uint64_t splat64(unsigned char c) { + return ONES64 * c; +} + +inline uint32_t has_zero_byte(uint32_t x) { + return (x - ONES32) & ~x & HIGHS32; +} + +inline uint64_t has_zero_byte(uint64_t x) { + return (x - ONES64) & ~x & HIGHS64; +} + +inline uint32_t eq_mask(uint32_t x, uint32_t c) { + return has_zero_byte(x ^ c); +} + +inline uint64_t eq_mask(uint64_t x, uint64_t c) { + return has_zero_byte(x ^ c); +} + +namespace chars { +constexpr uint32_t apostrophe32 = splat32('\''); +constexpr uint32_t solidus32 = splat32('\\'); +constexpr uint64_t apostrophe64 = splat64('\''); +constexpr uint64_t solidus64 = splat64('\\'); +} // namespace chars + +inline uint32_t has_special_char(uint32_t x) { + return has_zero_byte(x) | + eq_mask(x, chars::apostrophe32) | + eq_mask(x, chars::solidus32) | + (x & HIGHS32); +} + +inline uint64_t has_special_char(uint64_t x) { + return has_zero_byte(x) | + eq_mask(x, chars::apostrophe64) | + eq_mask(x, chars::solidus64) | + (x & HIGHS64); +} + +inline void append_ascii(std::u32string& builder, const char* bytes, size_t count) { + builder.reserve(builder.size() + count); + for (size_t i = 0; i < count; ++i) { + builder.push_back(static_cast(bytes[i])); + } +} + +} // namespace SWAR + +template +IfcCharacterDecoder::IfcCharacterDecoder(Reader* stream) { + stream_ = stream; + codepage_ = 0; + builder_.reserve(1024); +} + +template +IfcCharacterDecoder::~IfcCharacterDecoder() { } namespace { - std::string read_string(IfcParse::FileReader& stream_, IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) { - std::u32string builder_; - + template + std::string read_string(std::u32string& builder_, Reader& stream_, typename IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) { unsigned int parse_state = 0; builder_.clear(); // builder_.push_back('\''); @@ -97,7 +160,31 @@ namespace { unsigned int hex = 0; unsigned int hex_count = 0; - while ((current_char = stream_.peek()) != 0) { + while (!stream_.eof()) { + if (parse_state == 0U) { + if (stream_.remaining() >= 8) { + uint64_t x = stream_.peek_u64(); + if (SWAR::has_special_char(x) == 0) { + SWAR::append_ascii(builder_, reinterpret_cast(&x), 8); + stream_.increment(8); + continue; + } + } + if (stream_.remaining() >= 4) { + uint32_t x = stream_.peek_u32(); + if (SWAR::has_special_char(x) == 0) { + SWAR::append_ascii(builder_, reinterpret_cast(&x), 4); + stream_.increment(4); + continue; + } + } + } + + current_char = stream_.peek(); + if (current_char == 0) { + break; + } + if (EXPECTS_CHARACTER(parse_state)) { builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80)); parse_state = 0; @@ -171,7 +258,7 @@ namespace { } // builder_.push_back('\''); - if (mode == IfcParse::IfcCharacterDecoder::UTF8) { + if (mode == IfcParse::IfcCharacterDecoder::UTF8) { if (builder_.empty()) { static std::string empty; return empty; @@ -183,7 +270,7 @@ namespace { } return IfcUtil::convert_utf8(builder_); } - if (mode == IfcParse::IfcCharacterDecoder::SUBSTITUTE) { + if (mode == IfcParse::IfcCharacterDecoder::SUBSTITUTE) { std::string result; result.reserve(builder_.size()); std::transform(builder_.begin(), builder_.end(), std::back_inserter(result), [&substitution_character](std::u32string::value_type character) { @@ -194,7 +281,7 @@ namespace { }); return result; } - if (mode == IfcParse::IfcCharacterDecoder::ESCAPE) { + if (mode == IfcParse::IfcCharacterDecoder::ESCAPE) { std::stringstream stream; stream << std::hex << std::setw(4) << std::setfill('0'); std::for_each(builder_.begin(), builder_.end(), [&stream](std::u32string::value_type character) { @@ -210,20 +297,26 @@ namespace { } } // namespace -IfcCharacterDecoder::operator std::string() { - return read_string(*stream_, mode, substitution_character); +template +IfcCharacterDecoder::operator std::string() { + return read_string(builder_, *stream_, mode, substitution_character); } -std::string IfcCharacterDecoder::get(size_t& ptr) { +template +std::string IfcCharacterDecoder::get(size_t& ptr) { auto local_stream = *stream_; local_stream.seek(ptr); - auto s = read_string(local_stream, mode, substitution_character); + auto s = read_string(builder_, local_stream, mode, substitution_character); ptr = local_stream.tell(); return s; } -IfcCharacterDecoder::ConversionMode IfcCharacterDecoder::mode = IfcCharacterDecoder::UTF8; -char IfcCharacterDecoder::substitution_character = '_'; +template class IfcCharacterDecoder>; +template class IfcCharacterDecoder>; +template class IfcCharacterDecoder>; +#ifdef USE_MMAP +template class IfcCharacterDecoder; +#endif IfcCharacterEncoder::IfcCharacterEncoder(const std::string& input) : str_(IfcUtil::convert_utf8(input)) {} diff --git a/src/ifcparse/IfcCharacterDecoder.h b/src/ifcparse/IfcCharacterDecoder.h index 2a3fb0410b..7cfe6d4b2c 100644 --- a/src/ifcparse/IfcCharacterDecoder.h +++ b/src/ifcparse/IfcCharacterDecoder.h @@ -39,10 +39,12 @@ IFC_PARSE_API std::u32string convert_utf8(const std::string& string); namespace IfcParse { +template class IFC_PARSE_API IfcCharacterDecoder { private: - IfcParse::FileReader* stream_; + Reader* stream_; int codepage_; + std::u32string builder_; public: enum ConversionMode { @@ -50,9 +52,10 @@ class IFC_PARSE_API IfcCharacterDecoder { UTF8, ESCAPE }; - static ConversionMode mode; - static char substitution_character; - IfcCharacterDecoder(IfcParse::FileReader* stream); + inline static ConversionMode mode = UTF8; + inline static char substitution_character = '_'; + + IfcCharacterDecoder(Reader* stream); ~IfcCharacterDecoder(); // Gets a decoded string representation at the token stream // read pointer and advances the underlying token stream. diff --git a/src/ifcparse/IfcFile.cpp b/src/ifcparse/IfcFile.cpp index 5bf27b4997..48b0b8f0dc 100644 --- a/src/ifcparse/IfcFile.cpp +++ b/src/ifcparse/IfcFile.cpp @@ -21,9 +21,9 @@ IfcParse::parse_context::~parse_context() { } IfcParse::parse_context& IfcParse::parse_context::push() { - auto* pc = new IfcParse::parse_context; - tokens_.push_back(pc); - return *pc; + auto child = pool_->make(); + tokens_.emplace_back(child); + return *child; } void IfcParse::parse_context::push(Token t) { @@ -220,7 +220,7 @@ namespace { if constexpr (std::is_same_v, IfcParse::Token>) { // @todo get aggregate of enumeration dispatch_token(instance_id, attribute_id, v, aggr && aggr->type_of_element()->as_named_type() ? aggr->type_of_element()->as_named_type()->declared_type() : nullptr, append_to_aggregate_storage); - } else if constexpr (std::is_same_v, IfcParse::parse_context*>) { + } else if constexpr (std::is_same_v, IfcParse::parse_context_handle>) { // nested list if constexpr (Depth < 3) { construct_(instance_id, attribute_id, *v, nullptr, append_to_aggregate_storage); @@ -245,7 +245,7 @@ std::shared_ptr IfcParse::parse_context::construct(IfcParse::IfcFi transient_named_type.reset(new IfcParse::named_type(const_cast(decl))); parameter_types = { &*transient_named_type }; } else if ((decl != nullptr) && (decl->as_entity() != nullptr)) { - auto entity_attrs = decl->as_entity()->all_attributes(); + const auto& entity_attrs = decl->as_entity()->all_attributes(); std::transform( entity_attrs.begin(), entity_attrs.end(), @@ -307,7 +307,7 @@ std::shared_ptr IfcParse::parse_context::construct(IfcParse::IfcFi storage.set(index, v); } }); - } else if constexpr (std::is_same_v, IfcParse::parse_context*>) { + } else if constexpr (std::is_same_v, IfcParse::parse_context_handle>) { const auto *pt = param_type; if (pt) { while (pt->as_named_type() && pt->as_named_type()->declared_type()->as_type_declaration()) { @@ -642,125 +642,6 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) { } } -void IfcParse::InstanceStreamer::bypassTypes(const std::set& type_names) { - for (auto& name : type_names) { - try { - types_to_bypass_.push_back(schema_->declaration_by_name(name)); - } catch (const IfcException&) { - continue; - } - } - } - - -std::optional>> IfcParse::InstanceStreamer::readInstance() { - std::optional>> return_value; - - if (header_ && yielded_header_instances_ < 3) { - if (yielded_header_instances_ == 0) { - return_value.emplace( - 0, - &header_->file_description().declaration(), - header_->file_description().data_weak().lock() - ); - } else if (yielded_header_instances_ == 1) { - return_value.emplace( - 0, - &header_->file_name().declaration(), - header_->file_name().data_weak().lock() - ); - } else if (yielded_header_instances_ == 2) { - return_value.emplace( - 0, - &header_->file_schema().declaration(), - header_->file_schema().data_weak().lock() - ); - } - yielded_header_instances_ += 1; - return return_value; - } - - unsigned current_id = 0; - while (good_ && !lexer_->stream->eof() && !current_id) { - if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER && - token_stream_[1].type == IfcParse::Token::Token_OPERATOR && - token_stream_[1].value_char == '=' && - token_stream_[2].type == IfcParse::Token::Token_KEYWORD) { - current_id = token_stream_[0].as_identifier(); - const IfcParse::declaration* entity_type; - try { - entity_type = schema_->declaration_by_name(token_stream_[2].as_string()); - } catch (const IfcException& ex) { - Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos)); - current_id = 0; - goto advance; - } - - if (entity_type->as_entity() == nullptr) { - Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos)); - goto advance; - } - - for (auto& ty : types_to_bypass_) { - if (entity_type->is(*ty)) { - bypassed_instances_.push_back(current_id); - // Why is this a conditional clause in the loop? - current_id = 0; - goto advance; - } - } - - parse_context ps; - lexer_->Next(); - try { - storage_.load(current_id, entity_type->as_entity(), ps, -1); - } catch (const IfcInvalidTokenException& e) { - good_ = file_open_status::INVALID_SYNTAX; - Logger::Error(e); - break; - } - - /// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code? - // Update the status after every 1000 instances parsed - if (((++progress_) % 1000) == 0) { - std::stringstream ss; - ss << "\r#" << current_id; - Logger::Status(ss.str(), false); - } - - auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count); - - return_value.emplace( - (size_t)current_id, - entity_type, - data - ); - } - advance: - Token next_token; - try { - next_token = lexer_->Next(); - } catch (const IfcException& e) { - Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated"); - } catch (...) { - Logger::Message(Logger::LOG_ERROR, "Parsing terminated"); - } - - if (!lexer_->stream->eof() && !next_token) { - good_ = file_open_status::INVALID_SYNTAX; - break; - } - - token_stream_.push_back(next_token); - } - - // Free pages in front of cursor when variable-width tokens are materialized into entity instance data objects - (stream_ ? stream_ : (lexer_)->stream)->dropPages(); - lexer_->resetPool(); - - return return_value; -} - express::Base IfcParse::impl::rocks_db_file_storage::create(const IfcParse::declaration* decl, int id) { return express::Base{}; /* diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index 33adfd9d02..991ab2bfdf 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -34,6 +34,7 @@ #include #include #include +#include #ifdef IFOPSH_WITH_ROCKSDB @@ -87,10 +88,13 @@ enum filetype { IFC_PARSE_API filetype guess_file_type(const std::string& fn); +template > class IFC_PARSE_API InstanceStreamer { private: - FileReader* stream_; - IfcSpfLexer* lexer_; + std::unique_ptr owned_stream_; + Reader* stream_; + std::unique_ptr> lexer_; + std::unique_ptr owned_header_; IfcSpfHeader* header_; IfcParse::IfcFile* owner_; boost::circular_buffer token_stream_; @@ -100,14 +104,18 @@ private: int progress_; IfcParse::unresolved_references references_to_resolve_; int yielded_header_instances_ = 0; + bool yield_header_instances_ = true; std::vector types_to_bypass_; std::vector bypassed_instances_; + void initialize_header(); + IfcSpfHeader& ensure_header(); + public: bool coerce_attribute_count = true; operator bool() const { - return good_ && !lexer_->stream->eof(); + return good_ && lexer_ && !lexer_->stream->eof(); } IfcParse::file_open_status status() const { @@ -151,17 +159,17 @@ private: InstanceStreamer(void* data, int length, IfcParse::IfcFile* f = nullptr); - InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer, IfcParse::IfcFile* f = nullptr); + InstanceStreamer(Reader* stream, IfcParse::IfcFile* f = nullptr); void bypassTypes(const std::set& type_names); - ~InstanceStreamer() { - delete stream_; - if (stream_) { - delete lexer_; - } - delete header_; - } + void yieldHeaderInstances(bool value) { yield_header_instances_ = value; } + + const IfcParse::schema_definition* schema() const { return schema_; } + + const IfcSpfHeader* header() const { return header_; } + + ~InstanceStreamer() = default; std::optional>> readInstance(); }; @@ -249,12 +257,6 @@ public: /// IfcFile(void* data, int length); - /// - /// Constructs an IfcFile object from a given IFC SPF stream. - /// - /// A pointer to an IfcParse::FileReader object representing the input IFC SPF data stream. - IfcFile(IfcParse::FileReader* stream); - /// /// Constructs an IfcFile object with the specified schema, file type, and file path. /// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index 594127d7e9..eb86785016 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -102,16 +102,19 @@ void init_locale() { #endif -IfcSpfLexer::IfcSpfLexer(IfcParse::FileReader* stream_) { +template +IfcSpfLexer::IfcSpfLexer(Reader* stream_) { stream = stream_; - decoder_ = new IfcCharacterDecoder(stream_); + decoder_ = new IfcCharacterDecoder(stream_); } -IfcSpfLexer::~IfcSpfLexer() { +template +IfcSpfLexer::~IfcSpfLexer() { delete decoder_; } -size_t IfcSpfLexer::skipWhitespace() const { +template +size_t IfcSpfLexer::skipWhitespace() const { size_t index = 0; while (!stream->eof()) { char character = stream->peek(); @@ -125,7 +128,8 @@ size_t IfcSpfLexer::skipWhitespace() const { return index; } -size_t IfcSpfLexer::skipComment() const { +template +size_t IfcSpfLexer::skipComment() const { if (stream->eof()) { return 0; } @@ -153,7 +157,8 @@ size_t IfcSpfLexer::skipComment() const { return index; } -std::string& IfcSpfLexer::getTempString() const { +template +std::string& IfcSpfLexer::getTempString() const { const size_t idx = pool_index++; const size_t slice = idx >> 4; const size_t offset = idx & 0xF; @@ -161,6 +166,9 @@ std::string& IfcSpfLexer::getTempString() const { while (stringpool_.size() <= slice) { stringpool_.push_back(std::make_unique>()); } + + // std::wcout << "Num contexts: " << idx << std::endl; + return (*stringpool_[slice])[offset]; } @@ -192,17 +200,86 @@ bool parse_float_(const char* pStart, double& val) { } // namespace +namespace SWAR { +constexpr uint32_t ONES32 = 0x01010101u; +constexpr uint32_t HIGHS32 = 0x80808080u; +constexpr uint64_t ONES = 0x0101010101010101ull; +constexpr uint64_t HIGHS = 0x8080808080808080ull; + +constexpr uint64_t splat(unsigned char c) { + return ONES * c; +} + +inline uint32_t has_zero_byte(uint32_t x) { + return (x - ONES32) & ~x & HIGHS32; +} + +inline uint64_t has_zero_byte(uint64_t x) { + return (x - ONES) & ~x & HIGHS; +} + +inline uint32_t eq_mask(uint32_t x, uint32_t c) { + return has_zero_byte(x ^ c); +} + +inline uint64_t eq_mask(uint64_t x, uint64_t c) { + return has_zero_byte(x ^ c); +} + +namespace chars { +constexpr uint64_t lpar = splat('('); +constexpr uint64_t rpar = splat(')'); +constexpr uint64_t eq = splat('='); +constexpr uint64_t comma = splat(','); +constexpr uint64_t semi = splat(';'); +constexpr uint64_t slash = splat('/'); + +constexpr uint64_t space = splat(' '); +constexpr uint64_t cr = splat('\r'); +constexpr uint64_t lf = splat('\n'); +constexpr uint64_t tab = splat('\t'); + +constexpr uint64_t quote = splat('"'); +constexpr uint64_t dot = splat('.'); +} // namespace chars + +inline uint64_t has_special_char(uint64_t x) { + return eq_mask(x, chars::lpar) | + eq_mask(x, chars::rpar) | + eq_mask(x, chars::eq) | + eq_mask(x, chars::comma) | + eq_mask(x, chars::semi) | + eq_mask(x, chars::slash) | + eq_mask(x, chars::space) | + eq_mask(x, chars::cr) | + eq_mask(x, chars::lf) | + eq_mask(x, chars::tab) | + eq_mask(x, chars::quote) | + eq_mask(x, chars::dot); +} + +inline uint32_t has_special_char(uint32_t x) { + return eq_mask(x, static_cast(chars::lpar)) | + eq_mask(x, static_cast(chars::rpar)) | + eq_mask(x, static_cast(chars::eq)) | + eq_mask(x, static_cast(chars::comma)) | + eq_mask(x, static_cast(chars::semi)) | + eq_mask(x, static_cast(chars::slash)) | + eq_mask(x, static_cast(chars::space)) | + eq_mask(x, static_cast(chars::cr)) | + eq_mask(x, static_cast(chars::lf)) | + eq_mask(x, static_cast(chars::tab)) | + eq_mask(x, static_cast(chars::quote)) | + eq_mask(x, static_cast(chars::dot)); +} + +} + // // Returns the offset of the current Token and moves cursor to next // -Token IfcSpfLexer::Next() { - - if (stream->eof()) { - return Token{}; - } - - while ((skipWhitespace() != 0U) || (skipComment() != 0U)) { - } +template +Token IfcSpfLexer::Next() { if (stream->eof()) { return Token{}; @@ -211,6 +288,16 @@ Token IfcSpfLexer::Next() { auto pos = stream->tell(); char character = stream->read(); + if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') { + while ((skipWhitespace() != 0U) || (skipComment() != 0U)) { + } + if (stream->eof()) { + return Token{}; + } + pos = stream->tell(); + character = stream->read(); + } + // If the cursor is at [()=,;$*] we know token consists of single char if (character == '(' || character == ')' || @@ -246,8 +333,25 @@ Token IfcSpfLexer::Next() { } while (!stream->eof()) { + if (stream->remaining() >= 8) { + uint64_t x = stream->peek_u64(); + if (SWAR::has_special_char(x) == 0) { + str.append(reinterpret_cast(&x), 8); + stream->increment(8); + continue; + } + } + if (stream->remaining() >= 4) { + uint32_t x = stream->peek_u32(); + if (SWAR::has_special_char(x) == 0) { + str.append(reinterpret_cast(&x), 4); + stream->increment(4); + continue; + } + } + // Read character and increment pointer if not starting a new token - character = stream->peek(); + char character = stream->peek(); if (character == '(' || character == ')' || character == '=' || @@ -257,7 +361,8 @@ Token IfcSpfLexer::Next() { break; } if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) { - if ((ttype == Token::Token_BINARY && character == '"') || (ttype == Token::Token_ENUMERATION && character == '.')) { + if ((ttype == Token::Token_BINARY && character == '"') || + (ttype == Token::Token_ENUMERATION && character == '.')) { // Skip } else { str.push_back(character); @@ -300,6 +405,13 @@ Token IfcSpfLexer::Next() { } } +template class IfcSpfLexer>; +template class IfcSpfLexer>; +template class IfcSpfLexer>; +#ifdef USE_MMAP +template class IfcSpfLexer>; +#endif + bool Token::is_operator() { return type == Token_OPERATOR; } @@ -458,7 +570,8 @@ std::string Token::to_string() { // Reads the arguments from a list of token // Aditionally, registers the ids (i.e. #[\d]+) in the inverse map // -void IfcParse::impl::in_memory_file_storage::load(std::optional entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { +template +void IfcParse::impl::in_memory_file_storage::load(IfcParse::IfcSpfLexer* tokens, std::optional entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) { Token next = tokens->Next(); size_t attribute_index_within_data = 0; @@ -473,7 +586,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional entity_i break; } else if (next.is_operator('(')) { return_value++; - load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); + load(tokens, entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index); } else { return_value++; if (next.is_identifier() && entity && entity_instance_name) { @@ -483,7 +596,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional entity_i if (next.is_keyword()) { try { const auto* decl = (schema ? schema : file->schema())->declaration_by_name(next.as_string()); - parse_context ps; + parse_context ps(&context_pool_); tokens->Next(); // The only case we know where a defined type contains entity // instance references is IfcPropertySetDefinitionSet. For @@ -491,7 +604,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional entity_i // register inverses to the host entity (and not the defined // type) and to be able to actually register the references in // the 2nd pass. - load(entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index); + load(tokens, entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index); express::Base simple_type_instance(read_simple_type_instances.emplace_back( ps.construct(file, entity_instance_name, *references_to_resolve, decl, std::nullopt, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index)) ); @@ -510,7 +623,8 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional entity_i } } -void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const { +template +void IfcParse::impl::in_memory_file_storage::try_read_semicolon(IfcParse::IfcSpfLexer* tokens) const { auto old_offset = tokens->stream->tell(); Token semilocon = tokens->Next(); if (!semilocon.is_operator(';')) { @@ -1094,16 +1208,16 @@ IfcFile::IfcFile(const std::string& fn, bool mmap) { } bool IfcParse::IfcFile::initialize(const std::string& fn, bool mmap) { - std::unique_ptr s; if (mmap) { - s = std::make_unique(fn, FileReader::mmap_tag{}); + MMapFileReader s(fn); + storage_.emplace<1>(this); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); } else { - s = std::make_unique(fn); + FullBufferFileReader s(fn); + storage_.emplace<1>(this); + std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); } - storage_.emplace<1>(this); - std::get(storage_).read_from_stream(&*s, schema_, max_id_, types_to_bypass_loading_); - if ((good_ = std::get(storage_).good_)) { // @todo unify these names, it's already confusing enough as it stands byid_ = decltype(byid_)(&std::get(storage_).byid_read_); @@ -1124,7 +1238,7 @@ bool IfcParse::IfcFile::initialize(const std::string& path, filetype ty, bool re ty = guess_file_type(path); } if (ty == FT_IFCSPF) { - FileReader s(path); + FileReader s(path); storage_.emplace<1>(this); std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); @@ -1181,7 +1295,7 @@ IfcFile::IfcFile(std::istream& stream, int length) , max_id_(0) , header_(new IfcParse::IfcSpfHeader(this)) { - FileReader s(FileReader::caller_fed_tag{}); + FileReader s(caller_fed_tag{}); std::string string_data; string_data.resize(length); @@ -1203,7 +1317,7 @@ IfcFile::IfcFile(void* data, int length) , max_id_(0), header_(new IfcParse::IfcSpfHeader(this)) { - FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{}); + FileReader s(std::string((char*)data, length), caller_fed_tag{}); storage_.emplace<1>(this); std::get(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_); @@ -1215,20 +1329,6 @@ IfcFile::IfcFile(void* data, int length) byguid_ = decltype(byguid_)(&std::get(storage_).byguid_); } -IfcFile::IfcFile(IfcParse::FileReader* s) - : schema_(nullptr) - , max_id_(0) -{ - storage_.emplace<1>(this); - std::get(storage_).read_from_stream(s, schema_, max_id_, types_to_bypass_loading_); - good_ = std::get(storage_).good_; - ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr; - - byid_ = decltype(byid_)(&std::get(storage_).byid_read_); - byref_excl_ = decltype(byref_excl_)(&std::get(storage_).byref_excl_); - byguid_ = decltype(byguid_)(&std::get(storage_).byguid_); -} - IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const std::string& path) : schema_(schema) , ifcroot_type_(schema_->declaration_by_name("IfcRoot")) @@ -1260,9 +1360,127 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s setDefaultHeaderValues(); } -bool IfcParse::InstanceStreamer::hasSemicolon() const { +namespace { + +template +void read_terminal(IfcSpfLexer& lexer, const std::string& term, bool trailing_semicolon) { + if (lexer.Next().as_string() != term) { + throw IfcException(std::string("Expected " + term)); + } + if (trailing_semicolon) { + if (!lexer.Next().is_operator(';')) { + throw IfcException("Expected ;"); + } + } +} + +template +std::shared_ptr read_header_entity( + IfcParse::IfcFile* file, + IfcParse::impl::in_memory_file_storage& storage, + IfcSpfLexer& lexer, + IfcParse::unresolved_references& references_to_resolve, + const IfcParse::entity& decl) { + parse_context pc(&storage.context_pool_); + lexer.Next(); + storage.load(&lexer, std::nullopt, nullptr, pc, -1); + auto result = pc.construct(file, std::nullopt, references_to_resolve, &decl, decl.attribute_count(), -1); + storage.context_pool_.reset(); + return result; +} + +template +void parse_header( + IfcParse::IfcSpfHeader& header, + IfcParse::impl::in_memory_file_storage& storage, + IfcSpfLexer& lexer, + IfcParse::unresolved_references& references_to_resolve) { + static const char* const ISO_10303_21 = "ISO-10303-21"; + static const char* const HEADER = "HEADER"; + + read_terminal(lexer, ISO_10303_21, true); + read_terminal(lexer, HEADER, true); + + read_terminal(lexer, Header_section_schema::file_description::Class().name_uc(), false); + header.set_file_description(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_description::Class())); + if (!lexer.Next().is_operator(';')) { + throw IfcException("Expected ;"); + } + + read_terminal(lexer, Header_section_schema::file_name::Class().name_uc(), false); + header.set_file_name(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_name::Class())); + if (!lexer.Next().is_operator(';')) { + throw IfcException("Expected ;"); + } + + read_terminal(lexer, Header_section_schema::file_schema::Class().name_uc(), false); + header.set_file_schema(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_schema::Class())); + if (!lexer.Next().is_operator(';')) { + throw IfcException("Expected ;"); + } +} + +template +bool try_parse_header( + IfcParse::IfcSpfHeader& header, + IfcParse::impl::in_memory_file_storage& storage, + IfcSpfLexer& lexer, + IfcParse::unresolved_references& references_to_resolve) { + try { + parse_header(header, storage, lexer, references_to_resolve); + return true; + } catch (const std::exception& e) { + storage.context_pool_.reset(); + Logger::Error(e); + return false; + } +} + +} // namespace + +template +IfcSpfHeader& IfcParse::InstanceStreamer::ensure_header() { + if (header_) { + return *header_; + } + + if (owner_ != nullptr) { + header_ = &owner_->header(); + header_->file(owner_); + } else { + owned_header_ = std::make_unique(owner_); + header_ = owned_header_.get(); + } + + return *header_; +} + +template +void IfcParse::InstanceStreamer::initialize_header() { + storage_.file = owner_; + storage_.schema = schema_; + storage_.references_to_resolve = &references_to_resolve_; + + if (!lexer_ || !stream_ || !stream_->size() || stream_->eof()) { + return; + } + + auto& header = ensure_header(); + if (try_parse_header(header, storage_, *lexer_, references_to_resolve_) && header.file_schema().schema_identifiers().size() == 1) { + try { + schema_ = IfcParse::schema_by_name(header.file_schema().schema_identifiers().front()); + good_ = file_open_status::SUCCESS; + } catch (const IfcParse::IfcException&) { + } + } + + storage_.schema = schema_; +} + +template +bool IfcParse::InstanceStreamer::hasSemicolon() const { auto local_stream = stream_->clone(); - auto local_lexer = IfcSpfLexer(&local_stream); + auto local_lexer = IfcSpfLexer(&local_stream); Token t; try { t = local_lexer.Next(); @@ -1272,20 +1490,20 @@ bool IfcParse::InstanceStreamer::hasSemicolon() const { while (t.type != Token::Token_NONE) { if (t.is_operator(';')) { return true; - } + } try { t = local_lexer.Next(); } catch (const std::out_of_range&) { - // This most likely happens when a page boundary is contained within a string break; } } - return false; + return false; } -size_t IfcParse::InstanceStreamer::semicolonCount() const { +template +size_t IfcParse::InstanceStreamer::semicolonCount() const { auto local_stream = stream_->clone(); - auto local_lexer = IfcSpfLexer(&local_stream); + auto local_lexer = IfcSpfLexer(&local_stream); Token t; size_t count = 0; try { @@ -1300,145 +1518,248 @@ size_t IfcParse::InstanceStreamer::semicolonCount() const { try { t = local_lexer.Next(); } catch (const std::out_of_range&) { - // This most likely happens when a page boundary is contained within a string break; } } return count; } -void IfcParse::InstanceStreamer::pushPage(const std::string& page) -{ +template +void IfcParse::InstanceStreamer::pushPage(const std::string& page) { stream_->pushNextPage(page); if (good_ == file_open_status::NO_HEADER) { - header_ = new IfcParse::IfcSpfHeader(lexer_); - if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) { - try { - schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front()); - good_ = file_open_status::SUCCESS; - } catch (const IfcParse::IfcException&) { - } - } - storage_.file = nullptr; - storage_.schema = schema_; - storage_.tokens = lexer_; - storage_.references_to_resolve = &references_to_resolve_; + initialize_header(); } } -IfcParse::InstanceStreamer::InstanceStreamer(IfcParse::IfcFile* f) - : stream_(new FileReader(FileReader::caller_fed_tag{})) - , lexer_(new IfcSpfLexer(stream_)) - , token_stream_(3, Token{}) - , schema_(nullptr) - , progress_(0) - , owner_(f) -{ - init_locale(); - good_ = file_open_status::NO_HEADER; - storage_.file = f; -} - -IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap, IfcParse::IfcFile* f) - : stream_(mmap ? new FileReader(fn, FileReader::mmap_tag{}) : new FileReader(fn)) - , lexer_(new IfcSpfLexer(stream_)) - , token_stream_(3, Token{}) - , schema_(nullptr) - , progress_(0), owner_(f) -{ - init_locale(); - - good_ = file_open_status::NO_HEADER; - if (stream_->size() && !stream_->eof()) { - header_ = new IfcParse::IfcSpfHeader(lexer_); - if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) { - try { - schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front()); - good_ = file_open_status::SUCCESS; - } catch (const IfcParse::IfcException&) { - } - } - storage_.file = f; - storage_.schema = schema_; - storage_.tokens = lexer_; - storage_.references_to_resolve = &references_to_resolve_; - } -} - -IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length, IfcParse::IfcFile* f) - : stream_(new FileReader(std::string((char*) data, length), FileReader::caller_fed_tag{})) - , lexer_(new IfcSpfLexer(stream_)) - , token_stream_(3, Token{}) - , schema_(nullptr) - , progress_(0) - , owner_(f) -{ - init_locale(); - - good_ = file_open_status::NO_HEADER; - if (stream_->size() && !stream_->eof()) { - header_ = new IfcParse::IfcSpfHeader(lexer_); - if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) { - try { - schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front()); - good_ = file_open_status::SUCCESS; - } catch (const IfcParse::IfcException&) { - } - } - storage_.file = f; - storage_.schema = schema_; - storage_.tokens = lexer_; - storage_.references_to_resolve = &references_to_resolve_; - } -} - -IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer, IfcParse::IfcFile* f) +template +IfcParse::InstanceStreamer::InstanceStreamer(IfcParse::IfcFile* f) : stream_(nullptr) - , lexer_(lexer) , header_(nullptr) + , owner_(f) , token_stream_(3, Token{}) - , schema_(schema) + , schema_(nullptr) , progress_(0) - , owner_(f) { init_locale(); + if constexpr (std::is_same_v>) { + owned_stream_ = std::make_unique(caller_fed_tag{}); + } else { + static_assert(file_reader_dependent_false_v, "Default InstanceStreamer requires a pushed sequential reader"); + } + + stream_ = owned_stream_.get(); + lexer_ = std::make_unique>(stream_); + good_ = file_open_status::NO_HEADER; storage_.file = f; - storage_.schema = schema_; - storage_.tokens = lexer_; storage_.references_to_resolve = &references_to_resolve_; } -void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass) { - // Initialize a "C" locale for locale-independent - // number parsing. See comment above on line 41. +template +IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap, IfcParse::IfcFile* f) + : stream_(nullptr) + , header_(nullptr) + , owner_(f) + , token_stream_(3, Token{}) + , schema_(nullptr) + , progress_(0) { init_locale(); - tokens = nullptr; + if constexpr (std::is_same_v>) { + (void)mmap; + owned_stream_ = std::make_unique(fn); +#ifdef USE_MMAP + } else if constexpr (std::is_same_v) { + (void)mmap; + owned_stream_ = std::make_unique(fn); +#endif + } else { + static_assert(file_reader_dependent_false_v, "Path-based InstanceStreamer requires a file-backed reader"); + } + + stream_ = owned_stream_.get(); + lexer_ = std::make_unique>(stream_); + good_ = file_open_status::NO_HEADER; + initialize_header(); +} + +template +IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length, IfcParse::IfcFile* f) + : stream_(nullptr) + , header_(nullptr) + , owner_(f) + , token_stream_(3, Token{}) + , schema_(nullptr) + , progress_(0) +{ + init_locale(); + + if constexpr (std::is_same_v>) { + owned_stream_ = std::make_unique(std::string((char*)data, length), caller_fed_tag{}); + } else { + static_assert(file_reader_dependent_false_v, "Buffer-based InstanceStreamer requires a pushed sequential reader"); + } + + stream_ = owned_stream_.get(); + lexer_ = std::make_unique>(stream_); + good_ = file_open_status::NO_HEADER; + initialize_header(); +} + +template +IfcParse::InstanceStreamer::InstanceStreamer(Reader* stream, IfcParse::IfcFile* f) + : stream_(stream) + , header_(nullptr) + , owner_(f) + , token_stream_(3, Token{}) + , schema_(nullptr) + , progress_(0) { + init_locale(); + + lexer_ = std::make_unique>(stream_); + good_ = file_open_status::NO_HEADER; + initialize_header(); +} + +template +void IfcParse::InstanceStreamer::bypassTypes(const std::set& type_names) { + for (auto& name : type_names) { + try { + types_to_bypass_.push_back(schema_->declaration_by_name(name)); + } catch (const IfcException&) { + continue; + } + } +} + +template +std::optional>> IfcParse::InstanceStreamer::readInstance() { + std::optional>> return_value; + + if (yield_header_instances_ && header_ && yielded_header_instances_ < 3) { + if (yielded_header_instances_ == 0) { + return_value.emplace( + 0, + &header_->file_description().declaration(), + header_->file_description().data_weak().lock()); + } else if (yielded_header_instances_ == 1) { + return_value.emplace( + 0, + &header_->file_name().declaration(), + header_->file_name().data_weak().lock()); + } else if (yielded_header_instances_ == 2) { + return_value.emplace( + 0, + &header_->file_schema().declaration(), + header_->file_schema().data_weak().lock()); + } + yielded_header_instances_ += 1; + return return_value; + } + + unsigned current_id = 0; + while (good_ && !lexer_->stream->eof() && !current_id) { + if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER && + token_stream_[1].type == IfcParse::Token::Token_OPERATOR && + token_stream_[1].value_char == '=' && + token_stream_[2].type == IfcParse::Token::Token_KEYWORD) { + current_id = token_stream_[0].as_identifier(); + const IfcParse::declaration* entity_type; + try { + entity_type = schema_->declaration_by_name(token_stream_[2].as_string()); + } catch (const IfcException& ex) { + Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos)); + current_id = 0; + goto advance; + } + + if (entity_type->as_entity() == nullptr) { + Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos)); + goto advance; + } + + for (auto& ty : types_to_bypass_) { + if (entity_type->is(*ty)) { + bypassed_instances_.push_back(current_id); + current_id = 0; + goto advance; + } + } + + parse_context ps(&storage_.context_pool_); + lexer_->Next(); + try { + storage_.load(lexer_.get(), current_id, entity_type->as_entity(), ps, -1); + } catch (const IfcInvalidTokenException& e) { + good_ = file_open_status::INVALID_SYNTAX; + Logger::Error(e); + break; + } + + if (((++progress_) % 1000) == 0) { + std::stringstream ss; + ss << "\r#" << current_id; + Logger::Status(ss.str(), false); + } + + auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count); + storage_.context_pool_.reset(); + + return_value.emplace( + (size_t)current_id, + entity_type, + data); + } + advance: + Token next_token; + try { + next_token = lexer_->Next(); + } catch (const IfcException& e) { + Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated"); + } catch (...) { + Logger::Message(Logger::LOG_ERROR, "Parsing terminated"); + } + + if (!lexer_->stream->eof() && !next_token) { + good_ = file_open_status::INVALID_SYNTAX; + break; + } + + token_stream_.push_back(next_token); + } + + stream_->dropPages(); + lexer_->resetPool(); + + return return_value; +} + +template +void IfcParse::impl::in_memory_file_storage::read_from_stream(Reader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass) { + init_locale(); + + schema = nullptr; if (!s->size() || s->eof()) { - // @todo set good on parent file good_ = file_open_status::READ_ERROR; return; } - tokens = new IfcSpfLexer(s); - std::vector schemas; - file->header().file(file); + InstanceStreamer streamer(s, file); + streamer.yieldHeaderInstances(false); - if (file->header().tryRead()) { + if (const auto* header = streamer.header()) { try { - schemas = file->header().file_schema().schema_identifiers(); + schemas = header->file_schema().schema_identifiers(); } catch (...) { - // Purposely empty catch block } - } else { - good_ = file_open_status::NO_HEADER; } - if (schemas.size() == 1) { + schema = streamer.schema(); + if (schema == nullptr && schemas.size() == 1) { try { schema = IfcParse::schema_by_name(schemas.front()); } catch (const IfcParse::IfcException& e) { @@ -1448,28 +1769,28 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead } if (schema == nullptr) { + if (schemas.empty()) { + good_ = streamer.status(); + } else { + good_ = file_open_status::UNSUPPORTED_SCHEMA; + } Logger::Message(Logger::LOG_ERROR, "No support for file schema encountered (" + boost::algorithm::join(schemas, ", ") + ")"); return; } auto ifcroot_type_ = schema->declaration_by_name("IfcRoot"); - - InstanceStreamer streamer(schema, tokens, file); streamer.bypassTypes(typed_to_bypass); Logger::Status("Scanning file..."); while (streamer) { - auto inst = streamer.readInstance(); if (!inst) { - // No more instances to read break; - } + } auto current_id = std::get<0>(*inst); - express::Base instance(std::get<2>(*inst)); if (instance.declaration().is(*ifcroot_type_)) { @@ -1487,10 +1808,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead } const IfcParse::declaration* ty = &instance.declaration(); - - { - bytype_excl_[ty].push_back(instance); - } + bytype_excl_[ty].push_back(instance); if (byid_.find(current_id) != byid_.end()) { std::stringstream ss; @@ -1499,27 +1817,15 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead } byid_.insert({(uint32_t)current_id, std::get<2>(*inst)}); - max_id = (std::max)(max_id, (unsigned int) current_id); + max_id = (std::max)(max_id, (unsigned int)current_id); } - good_ = streamer.status(); - byref_excl_ = streamer.inverses(); - - // Move the storage of simple type instances so that they are retained during the lifetime of the file + good_ = streamer.status(); + byref_excl_ = streamer.inverses(); read_simple_type_instances = streamer.stealInstances(); - // Set file ownership on simple type instances, so that when adding them to other files, proper copies are created - /* - // @todo double check whether file ownership is property set earlier on - for (auto& inst : read_simple_type_instances) { - inst->file_ = file; - } - */ - Logger::Status("\rDone scanning file "); - delete tokens; - if (good_ != file_open_status::SUCCESS) { return; } @@ -1639,6 +1945,12 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead Logger::Status("Done resolving references"); } +template void IfcParse::impl::in_memory_file_storage::read_from_stream(FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); +template void IfcParse::impl::in_memory_file_storage::read_from_stream(FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); +#ifdef USE_MMAP +template void IfcParse::impl::in_memory_file_storage::read_from_stream(MMapFileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); +#endif + void IfcFile::recalculate_id_counter() { /* // @todo diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index 1fed5de76c..1f06390466 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -48,9 +48,10 @@ extern const char *IFCOPENSHELL_VERSION; namespace IfcParse { /// A stream of tokens to be read from a FileReader. +template class IFC_PARSE_API IfcSpfLexer { private: - IfcCharacterDecoder* decoder_; + IfcCharacterDecoder* decoder_; size_t skipWhitespace() const; size_t skipComment() const; @@ -68,9 +69,9 @@ class IFC_PARSE_API IfcSpfLexer { } } - FileReader* stream; + Reader* stream; // IfcFile* file; - IfcSpfLexer(FileReader* stream); + IfcSpfLexer(Reader* stream); Token Next(); ~IfcSpfLexer(); // void TokenString(size_t offset, std::string& result); diff --git a/src/ifcparse/IfcSchema.h b/src/ifcparse/IfcSchema.h index 46d1469a08..9f0f6dd0a4 100644 --- a/src/ifcparse/IfcSchema.h +++ b/src/ifcparse/IfcSchema.h @@ -28,6 +28,7 @@ #include #include #include +#include // Forward declarations class InstanceData; @@ -297,6 +298,7 @@ class IFC_PARSE_API entity : public declaration { std::vector subtypes_; std::vector attributes_; + mutable std::optional> all_attributes_; std::vector derived_; std::vector inverse_attributes_; @@ -354,15 +356,17 @@ class IFC_PARSE_API entity : public declaration { const std::vector& attributes() const { return attributes_; } const std::vector& derived() const { return derived_; } - const std::vector all_attributes() const { - std::vector attrs; - attrs.reserve(derived_.size()); - if (supertype_ != nullptr) { - const std::vector supertype_attrs = supertype_->all_attributes(); - std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs)); + const std::vector& all_attributes() const { + if (!all_attributes_) { + auto& attrs = all_attributes_.emplace(); + attrs.reserve(derived_.size()); + if (supertype_ != nullptr) { + const std::vector supertype_attrs = supertype_->all_attributes(); + std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs)); + } + std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs)); } - std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs)); - return attrs; + return *all_attributes_; } const std::vector all_inverse_attributes() const { diff --git a/src/ifcparse/IfcSpfHeader.cpp b/src/ifcparse/IfcSpfHeader.cpp index cc04ffcabb..7a2e1360f3 100644 --- a/src/ifcparse/IfcSpfHeader.cpp +++ b/src/ifcparse/IfcSpfHeader.cpp @@ -1,26 +1,6 @@ -/******************************************************************************** - * * - * This file is part of IfcOpenShell. * - * * - * IfcOpenShell is free software: you can redistribute it and/or modify * - * it under the terms of the Lesser GNU General Public License as published by * - * the Free Software Foundation, either version 3.0 of the License, or * - * (at your option) any later version. * - * * - * IfcOpenShell is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * Lesser GNU General Public License for more details. * - * * - * You should have received a copy of the Lesser GNU General Public License * - * along with this program. If not, see . * - * * - ********************************************************************************/ - #include "IfcSpfHeader.h" #include "IfcFile.h" -#include "IfcLogger.h" static const char* const ISO_10303_21 = "ISO-10303-21"; static const char* const HEADER = "HEADER"; @@ -30,116 +10,31 @@ static const char* const DATA = "DATA"; using namespace IfcParse; namespace { - std::shared_ptr read_from_spf_file(IfcParse::IfcFile* file, IfcParse::impl::in_memory_file_storage* storage, const IfcParse::entity* decl) { - if (storage != nullptr) { - parse_context pc; - storage->tokens->Next(); - storage->load(-1, nullptr, pc, -1); - return pc.construct(file, std::nullopt, *storage->references_to_resolve, decl, decl->as_entity()->attribute_count(), -1); - } else { - // std::unreachable(); - return nullptr; - } - } -} // namespace -void IfcSpfHeader::readSemicolon() { - if (storage_ != nullptr) { - if (!storage_->tokens->Next().is_operator(';')) { - throw IfcException(std::string("Expected ;")); - } - } else { - // std::unreachable(); - } -} - -void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) { - if (storage_ != nullptr) { - if (storage_->tokens->Next().as_string() != term) { - throw IfcException(std::string("Expected " + term)); - } - if (trail == TRAILING_SEMICOLON) { - readSemicolon(); - } - } else { - // std::unreachable(); - } -} - -IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file) - : file_(file) -{ - Header_section_schema::get_schema(); - - // @todo This might still not work in IfcFile's uninitialized mode - storage_ = std::visit([this](auto& m) -> decltype(storage_) { - if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - return &m; - } - return nullptr; +std::shared_ptr make_header_entity(IfcParse::IfcFile* file, const IfcParse::entity& decl) { + const bool in_memory = file == nullptr || std::visit([](auto& storage) { + return std::is_same_v, IfcParse::impl::in_memory_file_storage>; }, file->storage_); - - const bool in_memory = storage_ != nullptr; if (in_memory) { - header_entities_[0] = std::make_shared(file, &Header_section_schema::file_description::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_description::Class().attribute_count())); - header_entities_[1] = std::make_shared(file, &Header_section_schema::file_name::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_name::Class().attribute_count())); - header_entities_[2] = std::make_shared(file, &Header_section_schema::file_schema::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_schema::Class().attribute_count())); - } else { - header_entities_[0] = std::make_shared(file, &Header_section_schema::file_description::Class(), 0, rocks_db_attribute_storage{}); - header_entities_[1] = std::make_shared(file, &Header_section_schema::file_name::Class(), 0, rocks_db_attribute_storage{}); - header_entities_[2] = std::make_shared(file, &Header_section_schema::file_schema::Class(), 0, rocks_db_attribute_storage{}); + return std::make_shared(file, &decl, 0, in_memory_attribute_storage(decl.attribute_count())); } + + return std::make_shared(file, &decl, 0, rocks_db_attribute_storage{}); } -IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcSpfLexer* lexer) -{ +} // namespace + +IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file) + : file_(file) { Header_section_schema::get_schema(); - storage_ = new impl::in_memory_file_storage; - storage_->tokens = lexer; - file_ = nullptr; + header_entities_[0] = make_header_entity(file_, Header_section_schema::file_description::Class()); + header_entities_[1] = make_header_entity(file_, Header_section_schema::file_name::Class()); + header_entities_[2] = make_header_entity(file_, Header_section_schema::file_schema::Class()); } -IfcParse::IfcSpfHeader::~IfcSpfHeader() { -} - -void IfcSpfHeader::read() { - readTerminal(ISO_10303_21, TRAILING_SEMICOLON); - readTerminal(HEADER, TRAILING_SEMICOLON); - - // | The header section of every exchange structure shall contain one - // | instance of each of the following entities: file_description, file_name, - // | and file_schema, and they shall appear in that order. Instances of - // | file_population, section_language and section_context may appear after - // | file_schema. If instances of user-defined header section entities are - // | present, they shall appear after the header section entity instances - // | defined in this section. - // - // ISO 10303-21 Second edition 2002-01-15 p. 16 - - readTerminal(Header_section_schema::file_description::Class().name_uc(), NONE); - header_entities_[0] = read_from_spf_file(file_, storage_, &Header_section_schema::file_description::Class()); - readSemicolon(); - - readTerminal(Header_section_schema::file_name::Class().name_uc(), NONE); - header_entities_[1] = read_from_spf_file(file_, storage_, &Header_section_schema::file_name::Class()); - readSemicolon(); - - readTerminal(Header_section_schema::file_schema::Class().name_uc(), NONE); - header_entities_[2] = read_from_spf_file(file_, storage_, &Header_section_schema::file_schema::Class()); - readSemicolon(); -} - -bool IfcSpfHeader::tryRead() { - try { - read(); - return true; - } catch (const std::exception& e) { - Logger::Error(e); - return false; - } -} +IfcParse::IfcSpfHeader::~IfcSpfHeader() = default; void IfcSpfHeader::write(std::ostream& out) const { out << ISO_10303_21 << ";" @@ -163,18 +58,21 @@ void IfcSpfHeader::write(std::ostream& out) const { void IfcParse::IfcSpfHeader::file(IfcParse::IfcFile* file) { file_ = file; - if (file != nullptr) { - storage_ = std::visit([this](auto& m) -> decltype(storage_) { - if constexpr (std::is_same_v, impl::in_memory_file_storage>) { - return &m; - } - return nullptr; - }, - file_->storage_); - } } -const Header_section_schema::file_description IfcParse::IfcSpfHeader::file_description() const { +void IfcParse::IfcSpfHeader::set_file_description(const std::shared_ptr& data) { + header_entities_[0] = data; +} + +void IfcParse::IfcSpfHeader::set_file_name(const std::shared_ptr& data) { + header_entities_[1] = data; +} + +void IfcParse::IfcSpfHeader::set_file_schema(const std::shared_ptr& data) { + header_entities_[2] = data; +} + +const Header_section_schema::file_description IfcParse::IfcSpfHeader::file_description() const { return Header_section_schema::file_description(header_entities_[0]); } diff --git a/src/ifcparse/IfcSpfHeader.h b/src/ifcparse/IfcSpfHeader.h index 6f55e01cba..ec0ee00f14 100644 --- a/src/ifcparse/IfcSpfHeader.h +++ b/src/ifcparse/IfcSpfHeader.h @@ -23,7 +23,6 @@ #include "ifc_parse_api.h" #include "InstanceData.h" #include "Header_section_schema.h" -#include "storage.h" namespace IfcParse { @@ -32,39 +31,22 @@ class IfcFile; class IFC_PARSE_API IfcSpfHeader { private: IfcFile* file_; - IfcParse::impl::in_memory_file_storage* storage_ = nullptr; std::array, 3> header_entities_; - /* - mutable Header_section_schema::file_description* file_description_; - mutable Header_section_schema::file_name* file_name_; - mutable Header_section_schema::file_schema* file_schema_; - */ - void readSemicolon(); - enum Trail { - TRAILING_SEMICOLON, - NONE - }; - void readTerminal(const std::string& term, Trail trail); - public: explicit IfcSpfHeader(IfcParse::IfcFile* file); - explicit IfcSpfHeader(IfcParse::IfcSpfLexer* lexer); - ~IfcSpfHeader(); - // IfcParse::IfcFile* file() { return file_; } - // void file(IfcParse::IfcFile* file); - - void read(); - bool tryRead(); - void write(std::ostream& out) const; IfcParse::IfcFile* file() { return file_; } void file(IfcParse::IfcFile* file); + void set_file_description(const std::shared_ptr& data); + void set_file_name(const std::shared_ptr& data); + void set_file_schema(const std::shared_ptr& data); + const Header_section_schema::file_description file_description() const; const Header_section_schema::file_name file_name() const; const Header_section_schema::file_schema file_schema() const; diff --git a/src/ifcparse/storage.h b/src/ifcparse/storage.h index 0a84a04004..41680d6c93 100644 --- a/src/ifcparse/storage.h +++ b/src/ifcparse/storage.h @@ -131,8 +131,8 @@ namespace IfcParse { typedef std::list, std::vector>>>> unresolved_references; class IfcFile; + template class IfcSpfLexer; - class FileReader; struct Token { enum TokenType { @@ -202,15 +202,36 @@ namespace IfcParse { } }; + struct parse_context; + struct parse_context_pool; + + struct parse_context_handle { + parse_context_pool* pool = nullptr; + uint32_t index = 0; + + parse_context& get() const; + parse_context* operator->() const; + parse_context& operator*() const; + explicit operator bool() const { return pool != nullptr; } + }; + struct parse_context { - std::list< + std::vector< std::variant< express::Base, Token, - parse_context* + parse_context_handle >> tokens_; - parse_context() {}; + void reset() { + tokens_.clear(); + } + + parse_context_pool* pool_; + + parse_context(parse_context_pool* pool) : pool_(pool) { + tokens_.reserve(16); + }; ~parse_context(); parse_context(const parse_context&) = delete; @@ -228,16 +249,47 @@ namespace IfcParse { std::shared_ptr construct(IfcParse::IfcFile* owner, std::optional name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, std::optional expected_size, int resolve_reference_index, bool coerce_attribute_count=true); }; + struct parse_context_pool { + std::vector nodes_; + uint32_t used_ = 0; + + void reset() { used_ = 0; } + + parse_context_handle make() { + if (used_ == nodes_.size()) { + nodes_.emplace_back(this); + } + auto idx = used_++; + nodes_[idx].reset(); + return {this, idx}; + } + + parse_context& get(uint32_t index) { + return nodes_[index]; + } + }; + + inline parse_context& parse_context_handle::get() const { + return pool->get(index); + } + + inline parse_context* parse_context_handle::operator->() const { + return &pool->get(index); + } + + inline parse_context& parse_context_handle::operator*() const { + return pool->get(index); + } + namespace impl { struct IFC_PARSE_API in_memory_file_storage { + IfcParse::parse_context_pool context_pool_; + std::vector> read_simple_type_instances; std::vector> steal_instances() { return read_simple_type_instances; } - IfcParse::IfcSpfLexer* tokens; - // IfcParse::FileReader* stream; - // Either one of these needs to be set IfcParse::IfcFile* file; const IfcParse::schema_definition* schema; @@ -258,7 +310,7 @@ namespace IfcParse { typedef std::map> entities_by_ref_t; typedef entity_instance_by_name_t::iterator iterator; - in_memory_file_storage(IfcParse::IfcFile* f = nullptr) : tokens(nullptr), file(f), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr& d) { return express::Base(d); }) {}; + in_memory_file_storage(IfcParse::IfcFile* f = nullptr) : file(f), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr& d) { return express::Base(d); }) {}; in_memory_file_storage(const in_memory_file_storage&) = delete; in_memory_file_storage(const in_memory_file_storage&&) = delete; @@ -303,13 +355,16 @@ namespace IfcParse { entity_instance_by_guid_t byguid_; entity_instance_by_name_t byid_read_; - void load(std::optional entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); - void try_read_semicolon() const; + template + void load(IfcParse::IfcSpfLexer* tokens, std::optional entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1); + template + void try_read_semicolon(IfcParse::IfcSpfLexer* tokens) const; void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index); void unregister_inverse(unsigned, const IfcParse::entity* from_entity, const express::Base&, int attribute_index); - void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); + template + void read_from_stream(Reader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set& typed_to_bypass); file_open_status good_ = file_open_status::SUCCESS; diff --git a/src/ifcwrap/IfcParseWrapper.i b/src/ifcwrap/IfcParseWrapper.i index e7039b0f29..57579df835 100644 --- a/src/ifcwrap/IfcParseWrapper.i +++ b/src/ifcwrap/IfcParseWrapper.i @@ -31,10 +31,8 @@ %ignore IfcParse::IfcFile::byref_excl_; %ignore IfcParse::IfcFile::types_to_bypass_loading_; -%ignore IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer); - -%ignore IfcParse::InstanceStreamer::readInstance; -%ignore IfcParse::InstanceStreamer::stealInstances; +%ignore IfcParse::InstanceStreamer>::readInstance; +%ignore IfcParse::InstanceStreamer>::stealInstances; %ignore express::Entity; %ignore express::Select; @@ -54,9 +52,6 @@ %ignore IfcParse::FileSchema::FileSchema; %ignore IfcParse::IfcFile::tokens; -%ignore IfcParse::IfcSpfHeader::IfcSpfHeader(IfcSpfLexer*); -%ignore IfcParse::IfcSpfHeader::lexer; -%ignore IfcParse::IfcSpfHeader::stream; %ignore IfcParse::IfcSpfHeader::file_description; %ignore IfcParse::IfcSpfHeader::file_name; %ignore IfcParse::IfcSpfHeader::file_schema; @@ -922,6 +917,7 @@ object = custom_base %} %include "../ifcparse/IfcFile.h" +%template(InstanceStreamer) IfcParse::InstanceStreamer>; %pythoncode %{ ### hack hack hack @@ -977,10 +973,10 @@ object = _old_object return f; } - IfcParse::InstanceStreamer* stream_from_string(const std::string& data) { + IfcParse::InstanceStreamer>* stream_from_string(const std::string& data) { char* copiedData = new char[data.length()]; memcpy(copiedData, data.c_str(), data.length()); - return new IfcParse::InstanceStreamer((void *)copiedData, data.length()); + return new IfcParse::InstanceStreamer>((void *)copiedData, data.length()); } const char* version() { @@ -1247,7 +1243,7 @@ object = _old_object } %} -%extend IfcParse::InstanceStreamer { +%extend IfcParse::InstanceStreamer> { PyObject* readInstancePy(bool type_as_declaration_instance=false) { auto simply_type_to_dictionary = [&](const express::Base& t) -> PyObject* { const auto& nm = t.declaration().name(); @@ -1452,4 +1448,4 @@ object = _old_object return d; } } - \ No newline at end of file +