Try some things: (a) fewer allocations - parse context pool; lexer string pool (b) SWAR process multiple chars at once in keywords/enums/strs/stc.

src/ifcconvert/IfcConvert.cpp

@@ -522,9 +522,9 @@ int main(int argc, char** argv) {
 #ifdef HAVE_ICU
     if (!unicode_mode.empty()) {
         if (unicode_mode == "utf8") {
-            IfcParse::IfcCharacterDecoder::mode = IfcParse::IfcCharacterDecoder::UTF8;
+            IfcParse::RuntimeIfcCharacterDecoder::mode = IfcParse::RuntimeIfcCharacterDecoder::UTF8;
         } else if (unicode_mode == "escape") {
-            IfcParse::IfcCharacterDecoder::mode = IfcParse::IfcCharacterDecoder::JSON;
+            IfcParse::RuntimeIfcCharacterDecoder::mode = IfcParse::RuntimeIfcCharacterDecoder::ESCAPE;
         } else {
             cerr_ << "[Error] Invalid value for --unicode" << std::endl;
             print_options(serializer_options);

src/ifcparse/FileReader.cpp

@@ -1,18 +1,13 @@
 #include "FileReader.h"
 
 #include <algorithm>
+#include <cstring>
 #include <cstdio>
 #include <filesystem>
 #include <fstream>
-#include <list>
 #include <stdexcept>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-#include <deque>
 
 #ifdef USE_MMAP
-#include <boost/iostreams/device/mapped_file.hpp>
 #include <boost/filesystem/path.hpp>
 #endif
 
@@ -21,306 +16,350 @@
 namespace {
 
 #if defined(_WIN32)
-    inline void file_seek_abs(FILE* f, std::uint64_t off) {
+inline void file_seek_abs(FILE* f, std::uint64_t off) {
-        if (_fseeki64(f, static_cast<long long>(off), SEEK_SET) != 0)
+    if (_fseeki64(f, static_cast<long long>(off), SEEK_SET) != 0) {
-            throw std::runtime_error("fseek failed");
+        throw std::runtime_error("fseek failed");
     }
+}
 #else
-    inline void file_seek_abs(FILE* f, std::uint64_t off) {
+inline void file_seek_abs(FILE* f, std::uint64_t off) {
-        if (fseeko(f, static_cast<off_t>(off), SEEK_SET) != 0)
+    if (fseeko(f, static_cast<off_t>(off), SEEK_SET) != 0) {
-            throw std::runtime_error("fseeko failed");
+        throw std::runtime_error("fseeko failed");
     }
+}
 #endif
 
 } // namespace
 
 using namespace IfcParse;
 
-struct FullBufferImpl final : FileReader::Impl {
+namespace {
-    std::vector<char> buf_;
+
-    explicit FullBufferImpl(const std::string& fn) {
+template <typename T, typename Fn>
+T gather_value(Fn&& fn) {
+    char bytes[sizeof(T)];
+    for (size_t i = 0; i < sizeof(bytes); ++i) {
+        bytes[i] = fn(i);
+    }
+    T value;
+    std::memcpy(&value, bytes, sizeof(value));
+    return value;
+}
+
+} // namespace
+
+FullBufferImpl::FullBufferImpl(const std::string& fn) {
 #ifdef _MSC_VER
-        std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
+    std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
-        const wchar_t* fn_wide = fn_ws.c_str();
+    const wchar_t* fn_wide = fn_ws.c_str();
-        auto stream = _wfopen(fn_wide, L"rb");
+    auto stream = _wfopen(fn_wide, L"rb");
 #else
-        auto stream = fopen(fn.c_str(), "rb");
+    auto stream = fopen(fn.c_str(), "rb");
 #endif
-        fseek(stream, 0, SEEK_END);
+    if (!stream) {
-        buf_.resize((size_t)ftell(stream));
+        throw std::runtime_error("Failed to open file");
-        rewind(stream);
-        buf_.resize((size_t) fread(buf_.data(), 1, buf_.capacity(), stream));
-        fclose(stream);
     }
-    size_t size() const override { return buf_.size(); }
+    fseek(stream, 0, SEEK_END);
-    char get(size_t pos) const override {
+    size_ = (size_t)ftell(stream);
-        if (pos >= buf_.size()) throw std::out_of_range("get out of range");
+    buf_.resize(size_);
-        return buf_[pos];
+    rewind(stream);
+    buf_.resize((size_t)fread(buf_.data(), 1, buf_.capacity(), stream));
+    fclose(stream);
+}
+
+size_t FullBufferImpl::size() const { return size_; }
+
+char FullBufferImpl::get(size_t pos) const {
+    if (pos >= buf_.size()) {
+        throw std::out_of_range("get out of range");
     }
-};
+    return buf_[pos];
+}
 
-struct PagedFileImpl final : FileReader::Impl {
+uint64_t FullBufferImpl::get_u64(size_t pos) const {
-    std::string fn_;
+    if (pos + sizeof(uint64_t) > buf_.size()) {
-    FILE* fp_ = nullptr;
+        throw std::out_of_range("get_u64 out of range");
-    size_t file_size_ = 0;
+    }
-    size_t page_size_ = 4096;
+    uint64_t value;
+    std::memcpy(&value, buf_.data() + pos, sizeof(value));
+    return value;
+}
 
-    // LRU cache
+uint32_t FullBufferImpl::get_u32(size_t pos) const {
-    size_t capacity_ = 8;
+    if (pos + sizeof(uint32_t) > buf_.size()) {
-    mutable std::list<size_t> lru_;
+        throw std::out_of_range("get_u32 out of range");
-    struct Entry {
+    }
-        FileReader::Page page;
+    uint32_t value;
-        std::list<size_t>::iterator it;
+    std::memcpy(&value, buf_.data() + pos, sizeof(value));
-    };
+    return value;
-    mutable std::unordered_map<size_t, Entry> map_;
+}
 
-    PagedFileImpl(const std::string& fn, size_t page_size, size_t cap)
+void FullBufferImpl::pushNextPage(const std::string&) {
-        : fn_(fn), page_size_(std::max<size_t>(512, page_size)), capacity_(std::max<size_t>(2, cap))
+    throw std::logic_error("push_next_page: backend does not support pushed mode");
-    {
+}
+
+void FullBufferImpl::dropPages(size_t) {
+}
+
+PagedFileImpl::PagedFileImpl(const std::string& fn, size_t page_size, size_t cap)
+    : fn_(fn)
+    , page_size_(std::max<size_t>(512, page_size))
+    , capacity_(std::max<size_t>(2, cap)) {
 #ifdef _MSC_VER
-        std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
+    std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
-        const wchar_t* fn_wide = fn_ws.c_str();
+    const wchar_t* fn_wide = fn_ws.c_str();
-        fp_ = _wfopen(fn_wide, L"rb");
+    fp_ = _wfopen(fn_wide, L"rb");
 #else
-        fp_ = fopen(fn.c_str(), "rb");
+    fp_ = fopen(fn.c_str(), "rb");
 #endif
-        fseek(fp_, 0, SEEK_END);
+    if (!fp_) {
-        file_size_ = (size_t)ftell(fp_);
+        throw std::runtime_error("Failed to open file");
-        rewind(fp_);
+    }
+    fseek(fp_, 0, SEEK_END);
+    file_size_ = (size_t)ftell(fp_);
+    rewind(fp_);
+}
+
+PagedFileImpl::~PagedFileImpl() {
+    if (fp_) {
+        std::fclose(fp_);
+    }
+    fp_ = nullptr;
+}
+
+size_t PagedFileImpl::size() const { return file_size_; }
+
+char PagedFileImpl::get(size_t pos) const {
+    if (pos >= file_size_) {
+        throw std::out_of_range("get out of range");
+    }
+    const size_t pidx = pos / page_size_;
+    const FileReaderPage& p = fetchPage_(pidx);
+    const size_t off = pos % page_size_;
+    if (off >= p.data.size()) {
+        throw std::out_of_range("offset beyond valid page bytes");
+    }
+    return p.data[off];
+}
+
+uint64_t PagedFileImpl::get_u64(size_t pos) const {
+    if (pos + sizeof(uint64_t) > file_size_) {
+        throw std::out_of_range("get_u64 out of range");
     }
 
-    ~PagedFileImpl() override {
+    const size_t pidx = pos / page_size_;
-        if (fp_) std::fclose(fp_);
+    const FileReaderPage& p = fetchPage_(pidx);
-        fp_ = nullptr;
+    const size_t off = pos % page_size_;
+    if (off + sizeof(uint64_t) <= p.data.size()) {
+        uint64_t value;
+        std::memcpy(&value, p.data.data() + off, sizeof(value));
+        return value;
     }
 
-    size_t size() const override { return file_size_; }
+    return gather_value<uint64_t>([&](size_t i) { return get(pos + i); });
+}
 
-    char get(size_t pos) const override {
+uint32_t PagedFileImpl::get_u32(size_t pos) const {
-        if (pos >= file_size_) throw std::out_of_range("get out of range");
+    if (pos + sizeof(uint32_t) > file_size_) {
-        const size_t pidx = pos / page_size_;
+        throw std::out_of_range("get_u32 out of range");
-        const FileReader::Page& p = fetchPage_(pidx);
-        const size_t off = pos % page_size_;
-        if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes");
-        return p.data[off];
     }
 
-private:
+    const size_t pidx = pos / page_size_;
-    const FileReader::Page& fetchPage_(size_t idx) const {
+    const FileReaderPage& p = fetchPage_(pidx);
-        auto it = map_.find(idx);
+    const size_t off = pos % page_size_;
-        if (it != map_.end()) {
+    if (off + sizeof(uint32_t) <= p.data.size()) {
-            touch_(it);
+        uint32_t value;
-            return it->second.page;
+        std::memcpy(&value, p.data.data() + off, sizeof(value));
+        return value;
+    }
+
+    return gather_value<uint32_t>([&](size_t i) { return get(pos + i); });
+}
+
+void PagedFileImpl::pushNextPage(const std::string&) {
+    throw std::logic_error("push_next_page: backend does not support pushed mode");
+}
+
+void PagedFileImpl::dropPages(size_t) {
+}
+
+const FileReaderPage& PagedFileImpl::fetchPage_(size_t idx) const {
+    auto it = map_.find(idx);
+    if (it != map_.end()) {
+        touch_(it);
+        return it->second.page;
+    }
+
+    FileReaderPage pg;
+    pg.data.resize(page_size_);
+    const size_t begin = idx * page_size_;
+    const size_t avail = std::min(page_size_, file_size_ - begin);
+
+    file_seek_abs(fp_, begin);
+    if (avail > 0) {
+        const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
+        if (nread != avail) {
+            throw std::runtime_error("Short fread on page");
         }
-        // Load page from disk using persistent FILE*
-        FileReader::Page pg;
-        pg.data.resize(page_size_);
-        const size_t begin = idx * page_size_;
-        const size_t avail = std::min(page_size_, file_size_ - begin);
-
-        file_seek_abs(fp_, begin);
-        if (avail > 0) {
-            const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
-            if (nread != avail) throw std::runtime_error("Short fread on page");
-        }
-        // trim to actual size
-		pg.data.resize(avail);
-
-        // Insert into LRU
-        if (map_.size() >= capacity_) evict_();
-        lru_.push_front(idx);
-        auto lit = lru_.begin();
-        auto [emplaced_it, ok] = map_.emplace(idx, Entry{ std::move(pg), lit });
-        (void)ok;
-        return emplaced_it->second.page;
     }
+    pg.data.resize(avail);
 
-    void touch_(typename std::unordered_map<size_t, Entry>::iterator it) const {
+    if (map_.size() >= capacity_) {
-        lru_.erase(it->second.it);
+        evict_();
-        lru_.push_front(it->first);
-        it->second.it = lru_.begin();
     }
+    lru_.push_front(idx);
+    auto lit = lru_.begin();
+    auto [emplaced_it, ok] = map_.emplace(idx, Entry{std::move(pg), lit});
+    (void)ok;
+    return emplaced_it->second.page;
+}
 
-    void evict_() const {
+void PagedFileImpl::touch_(std::unordered_map<size_t, Entry>::iterator it) const {
-        if (lru_.empty()) return;
+    lru_.erase(it->second.it);
-        const size_t victim = lru_.back();
+    lru_.push_front(it->first);
-        lru_.pop_back();
+    it->second.it = lru_.begin();
-        map_.erase(victim);
+}
+
+void PagedFileImpl::evict_() const {
+    if (lru_.empty()) {
+        return;
     }
-};
+    const size_t victim = lru_.back();
+    lru_.pop_back();
+    map_.erase(victim);
+}
 
 #ifdef USE_MMAP
-struct MMapImpl final : FileReader::Impl {
+MMapImpl::MMapImpl(const std::string& fn) {
-    boost::iostreams::mapped_file_source map_;
+    map_.open(boost::filesystem::path(IfcUtil::path::from_utf8(fn)));
-    size_t size_ = 0;
+    if (!map_.is_open()) {
-
+        throw std::runtime_error("Failed to open mapped_file_source");
-    explicit MMapImpl(const std::string& fn) {
-        map_.open(boost::filesystem::path(IfcUtil::path::from_utf8(fn)));
-        if (!map_.is_open()) throw std::runtime_error("Failed to open mapped_file_source");
-        size_ = static_cast<size_t>(map_.size());
     }
+    size_ = static_cast<size_t>(map_.size());
+}
 
-    size_t size() const override { return size_; }
+size_t MMapImpl::size() const { return size_; }
 
-    char get(size_t pos) const override {
+char MMapImpl::get(size_t pos) const {
-        if (pos >= size_) throw std::out_of_range("get out of range");
+    if (pos >= size_) {
-        return map_.data()[pos];
+        throw std::out_of_range("get out of range");
     }
-};
+    return map_.data()[pos];
+}
+
+uint64_t MMapImpl::get_u64(size_t pos) const {
+    if (pos + sizeof(uint64_t) > size_) {
+        throw std::out_of_range("get_u64 out of range");
+    }
+    uint64_t value;
+    std::memcpy(&value, map_.data() + pos, sizeof(value));
+    return value;
+}
+
+uint32_t MMapImpl::get_u32(size_t pos) const {
+    if (pos + sizeof(uint32_t) > size_) {
+        throw std::out_of_range("get_u32 out of range");
+    }
+    uint32_t value;
+    std::memcpy(&value, map_.data() + pos, sizeof(value));
+    return value;
+}
+
+void MMapImpl::pushNextPage(const std::string&) {
+    throw std::logic_error("push_next_page: backend does not support pushed mode");
+}
+
+void MMapImpl::dropPages(size_t) {
+}
 #endif
 
-/// User-pushed sequential backend with an arbitrary-length queue of future pages.
+size_t PushedSequentialImpl::size() const {
-/// We keep a deque of pages; when reads move forward, we drop fully-consumed
+    size_t n = discarded_page_bytes_;
-/// pages from the front to release memory.
+    for (const auto& pg : pages_) {
-struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentialImpl>, FileReader::Impl {
+        n += pg.data.size();
-	// Deque of pages, front is earliest in file.
+    }
-    std::deque<FileReader::Page> pages_;
+    return n;
-    // total bytes in dropped pages
+}
-	size_t discarded_page_bytes_ = 0;
 
-    size_t size() const override {
+void PushedSequentialImpl::dropPages(size_t pos) {
-        size_t n = discarded_page_bytes_;
+    while (!pages_.empty()) {
-		for (auto& pg : pages_) n += pg.data.size();
+        if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
-		return n;
+            discarded_page_bytes_ += pages_.front().data.size();
+            pages_.pop_front();
+        } else {
+            break;
+        }
+    }
+}
+
+char PushedSequentialImpl::get(size_t pos) const {
+    const size_t avail_end = size();
+    if (pos >= avail_end) {
+        throw std::out_of_range("pushed backend: position not committed yet");
     }
 
-    // Drop fully-consumed pages so pos is guaranteed to be within the first page
+    pos -= discarded_page_bytes_;
-    void dropPages(size_t pos) override {
+
-        while (!pages_.empty()) {
+    size_t page_start = 0;
-            if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
+    for (const auto& pg : pages_) {
-				discarded_page_bytes_ += pages_.front().data.size();
+        if (pos < page_start + pg.data.size()) {
-                pages_.pop_front();
+            const size_t off = pos - page_start;
-            } else {
+            return pg.data[off];
-                break;
+        }
+        page_start += pg.data.size();
+    }
+
+    throw std::out_of_range("pushed backend: internal inconsistency");
+}
+
+uint64_t PushedSequentialImpl::get_u64(size_t pos) const {
+    if (pos + sizeof(uint64_t) > size()) {
+        throw std::out_of_range("get_u64 out of range");
+    }
+
+    size_t relative_pos = pos - discarded_page_bytes_;
+    size_t page_start = 0;
+    for (const auto& pg : pages_) {
+        if (relative_pos < page_start + pg.data.size()) {
+            const size_t off = relative_pos - page_start;
+            if (off + sizeof(uint64_t) <= pg.data.size()) {
+                uint64_t value;
+                std::memcpy(&value, pg.data.data() + off, sizeof(value));
+                return value;
             }
+            break;
         }
+        page_start += pg.data.size();
     }
 
-    char get(size_t pos) const override {
+    return gather_value<uint64_t>([&](size_t i) { return get(pos + i); });
-        /*
+}
-        auto self = const_cast<PushedSequentialImpl*>(this);
-        // We do not do this automatically because all variable width tokens:
-        // ENUM/STRING/BINARY/KEYWORD are stored as file offsets until a full
-        // entity instance is finalized.
-        if (this->shared_from_this().use_count() == 2) {
-            // only drop pages when there is only one active client.
-            // NB this->shared_from_this() increases count by 1
-            self->drop_consumed_up_to(pos);
-        }
-        */
 
-        const size_t avail_end = size();
+uint32_t PushedSequentialImpl::get_u32(size_t pos) const {
-        if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet");
+    if (pos + sizeof(uint32_t) > size()) {
+        throw std::out_of_range("get_u32 out of range");
+    }
 
-		pos -= discarded_page_bytes_;
+    size_t relative_pos = pos - discarded_page_bytes_;
-
+    size_t page_start = 0;
-		size_t page_start = 0;
+    for (const auto& pg : pages_) {
-        for (const auto& pg : pages_) {
+        if (relative_pos < page_start + pg.data.size()) {
-            if (pos < page_start + pg.data.size()) {
+            const size_t off = relative_pos - page_start;
-                const size_t off = pos - page_start;
+            if (off + sizeof(uint32_t) <= pg.data.size()) {
-                return pg.data[off];
+                uint32_t value;
-            } else {
+                std::memcpy(&value, pg.data.data() + off, sizeof(value));
-				page_start += pg.data.size();
+                return value;
             }
+            break;
         }
-
+        page_start += pg.data.size();
-        throw std::out_of_range("pushed backend: internal inconsistency");
     }
 
-    void pushNextPage(const std::string& data) override {
+    return gather_value<uint32_t>([&](size_t i) { return get(pos + i); });
-        FileReader::Page p; p.data.assign(data.data(), data.data() + data.size());
-        pages_.push_back(std::move(p));
-    }
-};
-
-IfcParse::FileReader::FileReader(const std::string& fn)
-    : cursor_(0)
-{
-	impl_ = std::make_shared<FullBufferImpl>(fn);
 }
 
-IfcParse::FileReader::FileReader(const std::string& fn, const mmap_tag&)
+void PushedSequentialImpl::pushNextPage(const std::string& data) {
-    : cursor_(0)
+    FileReaderPage p;
-{
+    p.data.assign(data.data(), data.data() + data.size());
-#ifdef USE_MMAP
+    pages_.push_back(std::move(p));
-	impl_ = std::make_shared<MMapImpl>(fn);
-#else
-    (void)fn;
-    throw std::runtime_error("IfcParse::FileReader: mmap_tag specified but library not compiled with USE_MMAP");
-#endif
-}
-
-IfcParse::FileReader::FileReader(const caller_fed_tag&)
-    : cursor_(0)
-{
-	impl_ = std::make_shared<PushedSequentialImpl>();
-}
-
-IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&)
-{
-    impl_ = std::make_shared<PushedSequentialImpl>();
-	impl_->pushNextPage(content);
-}
-
-IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
-    : cursor_(0)
-{
-	impl_ = std::make_shared<PagedFileImpl>(fn, page_size, page_capacity);
-}
-
-FileReader FileReader::clone() const {
-    FileReader c(*this);
-    c.cursor_ = this->cursor_;
-    return c;
-}
-
-void FileReader::seek(size_t pos) {
-    if (pos > impl_->size()) throw std::out_of_range("seek out of range");
-    cursor_ = pos;
-}
-
-size_t FileReader::tell() const { return cursor_; }
-
-size_t FileReader::size() const { return impl_->size(); }
-
-char FileReader::peek() const {
-    if (cursor_ >= impl_->size()) throw std::out_of_range("peek at EOF");
-    return impl_->get(cursor_);
-}
-
-void FileReader::increment(size_t n) {
-    if (cursor_ + n > impl_->size()) throw std::out_of_range("increment past EOF");
-    cursor_ += n;
-}
-
-void IfcParse::FileReader::pushNextPage(const std::string& data)
-{
-    impl_->pushNextPage(data);
-}
-
-void IfcParse::FileReader::dropPages()
-{
-    impl_->dropPages(0);
-}
-
-void IfcParse::FileReader::dropPages(size_t up_to_pos)
-{
-    impl_->dropPages(up_to_pos);
-}
-
-bool IfcParse::FileReader::eof() const
-{
-    return cursor_ >= impl_->size();
-}
-
-char IfcParse::FileReader::read()
-{
-	auto c = peek();
-	increment(1);
-	return c;
-}
-
-char IfcParse::FileReader::get(size_t offset) const
-{
-	return impl_->get(offset);
 }

src/ifcparse/FileReader.h

@@ -1,4 +1,4 @@
-/********************************************************************************
+/********************************************************************************
  *                                                                              *
  * This file is part of IfcOpenShell.                                           *
  *                                                                              *
@@ -30,104 +30,244 @@
 #include "ifc_parse_api.h"
 
 #include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <deque>
+#include <list>
 #include <memory>
-#include <optional>
-#include <string>
-#include <variant>
 #include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
 #include <vector>
 
+#ifdef USE_MMAP
+#include <boost/iostreams/device/mapped_file.hpp>
+#endif
+
 namespace IfcParse {
 
-/// \brief Read-only file accessor that supports four backends:
+struct FileReaderPage {
-///  - full-buffer in RAM
+    std::vector<char> data;
-///  - paged (LRU-cached with capacity)
+};
-///  - memory-mapped via boost::iostreams
+
-///  - user-pushed sequential pages (caller feeds pages intended for streaming reads in WASM)
+struct caller_fed_tag {};
-class IFC_PARSE_API FileReader {
+
+template <typename>
+inline constexpr bool file_reader_dependent_false_v = false;
+
+class IFC_PARSE_API FullBufferImpl;
+class IFC_PARSE_API PagedFileImpl;
+#ifdef USE_MMAP
+class IFC_PARSE_API MMapImpl;
+#endif
+class IFC_PARSE_API PushedSequentialImpl;
+
+template <typename Impl>
+class FileReader {
 public:
-    struct Page { std::vector<char> data; };
+    using impl_type = Impl;
+    using Page = FileReaderPage;
 
-    /// \brief Tag to choose memory-mapped backend via boost::iostreams::mapped_file_source.
+    FileReader() = default;
-    struct mmap_tag {};
-    /// \brief Tag to choose user-pushed sequential pages.
-    struct caller_fed_tag {};
 
-    /// \brief Construct a FileReader.
+    explicit FileReader(const std::string& fn)
-    /// \param fn File path.
+        : cursor_(0) {
-    /// \param maybe_mmaped_or_chunked
+        if constexpr (std::is_same_v<Impl, FullBufferImpl>
-    ///        - std::nullopt: full-buffer mode (entire file loaded into memory)
+#ifdef USE_MMAP
-    ///        - size_t: paged mode with this page size in bytes (e.g., 4096)
+            || std::is_same_v<Impl, MMapImpl>
-    ///        - mmap_tag: memory-mapped mode using boost::iostreams::mapped_file_source
+#endif
-    /// \param page_cache_capacity Number of pages to keep in the LRU when in paged mode.
+        ) {
-    FileReader(const std::string& fn);
+            impl_ = std::make_shared<Impl>(fn);
-
+        } else {
-    FileReader(const std::string& fn, const mmap_tag&);
+            static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
-
-    FileReader(const caller_fed_tag&);
-    FileReader(const std::string& content, const caller_fed_tag&);
-
-    FileReader(const std::string& fn, size_t page_size, size_t page_capacity);
-
-    /// \brief Copy-construct a new reader sharing the underlying storage but with its own cursor.
-    FileReader clone() const;
-
-    /// \brief Seek to an absolute byte position.
-    /// \throws std::out_of_range if pos > size().
-    void seek(size_t pos);
-
-    /// \brief Return the current cursor position.
-    size_t tell() const;
-
-    /// \brief Total file size in bytes.
-    size_t size() const;
-
-    /// \brief Peek the byte at the current cursor.
-    /// \throws std::out_of_range at EOF.
-    char peek() const;
-
-    /// \brief Advance the cursor by n bytes (default 1).
-    /// \throws std::out_of_range if advancing crosses EOF.
-    void increment(size_t n = 1);
-
-    /// \brief Push the next sequential page (pushed backend only).
-	/// \param data Contents of the page.
-    /// \throws std::logic_error if the current backend is not in pushed mode
-    void pushNextPage(const std::string& data);
-
-    /// \brief Drops pages up to cursor position or provided offset. Does nothing when current backend is not in pushed mode
-    /// \param up_to_pos Pages with an end offset before up_to_pos are dropped from memory
-    void dropPages();
-    void dropPages(size_t up_to_pos);
-
-    /// \brief Returns true if the cursor is at or beyond the end of available data.
-    /// For the pushed backend, EOF means all pushed bytes have been consumed.
-    bool eof() const;
-
-    /// \brief Equivalent of peek() followed by increment(1)
-    char read();
-
-    /// \brief Equivalent of peek() followed by increment(1)
-    char get(size_t offset) const;
-
-    struct Impl {
-        virtual ~Impl() = default;
-        virtual size_t size() const = 0;
-        virtual char get(size_t pos) const = 0;
-        /// \brief Backend may support pushing pages; default throws.
-        virtual void pushNextPage(const std::string&) {
-            throw std::logic_error("push_next_page: backend does not support pushed mode");
         }
-        virtual void dropPages(size_t) {
+    }
-            // empty on purpose
+
+    explicit FileReader(const caller_fed_tag&)
+        : cursor_(0) {
+        if constexpr (std::is_same_v<Impl, PushedSequentialImpl>) {
+            impl_ = std::make_shared<Impl>();
+        } else {
+            static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
         }
-    };
+    }
+
+    FileReader(const std::string& content, const caller_fed_tag&)
+        : FileReader(caller_fed_tag{}) {
+        if constexpr (std::is_same_v<Impl, PushedSequentialImpl>) {
+            impl_->pushNextPage(content);
+        } else {
+            static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
+        }
+    }
+
+    FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
+        : cursor_(0) {
+        if constexpr (std::is_same_v<Impl, PagedFileImpl>) {
+            impl_ = std::make_shared<Impl>(fn, page_size, page_capacity);
+        } else {
+            static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
+        }
+    }
+
+    FileReader clone() const {
+        FileReader c(*this);
+        c.cursor_ = cursor_;
+        return c;
+    }
+
+    void seek(size_t pos) {
+        if (pos > size()) {
+            throw std::out_of_range("seek out of range");
+        }
+        cursor_ = pos;
+    }
+
+    size_t tell() const { return cursor_; }
+
+    size_t size() const { return impl_->size(); }
+    size_t remaining() const { return size() - cursor_; }
+
+    char peek() const {
+        if (cursor_ >= size()) {
+            throw std::out_of_range("peek at EOF");
+        }
+        return impl_->get(cursor_);
+    }
+
+    uint64_t peek_u64() const {
+        if (remaining() < sizeof(uint64_t)) {
+            throw std::out_of_range("peek_u64 at EOF");
+        }
+        return impl_->get_u64(cursor_);
+    }
+
+    uint32_t peek_u32() const {
+        if (remaining() < sizeof(uint32_t)) {
+            throw std::out_of_range("peek_u32 at EOF");
+        }
+        return impl_->get_u32(cursor_);
+    }
+
+    void increment(size_t n = 1) {
+        if (cursor_ + n > size()) {
+            throw std::out_of_range("increment past EOF");
+        }
+        cursor_ += n;
+    }
+
+    void pushNextPage(const std::string& data) {
+        impl_->pushNextPage(data);
+    }
+
+    void dropPages() {
+        impl_->dropPages(0);
+    }
+
+    void dropPages(size_t up_to_pos) {
+        impl_->dropPages(up_to_pos);
+    }
+
+    bool eof() const {
+        return cursor_ >= size();
+    }
+
+    char read() {
+        auto c = peek();
+        increment(1);
+        return c;
+    }
+
+    char get(size_t offset) const {
+        return impl_->get(offset);
+    }
 
 private:
     std::shared_ptr<Impl> impl_;
     size_t cursor_ = 0;
 };
 
+class IFC_PARSE_API FullBufferImpl {
+public:
+    explicit FullBufferImpl(const std::string& fn);
+
+    size_t size() const;
+    char get(size_t pos) const;
+    uint32_t get_u32(size_t pos) const;
+    uint64_t get_u64(size_t pos) const;
+    void pushNextPage(const std::string& data);
+    void dropPages(size_t pos);
+
+private:
+    std::vector<char> buf_;
+    size_t size_;
+};
+
+class IFC_PARSE_API PagedFileImpl {
+public:
+    struct Entry {
+        FileReaderPage page;
+        std::list<size_t>::iterator it;
+    };
+
+    PagedFileImpl(const std::string& fn, size_t page_size, size_t cap);
+    ~PagedFileImpl();
+
+    size_t size() const;
+    char get(size_t pos) const;
+    uint32_t get_u32(size_t pos) const;
+    uint64_t get_u64(size_t pos) const;
+    void pushNextPage(const std::string& data);
+    void dropPages(size_t pos);
+
+private:
+    const FileReaderPage& fetchPage_(size_t idx) const;
+    void touch_(std::unordered_map<size_t, Entry>::iterator it) const;
+    void evict_() const;
+
+    std::string fn_;
+    FILE* fp_ = nullptr;
+    size_t file_size_ = 0;
+    size_t page_size_ = 4096;
+    size_t capacity_ = 8;
+    mutable std::list<size_t> lru_;
+    mutable std::unordered_map<size_t, Entry> map_;
+};
+
+#ifdef USE_MMAP
+class IFC_PARSE_API MMapImpl {
+public:
+    explicit MMapImpl(const std::string& fn);
+
+    size_t size() const;
+    char get(size_t pos) const;
+    uint32_t get_u32(size_t pos) const;
+    uint64_t get_u64(size_t pos) const;
+    void pushNextPage(const std::string& data);
+    void dropPages(size_t pos);
+
+private:
+    boost::iostreams::mapped_file_source map_;
+    size_t size_ = 0;
+};
+#endif
+
+class IFC_PARSE_API PushedSequentialImpl {
+public:
+    size_t size() const;
+    char get(size_t pos) const;
+    uint32_t get_u32(size_t pos) const;
+    uint64_t get_u64(size_t pos) const;
+    void pushNextPage(const std::string& data);
+    void dropPages(size_t pos);
+
+private:
+    std::deque<FileReaderPage> pages_;
+    size_t discarded_page_bytes_ = 0;
+};
+
 } // namespace IfcParse
 
 #endif

src/ifcparse/IfcCharacterDecoder.cpp

@@ -77,18 +77,81 @@
 
 using namespace IfcParse;
 
-IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::FileReader* stream) {
+namespace SWAR {
-    stream_ = stream;
+
-    codepage_ = 0;
+constexpr uint32_t ONES32 = 0x01010101u;
+constexpr uint32_t HIGHS32 = 0x80808080u;
+constexpr uint64_t ONES64 = 0x0101010101010101ull;
+constexpr uint64_t HIGHS64 = 0x8080808080808080ull;
+
+constexpr uint32_t splat32(unsigned char c) {
+    return ONES32 * c;
 }
 
-IfcCharacterDecoder::~IfcCharacterDecoder() {
+constexpr uint64_t splat64(unsigned char c) {
+    return ONES64 * c;
+}
+
+inline uint32_t has_zero_byte(uint32_t x) {
+    return (x - ONES32) & ~x & HIGHS32;
+}
+
+inline uint64_t has_zero_byte(uint64_t x) {
+    return (x - ONES64) & ~x & HIGHS64;
+}
+
+inline uint32_t eq_mask(uint32_t x, uint32_t c) {
+    return has_zero_byte(x ^ c);
+}
+
+inline uint64_t eq_mask(uint64_t x, uint64_t c) {
+    return has_zero_byte(x ^ c);
+}
+
+namespace chars {
+constexpr uint32_t apostrophe32 = splat32('\'');
+constexpr uint32_t solidus32 = splat32('\\');
+constexpr uint64_t apostrophe64 = splat64('\'');
+constexpr uint64_t solidus64 = splat64('\\');
+} // namespace chars
+
+inline uint32_t has_special_char(uint32_t x) {
+    return has_zero_byte(x) |
+           eq_mask(x, chars::apostrophe32) |
+           eq_mask(x, chars::solidus32) |
+           (x & HIGHS32);
+}
+
+inline uint64_t has_special_char(uint64_t x) {
+    return has_zero_byte(x) |
+           eq_mask(x, chars::apostrophe64) |
+           eq_mask(x, chars::solidus64) |
+           (x & HIGHS64);
+}
+
+inline void append_ascii(std::u32string& builder, const char* bytes, size_t count) {
+    builder.reserve(builder.size() + count);
+    for (size_t i = 0; i < count; ++i) {
+        builder.push_back(static_cast<unsigned char>(bytes[i]));
+    }
+}
+
+} // namespace SWAR
+
+template <typename Reader>
+IfcCharacterDecoder<Reader>::IfcCharacterDecoder(Reader* stream) {
+    stream_ = stream;
+    codepage_ = 0;
+    builder_.reserve(1024);
+}
+
+template <typename Reader>
+IfcCharacterDecoder<Reader>::~IfcCharacterDecoder() {
 }
 
 namespace {
-    std::string read_string(IfcParse::FileReader& stream_, IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
+    template <typename Reader>
-        std::u32string builder_;
+    std::string read_string(std::u32string& builder_, Reader& stream_, typename IfcParse::IfcCharacterDecoder<Reader>::ConversionMode mode, char substitution_character) {
-
         unsigned int parse_state = 0;
         builder_.clear();
         // builder_.push_back('\'');
@@ -97,7 +160,31 @@ namespace {
         unsigned int hex = 0;
         unsigned int hex_count = 0;
 
-        while ((current_char = stream_.peek()) != 0) {
+        while (!stream_.eof()) {
+            if (parse_state == 0U) {
+                if (stream_.remaining() >= 8) {
+                    uint64_t x = stream_.peek_u64();
+                    if (SWAR::has_special_char(x) == 0) {
+                        SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 8);
+                        stream_.increment(8);
+                        continue;
+                    }
+                }
+                if (stream_.remaining() >= 4) {
+                    uint32_t x = stream_.peek_u32();
+                    if (SWAR::has_special_char(x) == 0) {
+                        SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 4);
+                        stream_.increment(4);
+                        continue;
+                    }
+                }
+            }
+
+            current_char = stream_.peek();
+            if (current_char == 0) {
+                break;
+            }
+
             if (EXPECTS_CHARACTER(parse_state)) {
                 builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80));
                 parse_state = 0;
@@ -171,7 +258,7 @@ namespace {
         }
         // builder_.push_back('\'');
 
-        if (mode == IfcParse::IfcCharacterDecoder::UTF8) {
+        if (mode == IfcParse::IfcCharacterDecoder<Reader>::UTF8) {
             if (builder_.empty()) {
                 static std::string empty;
                 return empty;
@@ -183,7 +270,7 @@ namespace {
             }
             return IfcUtil::convert_utf8(builder_);
         }
-        if (mode == IfcParse::IfcCharacterDecoder::SUBSTITUTE) {
+        if (mode == IfcParse::IfcCharacterDecoder<Reader>::SUBSTITUTE) {
             std::string result;
             result.reserve(builder_.size());
             std::transform(builder_.begin(), builder_.end(), std::back_inserter(result), [&substitution_character](std::u32string::value_type character) {
@@ -194,7 +281,7 @@ namespace {
             });
             return result;
         }
-        if (mode == IfcParse::IfcCharacterDecoder::ESCAPE) {
+        if (mode == IfcParse::IfcCharacterDecoder<Reader>::ESCAPE) {
             std::stringstream stream;
             stream << std::hex << std::setw(4) << std::setfill('0');
             std::for_each(builder_.begin(), builder_.end(), [&stream](std::u32string::value_type character) {
@@ -210,20 +297,26 @@ namespace {
     }
 } // namespace
 
-IfcCharacterDecoder::operator std::string() {
+template <typename Reader>
-    return read_string(*stream_, mode, substitution_character);
+IfcCharacterDecoder<Reader>::operator std::string() {
+    return read_string(builder_, *stream_, mode, substitution_character);
 }
 
-std::string IfcCharacterDecoder::get(size_t& ptr) {
+template <typename Reader>
+std::string IfcCharacterDecoder<Reader>::get(size_t& ptr) {
     auto local_stream = *stream_;
     local_stream.seek(ptr);
-    auto s = read_string(local_stream, mode, substitution_character);
+    auto s = read_string(builder_, local_stream, mode, substitution_character);
 	ptr = local_stream.tell();
     return s;
 }
 
-IfcCharacterDecoder::ConversionMode IfcCharacterDecoder::mode = IfcCharacterDecoder::UTF8;
+template class IfcCharacterDecoder<FileReader<FullBufferImpl>>;
-char IfcCharacterDecoder::substitution_character = '_';
+template class IfcCharacterDecoder<FileReader<PagedFileImpl>>;
+template class IfcCharacterDecoder<FileReader<PushedSequentialImpl>>;
+#ifdef USE_MMAP
+template class IfcCharacterDecoder<MMapFileReader>;
+#endif
 
 IfcCharacterEncoder::IfcCharacterEncoder(const std::string& input)
     : str_(IfcUtil::convert_utf8(input)) {}

src/ifcparse/IfcCharacterDecoder.h

@@ -39,10 +39,12 @@ IFC_PARSE_API std::u32string convert_utf8(const std::string& string);
 
 namespace IfcParse {
 
+template <typename Reader>
 class IFC_PARSE_API IfcCharacterDecoder {
   private:
-    IfcParse::FileReader* stream_;
+    Reader* stream_;
     int codepage_;
+    std::u32string builder_;
 
   public:
     enum ConversionMode {
@@ -50,9 +52,10 @@ class IFC_PARSE_API IfcCharacterDecoder {
         UTF8,
         ESCAPE
     };
-    static ConversionMode mode;
+    inline static ConversionMode mode = UTF8;
-    static char substitution_character;
+    inline static char substitution_character = '_';
-    IfcCharacterDecoder(IfcParse::FileReader* stream);
+
+    IfcCharacterDecoder(Reader* stream);
     ~IfcCharacterDecoder();
     // Gets a decoded string representation at the token stream
     // read pointer and advances the underlying token stream.

src/ifcparse/IfcFile.cpp

@@ -21,9 +21,9 @@ IfcParse::parse_context::~parse_context() {
 }
 
 IfcParse::parse_context& IfcParse::parse_context::push() {
-    auto* pc = new IfcParse::parse_context;
+    auto child = pool_->make();
-    tokens_.push_back(pc);
+    tokens_.emplace_back(child);
-    return *pc;
+    return *child;
 }
 
 void IfcParse::parse_context::push(Token t) {
@@ -220,7 +220,7 @@ namespace {
                 if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::Token>) {
                     // @todo get aggregate of enumeration
                     dispatch_token(instance_id, attribute_id, v, aggr && aggr->type_of_element()->as_named_type() ? aggr->type_of_element()->as_named_type()->declared_type() : nullptr, append_to_aggregate_storage);
-                } else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context*>) {
+                } else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context_handle>) {
                     // nested list
                     if constexpr (Depth < 3) {
                         construct_<Depth + 1>(instance_id, attribute_id, *v, nullptr, append_to_aggregate_storage);
@@ -245,7 +245,7 @@ std::shared_ptr<InstanceData> IfcParse::parse_context::construct(IfcParse::IfcFi
         transient_named_type.reset(new IfcParse::named_type(const_cast<IfcParse::declaration*>(decl)));
         parameter_types = { &*transient_named_type };
     } else if ((decl != nullptr) && (decl->as_entity() != nullptr)) {
-        auto entity_attrs = decl->as_entity()->all_attributes();
+        const auto& entity_attrs = decl->as_entity()->all_attributes();
         std::transform(
             entity_attrs.begin(),
             entity_attrs.end(),
@@ -307,7 +307,7 @@ std::shared_ptr<InstanceData> IfcParse::parse_context::construct(IfcParse::IfcFi
                         storage.set(index, v);
                     }
                 });
-            } else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context*>) {
+            } else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context_handle>) {
                 const auto *pt = param_type;
                 if (pt) {
                     while (pt->as_named_type() && pt->as_named_type()->declared_type()->as_type_declaration()) {
@@ -642,125 +642,6 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
     }
 }
 
-void IfcParse::InstanceStreamer::bypassTypes(const std::set<std::string>& type_names) {
-    for (auto& name : type_names) {
-        try {
-            types_to_bypass_.push_back(schema_->declaration_by_name(name));
-        } catch (const IfcException&) {
-            continue;
-        }
-    }
- }
-
-
-std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> IfcParse::InstanceStreamer::readInstance() {
-     std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> return_value;
-
-    if (header_ && yielded_header_instances_ < 3) {
-        if (yielded_header_instances_ == 0) {
-            return_value.emplace(
-                0,
-                &header_->file_description().declaration(),
-                header_->file_description().data_weak().lock()
-             );
-        } else if (yielded_header_instances_ == 1) {
-            return_value.emplace(
-                0,
-                &header_->file_name().declaration(),
-                header_->file_name().data_weak().lock()
-            );
-        } else if (yielded_header_instances_ == 2) {
-            return_value.emplace(
-                0,
-                &header_->file_schema().declaration(),
-                header_->file_schema().data_weak().lock()
-            );
-        }
-        yielded_header_instances_ += 1;
-        return return_value;
-    }
-
-    unsigned current_id = 0;
-    while (good_ && !lexer_->stream->eof() && !current_id) {
-        if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER &&
-            token_stream_[1].type == IfcParse::Token::Token_OPERATOR &&
-            token_stream_[1].value_char == '=' &&
-            token_stream_[2].type == IfcParse::Token::Token_KEYWORD) {
-            current_id = token_stream_[0].as_identifier();
-            const IfcParse::declaration* entity_type;
-            try {
-                entity_type = schema_->declaration_by_name(token_stream_[2].as_string());
-            } catch (const IfcException& ex) {
-                Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos));
-                current_id = 0;
-                goto advance;
-            }
-
-            if (entity_type->as_entity() == nullptr) {
-                Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos));
-                goto advance;
-            }
-
-            for (auto& ty : types_to_bypass_) {
-                if (entity_type->is(*ty)) {
-                    bypassed_instances_.push_back(current_id);
-                    // Why is this a conditional clause in the loop?
-                    current_id = 0;
-                    goto advance;
-                }
-            }
-
-            parse_context ps;
-            lexer_->Next();
-            try {
-                storage_.load(current_id, entity_type->as_entity(), ps, -1);
-            } catch (const IfcInvalidTokenException& e) {
-                good_ = file_open_status::INVALID_SYNTAX;
-                Logger::Error(e);
-                break;
-            }
-
-            /// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code?
-            // Update the status after every 1000 instances parsed
-            if (((++progress_) % 1000) == 0) {
-                std::stringstream ss;
-                ss << "\r#" << current_id;
-                Logger::Status(ss.str(), false);
-            }
-
-            auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count);
-
-            return_value.emplace(
-                (size_t)current_id,
-                entity_type,
-                data
-            );
-        }
-    advance:
-        Token next_token;
-        try {
-            next_token = lexer_->Next();
-        } catch (const IfcException& e) {
-            Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated");
-        } catch (...) {
-            Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
-        }
-
-        if (!lexer_->stream->eof() && !next_token) {
-            good_ = file_open_status::INVALID_SYNTAX;
-            break;
-        }
-
-        token_stream_.push_back(next_token);
-    }
-
-    // Free pages in front of cursor when variable-width tokens are materialized into entity instance data objects
-    (stream_ ? stream_ : (lexer_)->stream)->dropPages();
-    lexer_->resetPool();
-
-    return return_value;
-}
-
 express::Base IfcParse::impl::rocks_db_file_storage::create(const IfcParse::declaration* decl, int id) {
     return express::Base{};
     /*

src/ifcparse/IfcFile.h

@@ -34,6 +34,7 @@
 #include <boost/circular_buffer.hpp>
 #include <iterator>
 #include <map>
+#include <memory>
 
 #ifdef IFOPSH_WITH_ROCKSDB
 
@@ -87,10 +88,13 @@ enum filetype {
 
 IFC_PARSE_API filetype guess_file_type(const std::string& fn);
 
+template <typename Reader = FileReader<FullBufferImpl>>
 class IFC_PARSE_API InstanceStreamer {
 private:
-    FileReader* stream_;
+    std::unique_ptr<Reader> owned_stream_;
-    IfcSpfLexer* lexer_;
+    Reader* stream_;
+    std::unique_ptr<IfcSpfLexer<Reader>> lexer_;
+    std::unique_ptr<IfcSpfHeader> owned_header_;
     IfcSpfHeader* header_;
     IfcParse::IfcFile* owner_;
     boost::circular_buffer<Token> token_stream_;
@@ -100,14 +104,18 @@ private:
     int progress_;
     IfcParse::unresolved_references references_to_resolve_;
     int yielded_header_instances_ = 0;
+    bool yield_header_instances_ = true;
     std::vector<const declaration*> types_to_bypass_;
     std::vector<unsigned> bypassed_instances_;
 
+    void initialize_header();
+    IfcSpfHeader& ensure_header();
+
   public:
 	bool coerce_attribute_count = true;
 
     operator bool() const {
-        return good_ && !lexer_->stream->eof();
+        return good_ && lexer_ && !lexer_->stream->eof();
     }
 
     IfcParse::file_open_status status() const {
@@ -151,17 +159,17 @@ private:
 
     InstanceStreamer(void* data, int length, IfcParse::IfcFile* f = nullptr);
 
-    InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer, IfcParse::IfcFile* f = nullptr);
+    InstanceStreamer(Reader* stream, IfcParse::IfcFile* f = nullptr);
 
     void bypassTypes(const std::set<std::string>& type_names);
 
-    ~InstanceStreamer() {
+    void yieldHeaderInstances(bool value) { yield_header_instances_ = value; }
-        delete stream_;
+
-        if (stream_) {
+    const IfcParse::schema_definition* schema() const { return schema_; }
-            delete lexer_;
+
-        }
+    const IfcSpfHeader* header() const { return header_; }
-        delete header_;
+
-    }
+    ~InstanceStreamer() = default;
 
     std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> readInstance();
 };
@@ -249,12 +257,6 @@ public:
     /// </summary>
     IfcFile(void* data, int length);
 
-    /// <summary>
-    /// Constructs an IfcFile object from a given IFC SPF stream.
-    /// </summary>
-    /// <param name="stream">A pointer to an IfcParse::FileReader object representing the input IFC SPF data stream.</param>
-    IfcFile(IfcParse::FileReader* stream);
-
     /// <summary>
     /// Constructs an IfcFile object with the specified schema, file type, and file path.
     /// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called

src/ifcparse/IfcParse.cpp

@@ -102,16 +102,19 @@ void init_locale() {
 
 #endif
 
-IfcSpfLexer::IfcSpfLexer(IfcParse::FileReader* stream_) {
+template <typename Reader>
+IfcSpfLexer<Reader>::IfcSpfLexer(Reader* stream_) {
     stream = stream_;
-    decoder_ = new IfcCharacterDecoder(stream_);
+    decoder_ = new IfcCharacterDecoder<Reader>(stream_);
 }
 
-IfcSpfLexer::~IfcSpfLexer() {
+template <typename Reader>
+IfcSpfLexer<Reader>::~IfcSpfLexer() {
     delete decoder_;
 }
 
-size_t IfcSpfLexer::skipWhitespace() const {
+template <typename Reader>
+size_t IfcSpfLexer<Reader>::skipWhitespace() const {
     size_t index = 0;
     while (!stream->eof()) {
         char character = stream->peek();
@@ -125,7 +128,8 @@ size_t IfcSpfLexer::skipWhitespace() const {
     return index;
 }
 
-size_t IfcSpfLexer::skipComment() const {
+template <typename Reader>
+size_t IfcSpfLexer<Reader>::skipComment() const {
     if (stream->eof()) {
         return 0;
     }
@@ -153,7 +157,8 @@ size_t IfcSpfLexer::skipComment() const {
     return index;
 }
 
-std::string& IfcSpfLexer::getTempString() const {
+template <typename Reader>
+std::string& IfcSpfLexer<Reader>::getTempString() const {
     const size_t idx = pool_index++;
     const size_t slice = idx >> 4;
     const size_t offset = idx & 0xF;
@@ -161,6 +166,9 @@ std::string& IfcSpfLexer::getTempString() const {
     while (stringpool_.size() <= slice) {
         stringpool_.push_back(std::make_unique<std::array<std::string, 16>>());
     }
+
+    // std::wcout << "Num contexts: " << idx << std::endl;
+
     return (*stringpool_[slice])[offset];
 }
 
@@ -192,17 +200,86 @@ bool parse_float_(const char* pStart, double& val) {
 
 } // namespace
 
+namespace SWAR {
+constexpr uint32_t ONES32 = 0x01010101u;
+constexpr uint32_t HIGHS32 = 0x80808080u;
+constexpr uint64_t ONES = 0x0101010101010101ull;
+constexpr uint64_t HIGHS = 0x8080808080808080ull;
+
+constexpr uint64_t splat(unsigned char c) {
+    return ONES * c;
+}
+
+inline uint32_t has_zero_byte(uint32_t x) {
+    return (x - ONES32) & ~x & HIGHS32;
+}
+
+inline uint64_t has_zero_byte(uint64_t x) {
+    return (x - ONES) & ~x & HIGHS;
+}
+
+inline uint32_t eq_mask(uint32_t x, uint32_t c) {
+    return has_zero_byte(x ^ c);
+}
+
+inline uint64_t eq_mask(uint64_t x, uint64_t c) {
+    return has_zero_byte(x ^ c);
+}
+
+namespace chars {
+constexpr uint64_t lpar = splat('(');
+constexpr uint64_t rpar = splat(')');
+constexpr uint64_t eq = splat('=');
+constexpr uint64_t comma = splat(',');
+constexpr uint64_t semi = splat(';');
+constexpr uint64_t slash = splat('/');
+
+constexpr uint64_t space = splat(' ');
+constexpr uint64_t cr = splat('\r');
+constexpr uint64_t lf = splat('\n');
+constexpr uint64_t tab = splat('\t');
+
+constexpr uint64_t quote = splat('"');
+constexpr uint64_t dot = splat('.');
+} // namespace chars
+
+inline uint64_t has_special_char(uint64_t x) {
+    return eq_mask(x, chars::lpar) |
+           eq_mask(x, chars::rpar) |
+           eq_mask(x, chars::eq) |
+           eq_mask(x, chars::comma) |
+           eq_mask(x, chars::semi) |
+           eq_mask(x, chars::slash) |
+           eq_mask(x, chars::space) |
+           eq_mask(x, chars::cr) |
+           eq_mask(x, chars::lf) |
+           eq_mask(x, chars::tab) |
+           eq_mask(x, chars::quote) |
+           eq_mask(x, chars::dot);
+}
+
+inline uint32_t has_special_char(uint32_t x) {
+    return eq_mask(x, static_cast<uint32_t>(chars::lpar)) |
+           eq_mask(x, static_cast<uint32_t>(chars::rpar)) |
+           eq_mask(x, static_cast<uint32_t>(chars::eq)) |
+           eq_mask(x, static_cast<uint32_t>(chars::comma)) |
+           eq_mask(x, static_cast<uint32_t>(chars::semi)) |
+           eq_mask(x, static_cast<uint32_t>(chars::slash)) |
+           eq_mask(x, static_cast<uint32_t>(chars::space)) |
+           eq_mask(x, static_cast<uint32_t>(chars::cr)) |
+           eq_mask(x, static_cast<uint32_t>(chars::lf)) |
+           eq_mask(x, static_cast<uint32_t>(chars::tab)) |
+           eq_mask(x, static_cast<uint32_t>(chars::quote)) |
+           eq_mask(x, static_cast<uint32_t>(chars::dot));
+}
+
+}
+
 //
 // Returns the offset of the current Token and moves cursor to next
 //
-Token IfcSpfLexer::Next() {
+template <typename Reader>
-
+Token IfcSpfLexer<Reader>::Next() {
-    if (stream->eof()) {
-        return Token{};
-    }
-
-    while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
-    }
 
     if (stream->eof()) {
         return Token{};
@@ -211,6 +288,16 @@ Token IfcSpfLexer::Next() {
     auto pos = stream->tell();
     char character = stream->read();
 
+    if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
+        while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
+        }
+        if (stream->eof()) {
+            return Token{};
+        }
+        pos = stream->tell();
+        character = stream->read();
+    }
+
     // If the cursor is at [()=,;$*] we know token consists of single char
     if (character == '(' ||
         character == ')' ||
@@ -246,8 +333,25 @@ Token IfcSpfLexer::Next() {
         }
 
         while (!stream->eof()) {
+            if (stream->remaining() >= 8) {
+                uint64_t x = stream->peek_u64();
+                if (SWAR::has_special_char(x) == 0) {
+                    str.append(reinterpret_cast<const char*>(&x), 8);
+                    stream->increment(8);
+                    continue;
+                }
+            }
+            if (stream->remaining() >= 4) {
+                uint32_t x = stream->peek_u32();
+                if (SWAR::has_special_char(x) == 0) {
+                    str.append(reinterpret_cast<const char*>(&x), 4);
+                    stream->increment(4);
+                    continue;
+                }
+            }
+
             // Read character and increment pointer if not starting a new token
-            character = stream->peek();
+            char character = stream->peek();
             if (character == '(' ||
                 character == ')' ||
                 character == '=' ||
@@ -257,7 +361,8 @@ Token IfcSpfLexer::Next() {
                 break;
             }
             if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
-                if ((ttype == Token::Token_BINARY && character == '"') || (ttype == Token::Token_ENUMERATION && character == '.')) {
+                if ((ttype == Token::Token_BINARY && character == '"') ||
+                    (ttype == Token::Token_ENUMERATION && character == '.')) {
                     // Skip
                 } else {
                     str.push_back(character);
@@ -300,6 +405,13 @@ Token IfcSpfLexer::Next() {
     }
 }
 
+template class IfcSpfLexer<FileReader<FullBufferImpl>>;
+template class IfcSpfLexer<FileReader<PagedFileImpl>>;
+template class IfcSpfLexer<FileReader<PushedSequentialImpl>>;
+#ifdef USE_MMAP
+template class IfcSpfLexer<FileReader<MMapImpl>>;
+#endif
+
 bool Token::is_operator() {
     return type == Token_OPERATOR;
 }
@@ -458,7 +570,8 @@ std::string Token::to_string() {
 // Reads the arguments from a list of token
 // Aditionally, registers the ids (i.e. #[\d]+) in the inverse map
 //
-void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
+template <typename Reader>
+void IfcParse::impl::in_memory_file_storage::load(IfcParse::IfcSpfLexer<Reader>* tokens, std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
     Token next = tokens->Next();
 
     size_t attribute_index_within_data = 0;
@@ -473,7 +586,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
             break;
         } else if (next.is_operator('(')) {
             return_value++;
-            load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
+            load(tokens, entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
         } else {
             return_value++;
             if (next.is_identifier() && entity && entity_instance_name) {
@@ -483,7 +596,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
             if (next.is_keyword()) {
                 try {
                     const auto* decl = (schema ? schema : file->schema())->declaration_by_name(next.as_string());
-                    parse_context ps;
+                    parse_context ps(&context_pool_);
                     tokens->Next();
                     // The only case we know where a defined type contains entity
                     // instance references is IfcPropertySetDefinitionSet. For
@@ -491,7 +604,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
                     // register inverses to the host entity (and not the defined
                     // type) and to be able to actually register the references in
                     // the 2nd pass.
-                    load(entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index);
+                    load(tokens, entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index);
                     express::Base simple_type_instance(read_simple_type_instances.emplace_back(
                         ps.construct(file, entity_instance_name, *references_to_resolve, decl, std::nullopt, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index))
                     );
@@ -510,7 +623,8 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
     }
 }
 
-void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const {
+template <typename Reader>
+void IfcParse::impl::in_memory_file_storage::try_read_semicolon(IfcParse::IfcSpfLexer<Reader>* tokens) const {
     auto old_offset = tokens->stream->tell();
     Token semilocon = tokens->Next();
     if (!semilocon.is_operator(';')) {
@@ -1094,16 +1208,16 @@ IfcFile::IfcFile(const std::string& fn, bool mmap) {
 }
 
 bool IfcParse::IfcFile::initialize(const std::string& fn, bool mmap) {
-    std::unique_ptr<FileReader> s;
     if (mmap) {
-        s = std::make_unique<FileReader>(fn, FileReader::mmap_tag{});
+        MMapFileReader s(fn);
+        storage_.emplace<1>(this);
+        std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
     } else {
-        s = std::make_unique<FileReader>(fn);
+        FullBufferFileReader s(fn);
+        storage_.emplace<1>(this);
+        std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
     }
 
-    storage_.emplace<1>(this);
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&*s, schema_, max_id_, types_to_bypass_loading_);
-
     if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
         // @todo unify these names, it's already confusing enough as it stands
         byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_read_);
@@ -1124,7 +1238,7 @@ bool IfcParse::IfcFile::initialize(const std::string& path, filetype ty, bool re
         ty = guess_file_type(path);
     }
     if (ty == FT_IFCSPF) {
-        FileReader s(path);
+        FileReader<FullBufferImpl> s(path);
         storage_.emplace<1>(this);
         std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
 
@@ -1181,7 +1295,7 @@ IfcFile::IfcFile(std::istream& stream, int length)
     , max_id_(0)
     , header_(new IfcParse::IfcSpfHeader(this))
 {
-    FileReader s(FileReader::caller_fed_tag{});
+    FileReader<PushedSequentialImpl> s(caller_fed_tag{});
 
     std::string string_data;
 	string_data.resize(length);
@@ -1203,7 +1317,7 @@ IfcFile::IfcFile(void* data, int length)
     , max_id_(0),
     header_(new IfcParse::IfcSpfHeader(this))
 {
-	FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{});
+	FileReader<PushedSequentialImpl> s(std::string((char*)data, length), caller_fed_tag{});
     
     storage_.emplace<1>(this);
     std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
@@ -1215,20 +1329,6 @@ IfcFile::IfcFile(void* data, int length)
     byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
 }
 
-IfcFile::IfcFile(IfcParse::FileReader* s)
-    : schema_(nullptr)
-    , max_id_(0)
-{
-    storage_.emplace<1>(this);
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(s, schema_, max_id_, types_to_bypass_loading_);
-    good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
-    ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
-
-    byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_read_);
-    byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
-    byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
-}
-
 IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const std::string& path)
     : schema_(schema)
     , ifcroot_type_(schema_->declaration_by_name("IfcRoot"))
@@ -1260,9 +1360,127 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
     setDefaultHeaderValues();
 }
 
-bool IfcParse::InstanceStreamer::hasSemicolon() const {
+namespace {
+
+template <typename Reader>
+void read_terminal(IfcSpfLexer<Reader>& lexer, const std::string& term, bool trailing_semicolon) {
+    if (lexer.Next().as_string() != term) {
+        throw IfcException(std::string("Expected " + term));
+    }
+    if (trailing_semicolon) {
+        if (!lexer.Next().is_operator(';')) {
+            throw IfcException("Expected ;");
+        }
+    }
+}
+
+template <typename Reader>
+std::shared_ptr<InstanceData> read_header_entity(
+    IfcParse::IfcFile* file,
+    IfcParse::impl::in_memory_file_storage& storage,
+    IfcSpfLexer<Reader>& lexer,
+    IfcParse::unresolved_references& references_to_resolve,
+    const IfcParse::entity& decl) {
+    parse_context pc(&storage.context_pool_);
+    lexer.Next();
+    storage.load(&lexer, std::nullopt, nullptr, pc, -1);
+    auto result = pc.construct(file, std::nullopt, references_to_resolve, &decl, decl.attribute_count(), -1);
+    storage.context_pool_.reset();
+    return result;
+}
+
+template <typename Reader>
+void parse_header(
+    IfcParse::IfcSpfHeader& header,
+    IfcParse::impl::in_memory_file_storage& storage,
+    IfcSpfLexer<Reader>& lexer,
+    IfcParse::unresolved_references& references_to_resolve) {
+    static const char* const ISO_10303_21 = "ISO-10303-21";
+    static const char* const HEADER = "HEADER";
+
+    read_terminal(lexer, ISO_10303_21, true);
+    read_terminal(lexer, HEADER, true);
+
+    read_terminal(lexer, Header_section_schema::file_description::Class().name_uc(), false);
+    header.set_file_description(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_description::Class()));
+    if (!lexer.Next().is_operator(';')) {
+        throw IfcException("Expected ;");
+    }
+
+    read_terminal(lexer, Header_section_schema::file_name::Class().name_uc(), false);
+    header.set_file_name(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_name::Class()));
+    if (!lexer.Next().is_operator(';')) {
+        throw IfcException("Expected ;");
+    }
+
+    read_terminal(lexer, Header_section_schema::file_schema::Class().name_uc(), false);
+    header.set_file_schema(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_schema::Class()));
+    if (!lexer.Next().is_operator(';')) {
+        throw IfcException("Expected ;");
+    }
+}
+
+template <typename Reader>
+bool try_parse_header(
+    IfcParse::IfcSpfHeader& header,
+    IfcParse::impl::in_memory_file_storage& storage,
+    IfcSpfLexer<Reader>& lexer,
+    IfcParse::unresolved_references& references_to_resolve) {
+    try {
+        parse_header(header, storage, lexer, references_to_resolve);
+        return true;
+    } catch (const std::exception& e) {
+        storage.context_pool_.reset();
+        Logger::Error(e);
+        return false;
+    }
+}
+
+} // namespace
+
+template <typename Reader>
+IfcSpfHeader& IfcParse::InstanceStreamer<Reader>::ensure_header() {
+    if (header_) {
+        return *header_;
+    }
+
+    if (owner_ != nullptr) {
+        header_ = &owner_->header();
+        header_->file(owner_);
+    } else {
+        owned_header_ = std::make_unique<IfcSpfHeader>(owner_);
+        header_ = owned_header_.get();
+    }
+
+    return *header_;
+}
+
+template <typename Reader>
+void IfcParse::InstanceStreamer<Reader>::initialize_header() {
+    storage_.file = owner_;
+    storage_.schema = schema_;
+    storage_.references_to_resolve = &references_to_resolve_;
+
+    if (!lexer_ || !stream_ || !stream_->size() || stream_->eof()) {
+        return;
+    }
+
+    auto& header = ensure_header();
+    if (try_parse_header(header, storage_, *lexer_, references_to_resolve_) && header.file_schema().schema_identifiers().size() == 1) {
+        try {
+            schema_ = IfcParse::schema_by_name(header.file_schema().schema_identifiers().front());
+            good_ = file_open_status::SUCCESS;
+        } catch (const IfcParse::IfcException&) {
+        }
+    }
+
+    storage_.schema = schema_;
+}
+
+template <typename Reader>
+bool IfcParse::InstanceStreamer<Reader>::hasSemicolon() const {
     auto local_stream = stream_->clone();
-	auto local_lexer = IfcSpfLexer(&local_stream);
+    auto local_lexer = IfcSpfLexer<Reader>(&local_stream);
     Token t;
     try {
         t = local_lexer.Next();
@@ -1272,20 +1490,20 @@ bool IfcParse::InstanceStreamer::hasSemicolon() const {
     while (t.type != Token::Token_NONE) {
         if (t.is_operator(';')) {
             return true;
-		}
+        }
         try {
             t = local_lexer.Next();
         } catch (const std::out_of_range&) {
-            // This most likely happens when a page boundary is contained within a string
             break;
         }
     }
-	return false;
+    return false;
 }
 
-size_t IfcParse::InstanceStreamer::semicolonCount() const {
+template <typename Reader>
+size_t IfcParse::InstanceStreamer<Reader>::semicolonCount() const {
     auto local_stream = stream_->clone();
-    auto local_lexer = IfcSpfLexer(&local_stream);
+    auto local_lexer = IfcSpfLexer<Reader>(&local_stream);
     Token t;
     size_t count = 0;
     try {
@@ -1300,145 +1518,248 @@ size_t IfcParse::InstanceStreamer::semicolonCount() const {
         try {
             t = local_lexer.Next();
         } catch (const std::out_of_range&) {
-            // This most likely happens when a page boundary is contained within a string
             break;
         }
     }
     return count;
 }
 
-void IfcParse::InstanceStreamer::pushPage(const std::string& page)
+template <typename Reader>
-{
+void IfcParse::InstanceStreamer<Reader>::pushPage(const std::string& page) {
     stream_->pushNextPage(page);
     if (good_ == file_open_status::NO_HEADER) {
-        header_ = new IfcParse::IfcSpfHeader(lexer_);
+        initialize_header();
-        if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) {
-            try {
-                schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front());
-                good_ = file_open_status::SUCCESS;
-            } catch (const IfcParse::IfcException&) {
-            }
-        }
-        storage_.file = nullptr;
-        storage_.schema = schema_;
-        storage_.tokens = lexer_;
-        storage_.references_to_resolve = &references_to_resolve_;
     }
 }
 
-IfcParse::InstanceStreamer::InstanceStreamer(IfcParse::IfcFile* f)
+template <typename Reader>
-    : stream_(new FileReader(FileReader::caller_fed_tag{}))
+IfcParse::InstanceStreamer<Reader>::InstanceStreamer(IfcParse::IfcFile* f)
-    , lexer_(new IfcSpfLexer(stream_))
-    , token_stream_(3, Token{})
-    , schema_(nullptr)
-    , progress_(0)
-    , owner_(f)
-{
-    init_locale();
-    good_ = file_open_status::NO_HEADER;
-    storage_.file = f;
-}
-
-IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap, IfcParse::IfcFile* f)
-    : stream_(mmap ? new FileReader(fn, FileReader::mmap_tag{}) : new FileReader(fn))
-    , lexer_(new IfcSpfLexer(stream_))
-    , token_stream_(3, Token{})
-    , schema_(nullptr)
-    , progress_(0), owner_(f)
-{
-    init_locale();
-
-    good_ = file_open_status::NO_HEADER;
-    if (stream_->size() && !stream_->eof()) {
-        header_ = new IfcParse::IfcSpfHeader(lexer_);
-        if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) {
-            try {
-                schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front());
-                good_ = file_open_status::SUCCESS;
-            } catch (const IfcParse::IfcException&) {
-            }
-        }
-        storage_.file = f;
-        storage_.schema = schema_;
-        storage_.tokens = lexer_;
-        storage_.references_to_resolve = &references_to_resolve_;
-    }
-}
-
-IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length, IfcParse::IfcFile* f)
-    : stream_(new FileReader(std::string((char*) data, length), FileReader::caller_fed_tag{}))
-    , lexer_(new IfcSpfLexer(stream_))
-    , token_stream_(3, Token{})
-    , schema_(nullptr)
-    , progress_(0)
-    , owner_(f)
-{
-    init_locale();
-
-    good_ = file_open_status::NO_HEADER;
-    if (stream_->size() && !stream_->eof()) {
-        header_ = new IfcParse::IfcSpfHeader(lexer_);
-        if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) {
-            try {
-                schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front());
-                good_ = file_open_status::SUCCESS;
-            } catch (const IfcParse::IfcException&) {
-            }
-        }
-        storage_.file = f;
-        storage_.schema = schema_;
-        storage_.tokens = lexer_;
-        storage_.references_to_resolve = &references_to_resolve_;
-    }
-}
-
-IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer, IfcParse::IfcFile* f)
     : stream_(nullptr)
-    , lexer_(lexer)
     , header_(nullptr)
-    , token_stream_(3, Token{})
-    , schema_(schema)
-    , progress_(0)
     , owner_(f)
+    , token_stream_(3, Token{})
+    , schema_(nullptr)
+    , progress_(0)
 {
     init_locale();
 
+    if constexpr (std::is_same_v<Reader, FileReader<PushedSequentialImpl>>) {
+        owned_stream_ = std::make_unique<Reader>(caller_fed_tag{});
+    } else {
+        static_assert(file_reader_dependent_false_v<Reader>, "Default InstanceStreamer requires a pushed sequential reader");
+    }
+
+    stream_ = owned_stream_.get();
+    lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
+    good_ = file_open_status::NO_HEADER;
     storage_.file = f;
-    storage_.schema = schema_;
-    storage_.tokens = lexer_;
     storage_.references_to_resolve = &references_to_resolve_;
 }
 
-void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass) {
+template <typename Reader>
-    // Initialize a "C" locale for locale-independent
+IfcParse::InstanceStreamer<Reader>::InstanceStreamer(const std::string& fn, bool mmap, IfcParse::IfcFile* f)
-    // number parsing. See comment above on line 41.
+    : stream_(nullptr)
+    , header_(nullptr)
+    , owner_(f)
+    , token_stream_(3, Token{})
+    , schema_(nullptr)
+    , progress_(0) {
     init_locale();
 
-    tokens = nullptr;
+    if constexpr (std::is_same_v<Reader, FileReader<FullBufferImpl>>) {
+        (void)mmap;
+        owned_stream_ = std::make_unique<Reader>(fn);
+#ifdef USE_MMAP
+    } else if constexpr (std::is_same_v<Reader, MMapFileReader>) {
+        (void)mmap;
+        owned_stream_ = std::make_unique<Reader>(fn);
+#endif
+    } else {
+        static_assert(file_reader_dependent_false_v<Reader>, "Path-based InstanceStreamer requires a file-backed reader");
+    }
+
+    stream_ = owned_stream_.get();
+    lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
+    good_ = file_open_status::NO_HEADER;
+    initialize_header();
+}
+
+template <typename Reader>
+IfcParse::InstanceStreamer<Reader>::InstanceStreamer(void* data, int length, IfcParse::IfcFile* f)
+    : stream_(nullptr)
+    , header_(nullptr)
+    , owner_(f)
+    , token_stream_(3, Token{})
+    , schema_(nullptr)
+    , progress_(0)
+{
+    init_locale();
+
+    if constexpr (std::is_same_v<Reader, FileReader<PushedSequentialImpl>>) {
+        owned_stream_ = std::make_unique<Reader>(std::string((char*)data, length), caller_fed_tag{});
+    } else {
+        static_assert(file_reader_dependent_false_v<Reader>, "Buffer-based InstanceStreamer requires a pushed sequential reader");
+    }
+
+    stream_ = owned_stream_.get();
+    lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
+    good_ = file_open_status::NO_HEADER;
+    initialize_header();
+}
+
+template <typename Reader>
+IfcParse::InstanceStreamer<Reader>::InstanceStreamer(Reader* stream, IfcParse::IfcFile* f)
+    : stream_(stream)
+    , header_(nullptr)
+    , owner_(f)
+    , token_stream_(3, Token{})
+    , schema_(nullptr)
+    , progress_(0) {
+    init_locale();
+
+    lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
+    good_ = file_open_status::NO_HEADER;
+    initialize_header();
+}
+
+template <typename Reader>
+void IfcParse::InstanceStreamer<Reader>::bypassTypes(const std::set<std::string>& type_names) {
+    for (auto& name : type_names) {
+        try {
+            types_to_bypass_.push_back(schema_->declaration_by_name(name));
+        } catch (const IfcException&) {
+            continue;
+        }
+    }
+}
+
+template <typename Reader>
+std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> IfcParse::InstanceStreamer<Reader>::readInstance() {
+    std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> return_value;
+
+    if (yield_header_instances_ && header_ && yielded_header_instances_ < 3) {
+        if (yielded_header_instances_ == 0) {
+            return_value.emplace(
+                0,
+                &header_->file_description().declaration(),
+                header_->file_description().data_weak().lock());
+        } else if (yielded_header_instances_ == 1) {
+            return_value.emplace(
+                0,
+                &header_->file_name().declaration(),
+                header_->file_name().data_weak().lock());
+        } else if (yielded_header_instances_ == 2) {
+            return_value.emplace(
+                0,
+                &header_->file_schema().declaration(),
+                header_->file_schema().data_weak().lock());
+        }
+        yielded_header_instances_ += 1;
+        return return_value;
+    }
+
+    unsigned current_id = 0;
+    while (good_ && !lexer_->stream->eof() && !current_id) {
+        if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER &&
+            token_stream_[1].type == IfcParse::Token::Token_OPERATOR &&
+            token_stream_[1].value_char == '=' &&
+            token_stream_[2].type == IfcParse::Token::Token_KEYWORD) {
+            current_id = token_stream_[0].as_identifier();
+            const IfcParse::declaration* entity_type;
+            try {
+                entity_type = schema_->declaration_by_name(token_stream_[2].as_string());
+            } catch (const IfcException& ex) {
+                Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos));
+                current_id = 0;
+                goto advance;
+            }
+
+            if (entity_type->as_entity() == nullptr) {
+                Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos));
+                goto advance;
+            }
+
+            for (auto& ty : types_to_bypass_) {
+                if (entity_type->is(*ty)) {
+                    bypassed_instances_.push_back(current_id);
+                    current_id = 0;
+                    goto advance;
+                }
+            }
+
+            parse_context ps(&storage_.context_pool_);
+            lexer_->Next();
+            try {
+                storage_.load(lexer_.get(), current_id, entity_type->as_entity(), ps, -1);
+            } catch (const IfcInvalidTokenException& e) {
+                good_ = file_open_status::INVALID_SYNTAX;
+                Logger::Error(e);
+                break;
+            }
+
+            if (((++progress_) % 1000) == 0) {
+                std::stringstream ss;
+                ss << "\r#" << current_id;
+                Logger::Status(ss.str(), false);
+            }
+
+            auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count);
+            storage_.context_pool_.reset();
+
+            return_value.emplace(
+                (size_t)current_id,
+                entity_type,
+                data);
+        }
+    advance:
+        Token next_token;
+        try {
+            next_token = lexer_->Next();
+        } catch (const IfcException& e) {
+            Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated");
+        } catch (...) {
+            Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
+        }
+
+        if (!lexer_->stream->eof() && !next_token) {
+            good_ = file_open_status::INVALID_SYNTAX;
+            break;
+        }
+
+        token_stream_.push_back(next_token);
+    }
+
+    stream_->dropPages();
+    lexer_->resetPool();
+
+    return return_value;
+}
+
+template <typename Reader>
+void IfcParse::impl::in_memory_file_storage::read_from_stream(Reader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass) {
+    init_locale();
+
+    schema = nullptr;
 
     if (!s->size() || s->eof()) {
-        // @todo set good on parent file
         good_ = file_open_status::READ_ERROR;
         return;
     }
 
-    tokens = new IfcSpfLexer(s);
-
     std::vector<std::string> schemas;
 
-    file->header().file(file);
+    InstanceStreamer<Reader> streamer(s, file);
+    streamer.yieldHeaderInstances(false);
 
-    if (file->header().tryRead()) {
+    if (const auto* header = streamer.header()) {
         try {
-            schemas = file->header().file_schema().schema_identifiers();
+            schemas = header->file_schema().schema_identifiers();
         } catch (...) {
-            // Purposely empty catch block
         }
-    } else {
-        good_ = file_open_status::NO_HEADER;
     }
 
-    if (schemas.size() == 1) {
+    schema = streamer.schema();
+    if (schema == nullptr && schemas.size() == 1) {
         try {
             schema = IfcParse::schema_by_name(schemas.front());
         } catch (const IfcParse::IfcException& e) {
@@ -1448,28 +1769,28 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
     }
 
     if (schema == nullptr) {
+        if (schemas.empty()) {
+            good_ = streamer.status();
+        } else {
+            good_ = file_open_status::UNSUPPORTED_SCHEMA;
+        }
         Logger::Message(Logger::LOG_ERROR, "No support for file schema encountered (" + boost::algorithm::join(schemas, ", ") + ")");
         return;
     }
 
     auto ifcroot_type_ = schema->declaration_by_name("IfcRoot");
-
-	InstanceStreamer streamer(schema, tokens, file);
     streamer.bypassTypes(typed_to_bypass);
 
     Logger::Status("Scanning file...");
 
     while (streamer) {
-
         auto inst = streamer.readInstance();
 
         if (!inst) {
-            // No more instances to read
             break;
-		}
+        }
 
         auto current_id = std::get<0>(*inst);
-
         express::Base instance(std::get<2>(*inst));
 
         if (instance.declaration().is(*ifcroot_type_)) {
@@ -1487,10 +1808,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
         }
 
         const IfcParse::declaration* ty = &instance.declaration();
-
+        bytype_excl_[ty].push_back(instance);
-        {
-            bytype_excl_[ty].push_back(instance);
-        }
 
         if (byid_.find(current_id) != byid_.end()) {
             std::stringstream ss;
@@ -1499,27 +1817,15 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
         }
 
         byid_.insert({(uint32_t)current_id, std::get<2>(*inst)});
-        max_id = (std::max)(max_id, (unsigned int) current_id);
+        max_id = (std::max)(max_id, (unsigned int)current_id);
     }
 
-	good_ = streamer.status();
+    good_ = streamer.status();
-	byref_excl_ = streamer.inverses();
+    byref_excl_ = streamer.inverses();
-    
-	// Move the storage of simple type instances so that they are retained during the lifetime of the file
     read_simple_type_instances = streamer.stealInstances();
 
-    // Set file ownership on simple type instances, so that when adding them to other files, proper copies are created
-    /*
-    // @todo double check whether file ownership is property set earlier on
-    for (auto& inst : read_simple_type_instances) {
-        inst->file_ = file;
-    }
-    */
-
     Logger::Status("\rDone scanning file   ");
 
-    delete tokens;
-
     if (good_ != file_open_status::SUCCESS) {
         return;
     }
@@ -1639,6 +1945,12 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
     Logger::Status("Done resolving references");
 }
 
+template void IfcParse::impl::in_memory_file_storage::read_from_stream(FileReader<FullBufferImpl>* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
+template void IfcParse::impl::in_memory_file_storage::read_from_stream(FileReader<PushedSequentialImpl>* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
+#ifdef USE_MMAP
+template void IfcParse::impl::in_memory_file_storage::read_from_stream(MMapFileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
+#endif
+
 void IfcFile::recalculate_id_counter() {
     /*
     // @todo

src/ifcparse/IfcParse.h

@@ -48,9 +48,10 @@ extern const char *IFCOPENSHELL_VERSION;
 namespace IfcParse {
 
 /// A stream of tokens to be read from a FileReader.
+template <typename Reader>
 class IFC_PARSE_API IfcSpfLexer {
   private:
-    IfcCharacterDecoder* decoder_;
+    IfcCharacterDecoder<Reader>* decoder_;
     size_t skipWhitespace() const;
     size_t skipComment() const;
 
@@ -68,9 +69,9 @@ class IFC_PARSE_API IfcSpfLexer {
         }
     }
 
-    FileReader* stream;
+    Reader* stream;
     // IfcFile* file;
-    IfcSpfLexer(FileReader* stream);
+    IfcSpfLexer(Reader* stream);
     Token Next();
     ~IfcSpfLexer();
     // void TokenString(size_t offset, std::string& result);

src/ifcparse/IfcSchema.h

@@ -28,6 +28,7 @@
 #include <iterator>
 #include <string>
 #include <vector>
+#include <optional>
 
 // Forward declarations
 class InstanceData;
@@ -297,6 +298,7 @@ class IFC_PARSE_API entity : public declaration {
     std::vector<const entity*> subtypes_;
 
     std::vector<const attribute*> attributes_;
+    mutable std::optional<std::vector<const attribute*>> all_attributes_;
     std::vector<bool> derived_;
 
     std::vector<const inverse_attribute*> inverse_attributes_;
@@ -354,15 +356,17 @@ class IFC_PARSE_API entity : public declaration {
     const std::vector<const attribute*>& attributes() const { return attributes_; }
     const std::vector<bool>& derived() const { return derived_; }
 
-    const std::vector<const attribute*> all_attributes() const {
+    const std::vector<const attribute*>& all_attributes() const {
-        std::vector<const attribute*> attrs;
+        if (!all_attributes_) {
-        attrs.reserve(derived_.size());
+            auto& attrs = all_attributes_.emplace();
-        if (supertype_ != nullptr) {
+            attrs.reserve(derived_.size());
-            const std::vector<const attribute*> supertype_attrs = supertype_->all_attributes();
+            if (supertype_ != nullptr) {
-            std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs));
+                const std::vector<const attribute*> supertype_attrs = supertype_->all_attributes();
+                std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs));
+            }
+            std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs));
         }
-        std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs));
+        return *all_attributes_;
-        return attrs;
     }
 
     const std::vector<const inverse_attribute*> all_inverse_attributes() const {

src/ifcparse/IfcSpfHeader.cpp

@@ -1,26 +1,6 @@
-/********************************************************************************
- *                                                                              *
- * This file is part of IfcOpenShell.                                           *
- *                                                                              *
- * IfcOpenShell is free software: you can redistribute it and/or modify         *
- * it under the terms of the Lesser GNU General Public License as published by  *
- * the Free Software Foundation, either version 3.0 of the License, or          *
- * (at your option) any later version.                                          *
- *                                                                              *
- * IfcOpenShell is distributed in the hope that it will be useful,              *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of               *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
- * Lesser GNU General Public License for more details.                          *
- *                                                                              *
- * You should have received a copy of the Lesser GNU General Public License     *
- * along with this program. If not, see <http://www.gnu.org/licenses/>.         *
- *                                                                              *
- ********************************************************************************/
-
 #include "IfcSpfHeader.h"
 
 #include "IfcFile.h"
-#include "IfcLogger.h"
 
 static const char* const ISO_10303_21 = "ISO-10303-21";
 static const char* const HEADER = "HEADER";
@@ -30,116 +10,31 @@ static const char* const DATA = "DATA";
 using namespace IfcParse;
 
 namespace {
-    std::shared_ptr<InstanceData> read_from_spf_file(IfcParse::IfcFile* file, IfcParse::impl::in_memory_file_storage* storage, const IfcParse::entity* decl) {
-        if (storage != nullptr) {
-            parse_context pc;
-            storage->tokens->Next();
-            storage->load(-1, nullptr, pc, -1);
-            return pc.construct(file, std::nullopt, *storage->references_to_resolve, decl, decl->as_entity()->attribute_count(), -1);
-        } else {
-            // std::unreachable();
-            return nullptr;
-        }
-    }
-} // namespace
 
-void IfcSpfHeader::readSemicolon() {
+std::shared_ptr<InstanceData> make_header_entity(IfcParse::IfcFile* file, const IfcParse::entity& decl) {
-    if (storage_ != nullptr) {
+    const bool in_memory = file == nullptr || std::visit([](auto& storage) {
-        if (!storage_->tokens->Next().is_operator(';')) {
+        return std::is_same_v<std::decay_t<decltype(storage)>, IfcParse::impl::in_memory_file_storage>;
-            throw IfcException(std::string("Expected ;"));
-        }
-    } else {
-        // std::unreachable();
-    }
-}
-
-void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) {
-    if (storage_ != nullptr) {
-        if (storage_->tokens->Next().as_string() != term) {
-            throw IfcException(std::string("Expected " + term));
-        }
-        if (trail == TRAILING_SEMICOLON) {
-            readSemicolon();
-        }
-    } else {
-        // std::unreachable();
-    }
-}
-
-IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file)
-    : file_(file)
-{
-    Header_section_schema::get_schema();
-
-    // @todo This might still not work in IfcFile's uninitialized mode
-     storage_ = std::visit([this](auto& m) -> decltype(storage_) {
-        if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
-            return &m;
-        }
-        return nullptr;
     }, file->storage_);
 
-    const bool in_memory = storage_ != nullptr;
-
     if (in_memory) {
-        header_entities_[0] = std::make_shared<InstanceData>(file, &Header_section_schema::file_description::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_description::Class().attribute_count()));
+        return std::make_shared<InstanceData>(file, &decl, 0, in_memory_attribute_storage(decl.attribute_count()));
-        header_entities_[1] = std::make_shared<InstanceData>(file, &Header_section_schema::file_name::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_name::Class().attribute_count()));
-        header_entities_[2] = std::make_shared<InstanceData>(file, &Header_section_schema::file_schema::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_schema::Class().attribute_count()));
-    } else {
-        header_entities_[0] = std::make_shared<InstanceData>(file, &Header_section_schema::file_description::Class(), 0, rocks_db_attribute_storage{});
-        header_entities_[1] = std::make_shared<InstanceData>(file, &Header_section_schema::file_name::Class(), 0, rocks_db_attribute_storage{});
-        header_entities_[2] = std::make_shared<InstanceData>(file, &Header_section_schema::file_schema::Class(), 0, rocks_db_attribute_storage{});
     }
+
+    return std::make_shared<InstanceData>(file, &decl, 0, rocks_db_attribute_storage{});
 }
 
-IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcSpfLexer* lexer)
+} // namespace
-{
+
+IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file)
+    : file_(file) {
     Header_section_schema::get_schema();
 
-	storage_ = new impl::in_memory_file_storage;
+    header_entities_[0] = make_header_entity(file_, Header_section_schema::file_description::Class());
-	storage_->tokens = lexer;
+    header_entities_[1] = make_header_entity(file_, Header_section_schema::file_name::Class());
-    file_ = nullptr;
+    header_entities_[2] = make_header_entity(file_, Header_section_schema::file_schema::Class());
 }
 
-IfcParse::IfcSpfHeader::~IfcSpfHeader() {
+IfcParse::IfcSpfHeader::~IfcSpfHeader() = default;
-}
-
-void IfcSpfHeader::read() {
-    readTerminal(ISO_10303_21, TRAILING_SEMICOLON);
-    readTerminal(HEADER, TRAILING_SEMICOLON);
-
-    // | The header section of every exchange structure shall contain one
-    // | instance of each of the following entities: file_description, file_name,
-    // | and file_schema, and they shall appear in that order. Instances of
-    // | file_population, section_language and section_context may appear after
-    // | file_schema. If instances of user-defined header section entities are
-    // | present, they shall appear after the header section entity instances
-    // | defined in this section.
-    //
-    // ISO 10303-21 Second edition 2002-01-15 p. 16
-
-    readTerminal(Header_section_schema::file_description::Class().name_uc(), NONE);
-    header_entities_[0] = read_from_spf_file(file_, storage_, &Header_section_schema::file_description::Class());
-    readSemicolon();
-
-    readTerminal(Header_section_schema::file_name::Class().name_uc(), NONE);
-    header_entities_[1] = read_from_spf_file(file_, storage_, &Header_section_schema::file_name::Class());
-    readSemicolon();
-
-    readTerminal(Header_section_schema::file_schema::Class().name_uc(), NONE);
-    header_entities_[2] = read_from_spf_file(file_, storage_, &Header_section_schema::file_schema::Class());
-    readSemicolon();
-}
-
-bool IfcSpfHeader::tryRead() {
-    try {
-        read();
-        return true;
-    } catch (const std::exception& e) {
-        Logger::Error(e);
-        return false;
-    }
-}
 
 void IfcSpfHeader::write(std::ostream& out) const {
     out << ISO_10303_21 << ";"
@@ -163,15 +58,18 @@ void IfcSpfHeader::write(std::ostream& out) const {
 
 void IfcParse::IfcSpfHeader::file(IfcParse::IfcFile* file) {
     file_ = file;
-    if (file != nullptr) {
+}
-        storage_ = std::visit([this](auto& m) -> decltype(storage_) {
+
-            if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
+void IfcParse::IfcSpfHeader::set_file_description(const std::shared_ptr<InstanceData>& data) {
-                return &m;
+    header_entities_[0] = data;
-            }
+}
-            return nullptr;
+
-        },
+void IfcParse::IfcSpfHeader::set_file_name(const std::shared_ptr<InstanceData>& data) {
-        file_->storage_);
+    header_entities_[1] = data;
-    }
+}
+
+void IfcParse::IfcSpfHeader::set_file_schema(const std::shared_ptr<InstanceData>& data) {
+    header_entities_[2] = data;
 }
 
 const Header_section_schema::file_description IfcParse::IfcSpfHeader::file_description() const {

src/ifcparse/IfcSpfHeader.h

@@ -23,7 +23,6 @@
 #include "ifc_parse_api.h"
 #include "InstanceData.h"
 #include "Header_section_schema.h"
-#include "storage.h"
 
 namespace IfcParse {
 
@@ -32,39 +31,22 @@ class IfcFile;
 class IFC_PARSE_API IfcSpfHeader {
   private:
     IfcFile* file_;
-	IfcParse::impl::in_memory_file_storage* storage_ = nullptr;
 
     std::array<std::shared_ptr<InstanceData>, 3> header_entities_;
 
-    /*
-    mutable Header_section_schema::file_description* file_description_;
-    mutable Header_section_schema::file_name* file_name_;
-    mutable Header_section_schema::file_schema* file_schema_;
-    */ 
-    void readSemicolon();
-    enum Trail {
-        TRAILING_SEMICOLON,
-        NONE
-    };
-    void readTerminal(const std::string& term, Trail trail);
-
   public:
     explicit IfcSpfHeader(IfcParse::IfcFile* file);
-    explicit IfcSpfHeader(IfcParse::IfcSpfLexer* lexer);
-
     ~IfcSpfHeader();
 
-    // IfcParse::IfcFile* file() { return file_; }
-    // void file(IfcParse::IfcFile* file);
-
-    void read();
-    bool tryRead();
-
     void write(std::ostream& out) const;
 
     IfcParse::IfcFile* file() { return file_; }
     void file(IfcParse::IfcFile* file);
 
+    void set_file_description(const std::shared_ptr<InstanceData>& data);
+    void set_file_name(const std::shared_ptr<InstanceData>& data);
+    void set_file_schema(const std::shared_ptr<InstanceData>& data);
+
     const Header_section_schema::file_description file_description() const;
     const Header_section_schema::file_name file_name() const;
     const Header_section_schema::file_schema file_schema() const;

src/ifcparse/storage.h

@@ -131,8 +131,8 @@ namespace IfcParse {
     typedef std::list<std::pair<MutableAttributeValue, std::variant<reference_or_simple_type, std::vector<reference_or_simple_type>, std::vector<std::vector<reference_or_simple_type>>>>> unresolved_references;
 
     class IfcFile;
+    template <typename Reader>
     class IfcSpfLexer;
-    class FileReader;
 
     struct Token {
         enum TokenType {
@@ -202,15 +202,36 @@ namespace IfcParse {
         }
     };
 
+    struct parse_context;
+    struct parse_context_pool;
+
+    struct parse_context_handle {
+        parse_context_pool* pool = nullptr;
+        uint32_t index = 0;
+
+        parse_context& get() const;
+        parse_context* operator->() const;
+        parse_context& operator*() const;
+        explicit operator bool() const { return pool != nullptr; }
+    };
+
     struct parse_context {
-        std::list<
+        std::vector<
             std::variant<
             express::Base,
             Token,
-            parse_context*
+            parse_context_handle
             >> tokens_;
 
-        parse_context() {};
+        void reset() {
+            tokens_.clear();
+        }
+
+        parse_context_pool* pool_;
+
+        parse_context(parse_context_pool* pool) : pool_(pool) {
+            tokens_.reserve(16);
+        };
         ~parse_context();
 
         parse_context(const parse_context&) = delete;
@@ -228,16 +249,47 @@ namespace IfcParse {
         std::shared_ptr<InstanceData> construct(IfcParse::IfcFile* owner, std::optional<size_t> name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, std::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count=true);
     };
 
+    struct parse_context_pool {
+        std::vector<parse_context> nodes_;
+        uint32_t used_ = 0;
+
+        void reset() { used_ = 0; }
+
+        parse_context_handle make() {
+            if (used_ == nodes_.size()) {
+                nodes_.emplace_back(this);
+            }
+            auto idx = used_++;
+            nodes_[idx].reset();
+            return {this, idx};
+        }
+
+        parse_context& get(uint32_t index) {
+            return nodes_[index];
+        }
+    };
+
+    inline parse_context& parse_context_handle::get() const {
+        return pool->get(index);
+    }
+
+    inline parse_context* parse_context_handle::operator->() const {
+        return &pool->get(index);
+    }
+
+    inline parse_context& parse_context_handle::operator*() const {
+        return pool->get(index);
+    }
+
     namespace impl {
         struct IFC_PARSE_API in_memory_file_storage {
+            IfcParse::parse_context_pool context_pool_;
+
             std::vector<std::shared_ptr<InstanceData>> read_simple_type_instances;
             std::vector<std::shared_ptr<InstanceData>> steal_instances() {
                 return read_simple_type_instances;
             }
 
-            IfcParse::IfcSpfLexer* tokens;
-            // IfcParse::FileReader* stream;
-
             // Either one of these needs to be set
             IfcParse::IfcFile* file;
             const IfcParse::schema_definition* schema;
@@ -258,7 +310,7 @@ namespace IfcParse {
             typedef std::map<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
             typedef entity_instance_by_name_t::iterator iterator;
 
-            in_memory_file_storage(IfcParse::IfcFile* f = nullptr) : tokens(nullptr), file(f), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr<InstanceData>& d) { return express::Base(d); }) {};
+            in_memory_file_storage(IfcParse::IfcFile* f = nullptr) : file(f), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr<InstanceData>& d) { return express::Base(d); }) {};
             in_memory_file_storage(const in_memory_file_storage&) = delete;
             in_memory_file_storage(const in_memory_file_storage&&) = delete;
 
@@ -303,13 +355,16 @@ namespace IfcParse {
             entity_instance_by_guid_t byguid_;
             entity_instance_by_name_t byid_read_;
 
-            void load(std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
+            template <typename Reader>
-            void try_read_semicolon() const;
+            void load(IfcParse::IfcSpfLexer<Reader>* tokens, std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
+            template <typename Reader>
+            void try_read_semicolon(IfcParse::IfcSpfLexer<Reader>* tokens) const;
 
             void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
             void unregister_inverse(unsigned, const IfcParse::entity* from_entity, const express::Base&, int attribute_index);
 
-            void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
+            template <typename Reader>
+            void read_from_stream(Reader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
 
             file_open_status good_ = file_open_status::SUCCESS;
 

src/ifcwrap/IfcParseWrapper.i

@@ -31,10 +31,8 @@
 %ignore IfcParse::IfcFile::byref_excl_;
 %ignore IfcParse::IfcFile::types_to_bypass_loading_;
 
-%ignore IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer);
+%ignore IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>::readInstance;
-
+%ignore IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>::stealInstances;
-%ignore IfcParse::InstanceStreamer::readInstance;
-%ignore IfcParse::InstanceStreamer::stealInstances;
 
 %ignore express::Entity;
 %ignore express::Select;
@@ -54,9 +52,6 @@
 %ignore IfcParse::FileSchema::FileSchema;
 %ignore IfcParse::IfcFile::tokens;
 
-%ignore IfcParse::IfcSpfHeader::IfcSpfHeader(IfcSpfLexer*);
-%ignore IfcParse::IfcSpfHeader::lexer;
-%ignore IfcParse::IfcSpfHeader::stream;
 %ignore IfcParse::IfcSpfHeader::file_description;
 %ignore IfcParse::IfcSpfHeader::file_name;
 %ignore IfcParse::IfcSpfHeader::file_schema;
@@ -922,6 +917,7 @@ object = custom_base
 %}
 
 %include "../ifcparse/IfcFile.h"
+%template(InstanceStreamer) IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>;
 
 %pythoncode %{
 ### hack hack hack
@@ -977,10 +973,10 @@ object = _old_object
 		return f;
 	}
 
-	IfcParse::InstanceStreamer* stream_from_string(const std::string& data) {
+	IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>* stream_from_string(const std::string& data) {
 		char* copiedData = new char[data.length()];
 		memcpy(copiedData, data.c_str(), data.length());
-		return new IfcParse::InstanceStreamer((void *)copiedData, data.length());
+		return new IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>((void *)copiedData, data.length());
 	}
 
 	const char* version() {
@@ -1247,7 +1243,7 @@ object = _old_object
 	}
 %}
 
-%extend IfcParse::InstanceStreamer {
+%extend IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>> {
 	PyObject* readInstancePy(bool type_as_declaration_instance=false) {
 		auto simply_type_to_dictionary = [&](const express::Base& t) -> PyObject* {
 			const auto& nm = t.declaration().name();