Storage rework WIP

src/ifcparse/FileReader.cpp

@@ -0,0 +1,312 @@
+#include "FileReader.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <list>
+#include <stdexcept>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+#include <deque>
+
+// Boost iostreams mmap
+#include <boost/iostreams/device/mapped_file.hpp>
+
+
+
+namespace {
+
+#if defined(_WIN32)
+    inline void file_seek_abs(FILE* f, std::uint64_t off) {
+        if (_fseeki64(f, static_cast<long long>(off), SEEK_SET) != 0)
+            throw std::runtime_error("fseek failed");
+    }
+#else
+    inline void file_seek_abs(FILE* f, std::uint64_t off) {
+        if (fseeko(f, static_cast<off_t>(off), SEEK_SET) != 0)
+            throw std::runtime_error("fseeko failed");
+    }
+#endif
+
+} // namespace
+
+// ===================== Concrete backends =====================
+
+using namespace IfcParse;
+
+struct FullBufferImpl final : FileReader::Impl {
+    std::vector<char> buf_;
+    explicit FullBufferImpl(const std::string& fn) {
+        std::ifstream ifs(fn, std::ios::binary);
+        if (!ifs) throw std::runtime_error("Failed to open: " + fn);
+        ifs.seekg(0, std::ios::end);
+        const std::streamsize sz = ifs.tellg();
+        ifs.seekg(0, std::ios::beg);
+        buf_.resize(static_cast<size_t>(sz));
+        if (sz > 0 && !ifs.read(buf_.data(), sz)) {
+            throw std::runtime_error("Failed to read file into buffer");
+        }
+    }
+    size_t size() const override { return buf_.size(); }
+    char get(size_t pos) const override {
+        if (pos >= buf_.size()) throw std::out_of_range("get out of range");
+        return buf_[pos];
+    }
+};
+
+struct PagedFileImpl final : FileReader::Impl {
+    std::string fn_;
+    FILE* fp_ = nullptr;
+    size_t file_size_ = 0;
+    size_t page_size_ = 4096;
+
+    // LRU cache
+    size_t capacity_ = 8;
+    mutable std::list<size_t> lru_;  // most recent at front
+    struct Entry {
+        FileReader::Page page;
+        std::list<size_t>::iterator it;
+    };
+    mutable std::unordered_map<size_t, Entry> map_;
+
+    PagedFileImpl(const std::string& fn, size_t page_size, size_t cap)
+        : fn_(fn), page_size_(std::max<size_t>(512, page_size)), capacity_(std::max<size_t>(2, cap)) {
+        namespace fs = std::filesystem;
+        if (!fs::exists(fn_)) throw std::runtime_error("File not found: " + fn_);
+        file_size_ = static_cast<size_t>(fs::file_size(fn_));
+        fp_ = std::fopen(fn_.c_str(), "rb");
+        if (!fp_) throw std::runtime_error("Failed to fopen: " + fn_);
+    }
+
+    ~PagedFileImpl() override {
+        if (fp_) std::fclose(fp_);
+        fp_ = nullptr;
+    }
+
+    size_t size() const override { return file_size_; }
+
+    char get(size_t pos) const override {
+        if (pos >= file_size_) throw std::out_of_range("get out of range");
+        const size_t pidx = pos / page_size_;
+        const FileReader::Page& p = fetch_page_(pidx);
+        const size_t off = pos % page_size_;
+        if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes");
+        // Opportunistic read-ahead for sequential scans
+        // if (off + 1 == p.data.size()) (void)try_prefetch_(pidx + 1);
+        return p.data[off];
+    }
+
+private:
+    const FileReader::Page& fetch_page_(size_t idx) const {
+        auto it = map_.find(idx);
+        if (it != map_.end()) {
+            touch_(it);
+            return it->second.page;
+        }
+        // Load page from disk using persistent FILE*
+        FileReader::Page pg;
+        pg.data.resize(page_size_);
+        const size_t begin = idx * page_size_;
+        const size_t avail = std::min(page_size_, file_size_ - begin);
+
+        file_seek_abs(fp_, begin);
+        if (avail > 0) {
+            const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
+            if (nread != avail) throw std::runtime_error("Short fread on page");
+        }
+		pg.data.resize(avail); // trim to actual size
+
+        // Insert into LRU
+        if (map_.size() >= capacity_) evict_();
+        lru_.push_front(idx);
+        auto lit = lru_.begin();
+        auto [emplaced_it, ok] = map_.emplace(idx, Entry{ std::move(pg), lit });
+        (void)ok;
+        return emplaced_it->second.page;
+    }
+
+    /*
+    bool try_prefetch_(size_t idx) const {
+        if (idx * page_size_ >= file_size_) return false;
+        if (map_.find(idx) != map_.end()) return true;
+        if (map_.size() + 1 > capacity_) return false;
+        (void)fetch_page_(idx);
+        return true;
+    }
+    */
+
+    void touch_(typename std::unordered_map<size_t, Entry>::iterator it) const {
+        lru_.erase(it->second.it);
+        lru_.push_front(it->first);
+        it->second.it = lru_.begin();
+    }
+
+    void evict_() const {
+        if (lru_.empty()) return;
+        const size_t victim = lru_.back();
+        lru_.pop_back();
+        map_.erase(victim);
+    }
+};
+
+struct MMapImpl final : FileReader::Impl {
+    boost::iostreams::mapped_file_source map_;
+    size_t size_ = 0;
+
+    explicit MMapImpl(const std::string& fn) {
+        namespace fs = std::filesystem;
+        if (!fs::exists(fn)) throw std::runtime_error("File not found: " + fn);
+        size_ = static_cast<size_t>(fs::file_size(fn));
+        if (size_ == 0) return; // empty file: map_ stays closed
+        map_.open(fn);
+        if (!map_.is_open()) throw std::runtime_error("Failed to open mapped_file_source");
+        size_ = static_cast<size_t>(map_.size());
+    }
+
+    size_t size() const override { return size_; }
+
+    char get(size_t pos) const override {
+        if (pos >= size_) throw std::out_of_range("get out of range");
+        return map_.data()[pos];
+    }
+};
+
+/// User-pushed sequential backend with an arbitrary-length queue of future pages.
+/// We keep a deque of pages; when reads move forward, we drop fully-consumed
+/// pages from the front to release memory.
+struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentialImpl>, FileReader::Impl {
+	// Deque of pages, front is earliest in file.
+    std::deque<FileReader::Page> pages_;
+    // total bytes in dropped pages
+	size_t discarded_page_bytes_ = 0;
+
+    size_t size() const override {
+        size_t n = discarded_page_bytes_;
+		for (auto& pg : pages_) n += pg.data.size();
+		return n;
+    }
+
+    // Drop fully-consumed pages so pos is guaranteed to be within the first page
+    void drop_consumed_up_to(size_t pos) {
+        while (!pages_.empty()) {
+            if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
+				discarded_page_bytes_ += pages_.front().data.size();
+                pages_.pop_front();
+            } else {
+                break;
+            }
+        }
+    }
+
+    char get(size_t pos) const override {
+        auto self = const_cast<PushedSequentialImpl*>(this);
+        if (this->shared_from_this().use_count() == 2) {
+            // only drop pages when there is only one active client.
+            // NB this->shared_from_this() increases count by 1
+            self->drop_consumed_up_to(pos);
+        }
+
+        const size_t avail_end = size();
+        if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet");
+
+		pos -= discarded_page_bytes_;
+
+		size_t page_start = 0;
+        for (const auto& pg : pages_) {
+            if (pos < page_start + pg.data.size()) {
+                const size_t off = pos - page_start;
+                return pg.data[off];
+            } else {
+				page_start += pg.data.size();
+            }
+        }
+
+        throw std::out_of_range("pushed backend: internal inconsistency");
+    }
+
+    void push_next_page(const std::string& data) override {
+        FileReader::Page p; p.data.assign(data.data(), data.data() + data.size());
+        pages_.push_back(std::move(p));
+    }
+};
+
+// ===================== FileReader public API =====================
+
+IfcParse::FileReader::FileReader(const std::string& fn)
+    : cursor_(0)
+{
+	impl_ = std::make_shared<FullBufferImpl>(fn);
+}
+
+IfcParse::FileReader::FileReader(const std::string& fn, const mmap_tag&)
+    : cursor_(0)
+{
+	impl_ = std::make_shared<MMapImpl>(fn);
+}
+
+IfcParse::FileReader::FileReader(const caller_fed_tag&)
+    : cursor_(0)
+{
+	impl_ = std::make_shared<PushedSequentialImpl>();
+}
+
+IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&)
+{
+    impl_ = std::make_shared<PushedSequentialImpl>();
+	impl_->push_next_page(content);
+}
+
+IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
+    : cursor_(0)
+{
+	impl_ = std::make_shared<PagedFileImpl>(fn, page_size, page_capacity);
+}
+
+FileReader FileReader::clone() const {
+    FileReader c(*this);
+    c.cursor_ = this->cursor_;
+    return c;
+}
+
+void FileReader::seek(size_t pos) {
+    if (pos > impl_->size()) throw std::out_of_range("seek out of range");
+    cursor_ = pos;
+}
+
+size_t FileReader::tell() const { return cursor_; }
+
+size_t FileReader::size() const { return impl_->size(); }
+
+char FileReader::peek() const {
+    if (cursor_ >= impl_->size()) throw std::out_of_range("peek at EOF");
+    return impl_->get(cursor_);
+}
+
+void FileReader::increment(size_t n) {
+    if (cursor_ + n > impl_->size()) throw std::out_of_range("increment past EOF");
+    cursor_ += n;
+}
+
+void IfcParse::FileReader::push_next_page(const std::string& data)
+{
+    impl_->push_next_page(data);
+}
+
+bool IfcParse::FileReader::eof() const
+{
+    return cursor_ >= impl_->size();
+}
+
+char IfcParse::FileReader::read()
+{
+	auto c = peek();
+	increment(1);
+	return c;
+}
+
+char IfcParse::FileReader::get(size_t offset) const
+{
+	return impl_->get(offset);
+}

src/ifcparse/FileReader.h

@@ -0,0 +1,180 @@
+/********************************************************************************
+ *                                                                              *
+ * This file is part of IfcOpenShell.                                           *
+ *                                                                              *
+ * IfcOpenShell is free software: you can redistribute it and/or modify         *
+ * it under the terms of the Lesser GNU General Public License as published by  *
+ * the Free Software Foundation, either version 3.0 of the License, or          *
+ * (at your option) any later version.                                          *
+ *                                                                              *
+ * IfcOpenShell is distributed in the hope that it will be useful,              *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of               *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
+ * Lesser GNU General Public License for more details.                          *
+ *                                                                              *
+ * You should have received a copy of the Lesser GNU General Public License     *
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.         *
+ *                                                                              *
+ ********************************************************************************/
+
+/*********************************************************************************
+ *                                                                               *
+ * Reads a file and provides functions to access its                             *
+ * contents randomly and character by character                                  *
+ *                                                                               *
+ ********************************************************************************/
+
+#ifndef IFCSPFSTREAM_H
+#define IFCSPFSTREAM_H
+
+#include "ifc_parse_api.h"
+
+/*
+#include <string>
+
+#ifdef USE_MMAP
+#include <boost/iostreams/device/mapped_file.hpp>
+#endif
+
+namespace IfcParse {
+/// The FileReader class represents a ISO 10303-21 IFC-SPF file in memory.
+/// The file is interpreted as a sequence of tokens which are lazily
+/// interpreted only when requested.
+class IFC_PARSE_API FileReader {
+  private:
+#ifdef USE_MMAP
+    boost::iostreams::mapped_file_source mfs;
+#endif
+    FILE* stream_;
+    const char* buffer_;
+    size_t ptr_;
+    size_t len_;
+	size_t buf_size_;
+    size_t ptr_offset_ = 0;
+
+  public:
+    bool valid;
+    bool eof;
+    size_t size;
+
+    FileReader(const std::string& path, bool mmap = false, size_t buf_size = 0);
+    FileReader(std::istream& stream, int length);
+    FileReader(void* data, int length);
+    ~FileReader();
+    /// Returns the character at the cursor
+    char peek();
+    /// Returns the character at specified offset
+    char Read(size_t offset);
+    /// Increment the file cursor and reads new page if necessary
+    void increment();
+    void Close();
+    /// Moves the file cursor to an arbitrary offset in the file
+    void seek(size_t offset);
+    /// Returns the cursor position
+    size_t Tell() const;
+
+    bool is_eof_at(size_t) const;
+    void increment_at(size_t&);
+    char peek_at(size_t);
+
+	operator bool() const { return valid && !eof; }
+};
+} // namespace IfcParse
+
+#endif
+*/
+
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <string>
+#include <variant>
+#include <stdexcept>
+#include <vector>
+
+namespace IfcParse {
+
+/// \brief Read-only file accessor that supports four backends:
+///  - full-buffer in RAM
+///  - paged (LRU-cached with capacity)
+///  - memory-mapped via boost::iostreams
+///  - user-pushed sequential pages (caller feeds pages intended for streaming reads in WASM)
+class IFC_PARSE_API FileReader {
+public:
+    struct Page { std::vector<char> data; };
+
+    /// \brief Tag to choose memory-mapped backend via boost::iostreams::mapped_file_source.
+    struct mmap_tag {};
+    /// \brief Tag to choose user-pushed sequential pages.
+    struct caller_fed_tag {};
+
+    /// \brief Construct a FileReader.
+    /// \param fn File path.
+    /// \param maybe_mmaped_or_chunked
+    ///        - std::nullopt: full-buffer mode (entire file loaded into memory)
+    ///        - size_t: paged mode with this page size in bytes (e.g., 4096)
+    ///        - mmap_tag: memory-mapped mode using boost::iostreams::mapped_file_source
+    /// \param page_cache_capacity Number of pages to keep in the LRU when in paged mode.
+    FileReader(const std::string& fn);
+
+    FileReader(const std::string& fn, const mmap_tag&);
+
+    FileReader(const caller_fed_tag&);
+    FileReader(const std::string& content, const caller_fed_tag&);
+
+    FileReader(const std::string& fn, size_t page_size, size_t page_capacity);
+
+    /// \brief Copy-construct a new reader sharing the underlying storage but with its own cursor.
+    FileReader clone() const;
+
+    /// \brief Seek to an absolute byte position.
+    /// \throws std::out_of_range if pos > size().
+    void seek(size_t pos);
+
+    /// \brief Return the current cursor position.
+    size_t tell() const;
+
+    /// \brief Total file size in bytes.
+    size_t size() const;
+
+    /// \brief Peek the byte at the current cursor.
+    /// \throws std::out_of_range at EOF.
+    char peek() const;
+
+    /// \brief Advance the cursor by n bytes (default 1).
+    /// \throws std::out_of_range if advancing crosses EOF.
+    void increment(size_t n = 1);
+
+    /// \brief Push the next sequential page (pushed backend only).
+	/// \param data Contents of the page.
+    /// \throws std::logic_error if the current backend is not pushed mode, or if a next page is already queued.
+    void push_next_page(const std::string& data);
+
+    /// \brief Returns true if the cursor is at or beyond the end of available data.
+    /// For the pushed backend, EOF means all pushed bytes have been consumed.
+    bool eof() const;
+
+    /// \brief Equivalent of peek() followed by increment(1)
+    char read();
+
+    /// \brief Equivalent of peek() followed by increment(1)
+    char get(size_t offset) const;
+
+    struct Impl {
+        virtual ~Impl() = default;
+        virtual size_t size() const = 0;
+        virtual char get(size_t pos) const = 0;
+        /// \brief Backend may support pushing pages; default throws.
+        virtual void push_next_page(const std::string&) {
+            throw std::logic_error("push_next_page: backend does not support pushed mode");
+        }
+    };
+
+private:
+    std::shared_ptr<Impl> impl_;
+    size_t cursor_ = 0;
+};
+
+} // namespace IfcParse
+
+#endif

src/ifcparse/IfcCharacterDecoder.cpp

@@ -29,7 +29,7 @@
 #include "IfcCharacterDecoder.h"
 
 #include "IfcException.h"
-#include "IfcSpfStream.h"
+#include "FileReader.h"
 #include "IfcLogger.h"
 
 #include <codecvt>
@@ -77,7 +77,7 @@
 
 using namespace IfcParse;
 
-IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::IfcSpfStream* stream) {
+IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::FileReader* stream) {
     stream_ = stream;
     codepage_ = 0;
 }
@@ -86,49 +86,9 @@ IfcCharacterDecoder::~IfcCharacterDecoder() {
 }
 
 namespace {
-unsigned int reference_helper = 0;
+    std::string read_string(IfcParse::FileReader& stream_, IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
+        std::u32string builder_;
 
-class pure_impure_helper {
-  private:
-    bool pure_;
-    IfcParse::IfcSpfStream* stream_;
-    unsigned int& pointer_;
-    std::u32string builder_;
-
-    char peek() {
-        if (pure_) {
-            return stream_->peek_at(pointer_);
-        }
-        return stream_->Peek();
-    }
-
-    unsigned int tell() {
-        if (pure_) {
-            return pointer_;
-        }
-        return stream_->Tell();
-    }
-
-    void increment() {
-        if (pure_) {
-            stream_->increment_at(pointer_);
-        } else {
-            stream_->Inc();
-        }
-    }
-
-  public:
-    pure_impure_helper(IfcParse::IfcSpfStream* stream)
-        : pure_(false),
-          stream_(stream),
-          pointer_(reference_helper) {}
-
-    pure_impure_helper(IfcParse::IfcSpfStream* stream, unsigned int& pointer)
-        : pure_(true),
-          stream_(stream),
-          pointer_(pointer) {}
-
-    std::string get(IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
         unsigned int parse_state = 0;
         builder_.clear();
         builder_.push_back('\'');
@@ -137,7 +97,7 @@ class pure_impure_helper {
         unsigned int hex = 0;
         unsigned int hex_count = 0;
 
-        while ((current_char = peek()) != 0) {
+        while ((current_char = stream_.peek()) != 0) {
             if (EXPECTS_CHARACTER(parse_state)) {
                 builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80));
                 parse_state = 0;
@@ -178,7 +138,7 @@ class pure_impure_helper {
             } else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) {
                 if (IS_LOWERCASE_HEX(current_char)) {
                     Logger::Warning("Lowercase hexadecimal character '" + std::string(1, current_char) +
-                                    "' found at offset " + std::to_string(pointer_) +
+                                    "' found at offset " + std::to_string(stream_.tell()) +
                                     ". It is recommended to use uppercase for hexadecimal.");
                 }
                 hex <<= 4;
@@ -202,12 +162,12 @@ class pure_impure_helper {
                 if (parse_state == APOSTROPHE && current_char != '\'') {
                     break;
                 }
-                throw IfcInvalidTokenException(tell(), current_char);
+                throw IfcInvalidTokenException(stream_.tell(), current_char);
             } else {
                 parse_state = hex = hex_count = 0;
                 builder_.push_back(current_char);
             }
-            increment();
+            stream_.increment();
         }
         builder_.push_back('\'');
 
@@ -248,22 +208,25 @@ class pure_impure_helper {
         }
         throw IfcParse::IfcException("Invalid conversion mode");
     }
-};
 } // namespace
 
 IfcCharacterDecoder::operator std::string() {
-    return pure_impure_helper(stream_).get(mode, substitution_character);
+    return read_string(*stream_, mode, substitution_character);
 }
 
-std::string IfcCharacterDecoder::get(unsigned int& ptr) {
+std::string IfcCharacterDecoder::get(size_t& ptr) {
-    return pure_impure_helper(stream_, ptr).get(mode, substitution_character);
+    auto local_stream = *stream_;
+    local_stream.seek(ptr);
+    auto s = read_string(local_stream, mode, substitution_character);
+	ptr = local_stream.tell();
+    return s;
 }
 
 void IfcCharacterDecoder::skip() {
     unsigned int parse_state = 0;
     char current_char;
     unsigned int hex_count = 0;
-    while ((current_char = stream_->Peek()) != 0) {
+    while ((current_char = stream_->peek()) != 0) {
         if (EXPECTS_CHARACTER(parse_state)) {
             parse_state = 0;
         } else if (current_char == '\'' && (parse_state == 0U)) {
@@ -318,11 +281,11 @@ void IfcCharacterDecoder::skip() {
             if (parse_state == APOSTROPHE && current_char != '\'') {
                 break;
             }
-            throw IfcInvalidTokenException(stream_->Tell(), current_char);
+            throw IfcInvalidTokenException(stream_->tell(), current_char);
         } else {
             parse_state = hex_count = 0;
         }
-        stream_->Inc();
+        stream_->increment();
     }
 }
 

src/ifcparse/IfcCharacterDecoder.h

@@ -27,7 +27,7 @@
 #ifndef IFCCHARACTERDECODER_H
 #define IFCCHARACTERDECODER_H
 
-#include "IfcSpfStream.h"
+#include "FileReader.h"
 
 #include <string>
 
@@ -41,7 +41,7 @@ namespace IfcParse {
 
 class IFC_PARSE_API IfcCharacterDecoder {
   private:
-    IfcParse::IfcSpfStream* stream_;
+    IfcParse::FileReader* stream_;
     int codepage_;
 
   public:
@@ -52,7 +52,7 @@ class IFC_PARSE_API IfcCharacterDecoder {
     };
     static ConversionMode mode;
     static char substitution_character;
-    IfcCharacterDecoder(IfcParse::IfcSpfStream* stream);
+    IfcCharacterDecoder(IfcParse::FileReader* stream);
     ~IfcCharacterDecoder();
     // Only advances the underlying token stream read pointer
     // to the next token.
@@ -62,7 +62,7 @@ class IFC_PARSE_API IfcCharacterDecoder {
     operator std::string();
     // Gets a decoded string representation at the offset provided,
     // does not mutate the underlying token stream read pointer.
-    std::string get(unsigned int&);
+    std::string get(size_t&);
 };
 
 } // namespace IfcParse

src/ifcparse/IfcEntityInstanceData.h

@@ -155,7 +155,7 @@ struct TypeEncoder_t<parameter_pack<Types...>> {
 using TypeEncoder = TypeEncoder_t<type_variant_parameter_pack>;
 
 struct IFC_PARSE_API MutableAttributeValue {
-    int name_;
+    uint32_t name_;
     uint8_t index_;
 };
 

src/ifcparse/IfcException.h

@@ -57,7 +57,7 @@ class IFC_PARSE_API IfcAttributeOutOfRangeException : public IfcException {
 class IFC_PARSE_API IfcInvalidTokenException : public IfcException {
   public:
     IfcInvalidTokenException(
-        int token_start,
+        size_t token_start,
         const std::string& token_string,
         const std::string& expected_type)
         : IfcException(
@@ -65,7 +65,7 @@ class IFC_PARSE_API IfcInvalidTokenException : public IfcException {
               boost::lexical_cast<std::string>(token_start) +
               " invalid " + expected_type) {}
     IfcInvalidTokenException(
-        int token_start,
+        size_t token_start,
         char character)
         : IfcException(
               std::string("Unexpected '") + std::string(1, character) + "' at offset " +

src/ifcparse/IfcFile.cpp

@@ -56,7 +56,7 @@ namespace {
     constexpr bool is_type_in_variant_v = is_type_in_variant<Variant, T>::value;
 
     template <typename Fn>
-    void dispatch_token(int instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) {
+    void dispatch_token(boost::optional<size_t> instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) {
         if (t.type == IfcParse::Token_BINARY) {
             fn(IfcParse::TokenFunc::asBinary(t));
         } else if (IfcParse::TokenFunc::isBool(t)) {
@@ -89,7 +89,7 @@ namespace {
     }
 
     template <size_t Depth, typename Fn>
-    void construct_(int instance_id, int attribute_id, IfcParse::parse_context& p, const IfcParse::aggregation_type* aggr, Fn fn) {
+    void construct_(boost::optional<size_t> instance_id, int attribute_id, IfcParse::parse_context& p, const IfcParse::aggregation_type* aggr, Fn fn) {
         if (p.tokens_.empty()) {
             // @todo instead of ugly if-else we could also default initialize the respective
             // variant types below.
@@ -234,7 +234,7 @@ namespace {
     }
 }
 
-IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count) {
+IfcEntityInstanceData IfcParse::parse_context::construct(boost::optional<size_t> name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count) {
     std::vector<const IfcParse::parameter_type*> parameter_types;
     std::unique_ptr<IfcParse::named_type> transient_named_type;
 
@@ -259,7 +259,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
         expected_size && *expected_size != tokens_.size())
     {
         size_t expected = expected_size ? *expected_size : parameter_types.size();
-        Logger::Warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + " for instance #" + std::to_string(name > 0 ? name : 0));
+        Logger::Warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + (name ? std::string(" for instance #" + std::to_string(*name)) : std::string("")));
     }
 
     if (tokens_.empty()) {
@@ -289,12 +289,12 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
             if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::Token>) {
                 dispatch_token(name, index, v, param_type && param_type->as_named_type() ? param_type->as_named_type()->declared_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](auto v) {
                     if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::reference_or_simple_type>) {
-                        if (name > 0) {
+                        if (name) {
                             references_to_resolve.push_back(std::make_pair(
                                 // @todo previously this was storage but apparently the 
                                 // pointer is not constant with the moving and temporary nature
                                 // maybe it ought to be and in that case a pointer is more direct
-                                MutableAttributeValue{ name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index },
+                                MutableAttributeValue{ (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index },
                                 v
                             ));
                         }
@@ -311,12 +311,12 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
                 }
                 construct_<0>(name, index, *v, pt ? pt->as_aggregation_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](const auto& v) {
                     if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::vector<reference_or_simple_type>>) {
-                        if (name > 0) {
+                        if (name) {
-                            references_to_resolve.push_back({ {name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
+                            references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
                         }
                     } else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::vector<std::vector<reference_or_simple_type>>>) {
-                        if (name > 0) {
+                        if (name) {
-                            references_to_resolve.push_back({ {name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
+                            references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
                         }
                     } else {
                         storage.set(index, v);
@@ -652,6 +652,8 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
 }
 
 std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> IfcParse::InstanceStreamer::read_instance() {
+    // std::cout << "global: " << stream_->tell() << std::endl;
+
     std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> return_value;
 
     if (header_ && yielded_header_instances_ < 3) {
@@ -679,7 +681,7 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstance
     }
 
     unsigned current_id = 0;
-    while (good_ && !lexer_->stream->eof && !current_id) {
+    while (good_ && !lexer_->stream->eof() && !current_id) {
         if (token_stream_[0].type == IfcParse::Token_IDENTIFIER &&
             token_stream_[1].type == IfcParse::Token_OPERATOR &&
             token_stream_[1].value_char == '=' &&
@@ -734,11 +736,13 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstance
             Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
         }
 
-        if (!lexer_->stream->eof && next_token.type == Token_NONE) {
+        if (!lexer_->stream->eof() && next_token.type == Token_NONE) {
             good_ = file_open_status::INVALID_SYNTAX;
             break;
         }
 
+        // std::cout << next_token.startPos << " " << TokenFunc::toString(next_token) << std::endl;
+
         token_stream_.push_back(next_token);
     }
 

src/ifcparse/IfcFile.h

@@ -88,12 +88,11 @@ IFC_PARSE_API filetype guess_file_type(const std::string& fn);
 
 class IFC_PARSE_API InstanceStreamer {
 private:
-    IfcSpfStream* stream_;
+    FileReader* stream_;
     IfcSpfLexer* lexer_;
     IfcSpfHeader* header_;
     boost::circular_buffer<Token> token_stream_;
     const IfcParse::schema_definition* schema_;
-    const IfcParse::declaration* ifcroot_type_;
     IfcParse::impl::in_memory_file_storage storage_;
     IfcParse::file_open_status good_ = IfcParse::file_open_status::SUCCESS;
     int progress_;
@@ -104,7 +103,7 @@ public:
 	bool coerce_attribute_count = true;
 
     operator bool() const {
-        return good_ && !lexer_->stream->eof;
+        return good_ && !lexer_->stream->eof();
     }
 
     IfcParse::file_open_status status() const {
@@ -131,7 +130,13 @@ public:
         return storage_.steal_instances();
     }
 
-    InstanceStreamer(const std::string& fn);
+    bool has_semicolon() const;
+
+	void push_page(const std::string& page);
+
+    InstanceStreamer();
+
+    InstanceStreamer(const std::string& fn, bool mmap=false);
 
     InstanceStreamer(void* data, int length);
 
@@ -201,14 +206,44 @@ private:
 
   public:
 #ifdef USE_MMAP
-    IfcFile(const std::string& path, bool mmap = false);
+    /// <summary>
-#else
+	/// Constructs an IfcFile object from a file path, optionally using memory-mapped I/O, only supports IFC-SPF files.
-    IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false);
+    /// </summary>
+    /// <param name="path">UTF-8 file path to an IFC-SPF file</param>
+    /// <param name="mmap">Whether to use memory-mapped I/O</param>
+    IfcFile(const std::string& path, bool mmap);
 #endif
+    /// <summary>
+	/// Constructs an IfcFile object from a file path, supports IFC-SPF and the IfcOpenShell-specific RocksDB format.
+    /// </summary>
+    /// <param name="path">UTF-8 file path to an IFC-SPF file or RocksDB database directory</param>
+    /// <param name="ty">File type of the path</param>
+    /// <param name="readonly">Whether to open in read-only mode, only supported on RocksDB databases</param>
+    IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false);
+
+    /// <summary>
+	/// Constructs an IfcFile object from a stream containing IFC-SPF data.
+    /// </summary>
     IfcFile(std::istream& stream, int length);
+
+    /// <summary>
+	/// Constructs an IfcFile object from a memory buffer containing IFC-SPF data.
+    /// </summary>
     IfcFile(void* data, int length);
-    IfcFile(IfcParse::IfcSpfStream* stream);
+
-    // @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
+    /// <summary>
+    /// Constructs an IfcFile object from a given IFC SPF stream.
+    /// </summary>
+    /// <param name="stream">A pointer to an IfcParse::FileReader object representing the input IFC SPF data stream.</param>
+    IfcFile(IfcParse::FileReader* stream);
+
+    /// <summary>
+    /// Constructs an IfcFile object with the specified schema, file type, and file path.
+    /// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
+    /// </summary>
+    /// <param name="schema">Pointer to the schema definition to use. Defaults to the IFC4 schema if not specified.</param>
+    /// <param name="ty">The file type to use for the file. Defaults to FT_AUTODETECT.</param>
+    /// <param name="path">The file system path to the IFC file. Defaults to an empty string.</param>
     IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = FT_AUTODETECT, const std::string& path = "");
 
     ~IfcFile();

src/ifcparse/IfcParse.cpp

@@ -26,7 +26,7 @@
 #include "IfcLogger.h"
 #include "IfcSchema.h"
 #include "IfcSIPrefix.h"
-#include "IfcSpfStream.h"
+#include "FileReader.h"
 #include "utils.h"
 
 #include <algorithm>
@@ -102,167 +102,7 @@ void init_locale() {
 
 #endif
 
-//
+IfcSpfLexer::IfcSpfLexer(IfcParse::FileReader* stream_) {
-// Opens the file and gets the filesize
-//
-#ifdef USE_MMAP
-IfcSpfStream::IfcSpfStream(const std::string& path, bool mmap)
-#else
-IfcSpfStream::IfcSpfStream(const std::string& path)
-#endif
-    : stream_(0),
-      buffer_(0),
-      valid(false),
-      eof(false) {
-#ifdef _MSC_VER
-    std::wstring fn_ws = IfcUtil::path::from_utf8(path);
-    const wchar_t* fn_wide = fn_ws.c_str();
-
-#ifdef USE_MMAP
-    if (mmap) {
-        mfs = boost::iostreams::mapped_file_source(boost::filesystem::wpath(fn_wide));
-    } else {
-#endif
-        stream_ = _wfopen(fn_wide, L"rb");
-#ifdef USE_MMAP
-    }
-#endif
-
-#else
-
-#ifdef USE_MMAP
-    if (mmap) {
-        mfs = boost::iostreams::mapped_file_source(path);
-    } else {
-#endif
-        stream_ = fopen(path.c_str(), "rb");
-#ifdef USE_MMAP
-    }
-#endif
-
-#endif
-
-#ifdef USE_MMAP
-    if (mmap) {
-        if (!mfs.is_open()) {
-            return;
-        }
-
-        valid = true;
-        buffer_ = mfs.data();
-        ptr_ = 0;
-        len_ = mfs.size();
-    } else {
-#endif
-        if (stream_ == NULL) {
-            return;
-        }
-
-        valid = true;
-        fseek(stream_, 0, SEEK_END);
-        size = (unsigned int)ftell(stream_);
-        rewind(stream_);
-        char* buffer_rw = new char[size];
-        len_ = (unsigned int)fread(buffer_rw, 1, size, stream_);
-        buffer_ = buffer_rw;
-        eof = len_ == 0;
-        ptr_ = 0;
-        fclose(stream_);
-        stream_ = nullptr;
-#ifdef USE_MMAP
-    }
-#endif
-}
-
-IfcSpfStream::IfcSpfStream(std::istream& stream, int length)
-    : stream_(0),
-      buffer_(0) {
-    eof = false;
-    size = length;
-    char* buffer_rw = new char[size];
-    stream.read(buffer_rw, size);
-    buffer_ = buffer_rw;
-    valid = stream.gcount() == size;
-    ptr_ = 0;
-    len_ = length;
-}
-
-IfcSpfStream::IfcSpfStream(void* data, int length)
-    : stream_(0),
-      buffer_(0) {
-    eof = false;
-    size = length;
-    buffer_ = (char*)data;
-    valid = true;
-    ptr_ = 0;
-    len_ = length;
-}
-
-IfcSpfStream::~IfcSpfStream() {
-    Close();
-}
-
-void IfcSpfStream::Close() {
-#ifdef USE_MMAP
-    if (mfs.is_open()) {
-        mfs.close();
-        return;
-    }
-#endif
-    delete[] buffer_;
-    if (stream_ != nullptr) {
-        fclose(stream_);
-    }
-}
-
-//
-// Seeks an arbitrary position in the file
-//
-void IfcSpfStream::Seek(unsigned int offset) {
-    ptr_ = offset;
-    if (ptr_ >= len_) {
-        throw IfcException("Reading outside of file limits");
-    }
-    eof = false;
-}
-
-//
-// Returns the character at the cursor
-//
-char IfcSpfStream::Peek() {
-    return buffer_[ptr_];
-}
-
-//
-// Returns the character at specified offset
-//
-char IfcSpfStream::Read(unsigned int offset) {
-    return buffer_[offset];
-}
-
-//
-// Returns the cursor position
-//
-unsigned int IfcSpfStream::Tell() const {
-    return ptr_;
-}
-
-//
-// Increments cursor and reads new chunk if necessary
-//
-void IfcSpfStream::Inc() {
-    if (++ptr_ == len_) {
-        eof = true;
-        return;
-    }
-    const char current = IfcSpfStream::Peek();
-    if (current == '\n' || current == '\r') {
-        // NB this is recursive. It might as well be a loop.
-        IfcSpfStream::Inc();
-    }
-}
-
-IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_) {
     stream = stream_;
     decoder_ = new IfcCharacterDecoder(stream_);
 }
@@ -271,12 +111,12 @@ IfcSpfLexer::~IfcSpfLexer() {
     delete decoder_;
 }
 
-unsigned int IfcSpfLexer::skipWhitespace() const {
+size_t IfcSpfLexer::skipWhitespace() const {
-    unsigned int index = 0;
+    size_t index = 0;
-    while (!stream->eof) {
+    while (!stream->eof()) {
-        char character = stream->Peek();
+        char character = stream->peek();
         if ((character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
-            stream->Inc();
+            stream->increment();
             ++index;
         } else {
             break;
@@ -285,22 +125,22 @@ unsigned int IfcSpfLexer::skipWhitespace() const {
     return index;
 }
 
-unsigned int IfcSpfLexer::skipComment() const {
+size_t IfcSpfLexer::skipComment() const {
-    char character = stream->Peek();
+    char character = stream->peek();
     if (character != '/') {
         return 0;
     }
-    stream->Inc();
+    stream->increment();
-    character = stream->Peek();
+    character = stream->peek();
     if (character != '*') {
-        stream->Seek(stream->Tell() - 1);
+        stream->seek(stream->tell() - 1);
         return 0;
     }
-    unsigned int index = 2;
+    size_t index = 2;
     char intermediate = 0;
-    while (!stream->eof) {
+    while (!stream->eof()) {
-        character = stream->Peek();
+        character = stream->peek();
-        stream->Inc();
+        stream->increment();
         ++index;
         if (character == '/' && intermediate == '*') {
             break;
@@ -315,19 +155,20 @@ unsigned int IfcSpfLexer::skipComment() const {
 //
 Token IfcSpfLexer::Next() {
 
-    if (stream->eof) {
+    if (stream->eof()) {
         return Token{};
     }
 
     while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
     }
 
-    if (stream->eof) {
+    if (stream->eof()) {
         return Token{};
     }
-    unsigned int pos = stream->Tell();
 
-    char character = stream->Peek();
+    auto& str = GetTempString();
+    auto pos = stream->tell();
+    char character = stream->read();
 
     // If the cursor is at [()=,;$*] we know token consists of single char
     if (character == '(' ||
@@ -336,69 +177,45 @@ Token IfcSpfLexer::Next() {
         character == ',' ||
         character == ';' ||
         character == '$' ||
-        character == '*') {
+        character == '*')
-        stream->Inc();
+    {
-        return OperatorTokenPtr(this, pos, pos + 1);
+        return OperatorTokenPtr(this, pos, character);
     }
 
-    int len = 0;
+    if (character == '\'') {
-
-    while (!stream->eof) {
-
-        // Read character and increment pointer if not starting a new token
-        character = stream->Peek();
-        if ((len != 0) && (character == '(' ||
-                           character == ')' ||
-                           character == '=' ||
-                           character == ',' ||
-                           character == ';' ||
-                           character == '/')) {
-            break;
-        }
-        stream->Inc();
-        len++;
-
         // If a string is encountered defer processing to the IfcCharacterDecoder
-        if (character == '\'') {
+        str = *decoder_;
-            decoder_->skip();
+    } else {
+        str.assign(&character, 1);
+
+        while (!stream->eof()) {
+            // Read character and increment pointer if not starting a new token
+            character = stream->peek();
+            if (character == '(' ||
+                character == ')' ||
+                character == '=' ||
+                character == ',' ||
+                character == ';' ||
+                character == '/') {
+                break;
+            }
+            str.push_back(character);
+            stream->increment();
         }
     }
-    Token t;
+    return GeneralTokenPtr(this, pos, str);
-    if (len != 0) {
-        t = GeneralTokenPtr(this, pos, stream->Tell());
-    } else {
-        t = Token{};
-    }
-    // std::wcout << "token: " << pos << " " << TokenFunc::asStringRef(t).c_str() << std::endl;
-    return t;
-}
-
-bool IfcSpfStream::is_eof_at(unsigned int local_ptr) const {
-    return local_ptr >= len_;
-}
-
-void IfcSpfStream::increment_at(unsigned int& local_ptr) {
-    if (++local_ptr == len_) {
-        return;
-    }
-    const char current = IfcSpfStream::peek_at(local_ptr);
-    if (current == '\n' || current == '\r') {
-        IfcSpfStream::increment_at(local_ptr);
-    }
-}
-
-char IfcSpfStream::peek_at(unsigned int local_ptr) {
-    return buffer_[local_ptr];
 }
 
 //
 // Reads a std::string from the file at specified offset
 // Omits whitespace and comments
 //
-void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
+void IfcSpfLexer::TokenString(size_t offset, std::string& buffer) {
     buffer.clear();
-    while (!stream->is_eof_at(offset)) {
+	auto local_stream = *this->stream;
-        char character = stream->peek_at(offset);
+	local_stream.seek(offset);
+    while (!local_stream.eof()) {
+        char character = local_stream.peek();
         if (!buffer.empty() && (character == '(' ||
                                 character == ')' ||
                                 character == '=' ||
@@ -407,7 +224,7 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
                                 character == '/')) {
             break;
         }
-        stream->increment_at(offset);
+        local_stream.increment();
         if (character == ' ' ||
             character == '\r' ||
             character == '\n' ||
@@ -416,6 +233,7 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
         }
         if (character == '\'') {
             // todo, make decoder use local offset ptr
+            auto offset = local_stream.tell();
             buffer = decoder_->get(offset);
             break;
         }
@@ -424,10 +242,11 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
 }
 
 //Note: according to STEP standard, there may be newlines in tokens
-inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned end, std::string& oDestination) {
+/*
+inline void RemoveTokenSeparators(FileReader* stream, size_t start, size_t end, std::string& oDestination) {
     oDestination.clear();
     for (unsigned i = start; i < end; i++) {
-        char character = stream->Read(i);
+        char character = stream->get(i);
         if (character == ' ' ||
             character == '\r' ||
             character == '\n' ||
@@ -437,6 +256,7 @@ inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned
         oDestination += character;
     }
 }
+*/
 
 bool ParseInt(const char* pStart, int& val) {
     char* pEnd;
@@ -481,22 +301,17 @@ bool ParseBool(const char* pStart, int& val) {
     return true;
 }
 
-Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) {
+Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, size_t start, char data) {
-    char first = lexer->stream->Read(start);
+    Token token(lexer, start, Token_OPERATOR);
-    Token token(lexer, start, end, Token_OPERATOR);
+    token.value_char = data;
-    token.value_char = first;
     return token;
 }
 
-Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) {
+Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, size_t start, const std::string& tokenStr) {
-    Token token(lexer, start, end, Token_NONE);
+    Token token(lexer, start, Token_NONE);
-
-    //extract token into temp buffer (remove eol-s, no encoding changes)
-    std::string& tokenStr = lexer->GetTempString();
-    RemoveTokenSeparators(lexer->stream, start, end, tokenStr);
 
     //determine type of the token
-    char first = lexer->stream->Read(start);
+    const char& first = tokenStr.front();
     if (first == '#') {
         token.type = Token_IDENTIFIER;
         if (!ParseInt(tokenStr.c_str() + 1, token.value_int)) {
@@ -521,6 +336,11 @@ Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end
         token.type = Token_KEYWORD;
     }
 
+    // todo STRING/BINARY/ENUM/KEYWORD not stored
+    // tokenStr is gone after this.
+    // Orrrr we could only free pages when after a full instance is formed because after that no
+	// references to prior token ranges exist anymore.
+
     return token;
 }
 
@@ -676,7 +496,25 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) {
 
 std::string TokenFunc::toString(const Token& token) {
     std::string result;
-    token.lexer->TokenString(token.startPos, result);
+    if (token.type == Token_OPERATOR) {
+		result.push_back(token.value_char);
+    } else if (token.type == Token_INT) {
+        result = std::to_string(token.value_int);
+    } else if (token.type == Token_BOOL) {
+        if (token.value_int == 1) {
+            result = ".T.";
+        } else if (token.value_int == 0) {
+            result = ".F.";
+        } else {
+            result = ".U.";
+        }
+    } else if (token.type == Token_FLOAT) {
+        std::ostringstream oss;
+        oss << std::setprecision(15) << token.value_double;
+        result = oss.str();
+	} else {
+        token.lexer->TokenString(token.startPos, result);
+    }
     return result;
 }
 
@@ -684,7 +522,7 @@ std::string TokenFunc::toString(const Token& token) {
 // Reads the arguments from a list of token
 // Aditionally, registers the ids (i.e. #[\d]+) in the inverse map
 //
-void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
+void IfcParse::impl::in_memory_file_storage::load(boost::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
     Token next = tokens->Next();
 
     /*
@@ -708,8 +546,8 @@ void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name,
             load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
         } else {
             return_value++;
-            if (TokenFunc::isIdentifier(next) && entity) {
+            if (TokenFunc::isIdentifier(next) && entity && entity_instance_name) {
-                register_inverse(entity_instance_name, entity, next.value_int, attribute_index == -1 ? attribute_index_within_data : attribute_index);
+                register_inverse(*entity_instance_name, entity, next.value_int, attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
             }
 
             if (TokenFunc::isKeyword(next)) {
@@ -758,10 +596,10 @@ IfcEntityInstanceData IfcParse::impl::in_memory_file_storage::read(unsigned int
 }
 
 void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const {
-    unsigned int old_offset = tokens->stream->Tell();
+    auto old_offset = tokens->stream->tell();
     Token semilocon = tokens->Next();
     if (!TokenFunc::isOperator(semilocon, ';')) {
-        tokens->stream->Seek(old_offset);
+        tokens->stream->seek(old_offset);
     }
 }
 
@@ -783,7 +621,7 @@ void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from
 
 namespace {
     template <typename T>
-    std::string to_string_fixed_width(const T& t, size_t w) {
+    std::string to_string_fixed_width(const T& t, size_t) {
         // @todo currently inactive
         std::ostringstream oss;
         oss << /*std::setfill('0') << std::setw(w) <<*/ t;
@@ -1345,11 +1183,27 @@ IfcUtil::IfcBaseClass::set_attribute_value(const std::string& s, const T& t) {
 //
 #ifdef USE_MMAP
 IfcFile::IfcFile(const std::string& fn, bool mmap) {
-    IfcSpfStream s(fn, mmap);
+	std::unique_ptr<FileReader> s;
-    storage_ = impl::in_memory_file_storage{};
+    if (mmap) {
-    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s);
+        s = std::make_unique<FileReader>(fn, FileReader::mmap_tag{});
+    } else {
+        s = std::make_unique<FileReader>(fn);
+	}   
+
+    storage_.emplace<1>(this);
+    std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&*s, schema_, max_id_);
+
+    if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
+        // @todo unify these names, it's already confusing enough as it stands
+        byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
+        byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
+        byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
+    }
+
+    ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
 }
-#else
+#endif
+
 IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
     : schema_(nullptr)
     , max_id_(0)
@@ -1360,11 +1214,11 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
         ty = guess_file_type(path);
     }
     if (ty == FT_IFCSPF) {
-        IfcSpfStream s(path);
+        FileReader s(path);
         storage_.emplace<1>(this);
         std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
 
-        if (good_ = std::get<impl::in_memory_file_storage>(storage_).good_) {
+        if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
             // @todo unify these names, it's already confusing enough as it stands
             byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
             byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
@@ -1398,13 +1252,18 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
     }
     ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
 }
-#endif
 
 IfcFile::IfcFile(std::istream& stream, int length)
     : schema_(nullptr)
     , max_id_(0)
 {
-    IfcSpfStream s(stream, length);
+    FileReader s(FileReader::caller_fed_tag{});
+
+    std::string string_data;
+	string_data.resize(length);
+	stream.read(string_data.data(), length);
+    s.push_next_page(string_data);
+
     storage_.emplace<1>(this);
     std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
     good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
@@ -1419,7 +1278,8 @@ IfcFile::IfcFile(void* data, int length)
     : schema_(nullptr)
     , max_id_(0)
 {
-    IfcSpfStream s(data, length);
+	FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{});
+    
     storage_.emplace<1>(this);
     std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
     good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
@@ -1430,7 +1290,7 @@ IfcFile::IfcFile(void* data, int length)
     byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
 }
 
-IfcFile::IfcFile(IfcParse::IfcSpfStream* s)
+IfcFile::IfcFile(IfcParse::FileReader* s)
     : schema_(nullptr)
     , max_id_(0)
 {
@@ -1475,18 +1335,65 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
     setDefaultHeaderValues();
 }
 
-IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn)
+bool IfcParse::InstanceStreamer::has_semicolon() const
-    : stream_(new IfcSpfStream(fn))
+{
+    auto local_stream = stream_->clone();
+	auto local_lexer = IfcSpfLexer(&local_stream);
+    Token t = local_lexer.Next();
+    while (t.type != Token_NONE) {
+        if (TokenFunc::isOperator(t, ';')) {
+            return true;
+		}
+        // probably the issue is that this moves the cursor past the page boundary on the local stream which also affects the global one
+        // 'freeze' the stream so that no pages are cleared
+        t = local_lexer.Next();
+    }
+	std::cout << "local: " << local_stream.tell() << " global: " << stream_->tell() << std::endl;
+	return false;
+}
+
+void IfcParse::InstanceStreamer::push_page(const std::string& page)
+{
+    std::cout << "global: " << stream_->tell() << std::endl;
+    stream_->push_next_page(page);
+    if (good_ == file_open_status::NO_HEADER) {
+        header_ = new IfcParse::IfcSpfHeader(lexer_);
+        if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
+            try {
+                schema_ = IfcParse::schema_by_name(header_->file_schema()->schema_identifiers().front());
+                good_ = file_open_status::SUCCESS;
+            } catch (const IfcParse::IfcException&) {
+            }
+        }
+        storage_.file = nullptr;
+        storage_.schema = schema_;
+        storage_.tokens = lexer_;
+        storage_.references_to_resolve = &references_to_resolve_;
+    }
+}
+
+IfcParse::InstanceStreamer::InstanceStreamer()
+    : stream_(new FileReader(FileReader::caller_fed_tag{}))
+    , lexer_(new IfcSpfLexer(stream_))
+    , token_stream_(3, Token{})
+    , schema_(nullptr)
+    , progress_(0)
+{
+    init_locale();
+    good_ = file_open_status::NO_HEADER;
+}
+
+IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap)
+    : stream_(mmap ? new FileReader(fn, FileReader::mmap_tag{}) : new FileReader(fn))
     , lexer_(new IfcSpfLexer(stream_))
     , token_stream_(3, Token{})
     , schema_(nullptr)
-    , ifcroot_type_(nullptr)
     , progress_(0)
 {
     init_locale();
 
     good_ = file_open_status::NO_HEADER;
-    if (*stream_) {
+    if (stream_->size() && !stream_->eof()) {
         header_ = new IfcParse::IfcSpfHeader(lexer_);
         if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
             try {
@@ -1503,17 +1410,16 @@ IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn)
 }
 
 IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length)
-    : stream_(new IfcSpfStream(data, length))
+    : stream_(new FileReader(std::string((char*) data, length), FileReader::caller_fed_tag{}))
     , lexer_(new IfcSpfLexer(stream_))
     , token_stream_(3, Token{})
     , schema_(nullptr)
-    , ifcroot_type_(nullptr)
     , progress_(0)
 {
     init_locale();
 
     good_ = file_open_status::NO_HEADER;
-    if (*stream_) {
+    if (stream_->size() && !stream_->eof()) {
         header_ = new IfcParse::IfcSpfHeader(lexer_);
         if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
             try {
@@ -1535,7 +1441,6 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition*
     , header_(nullptr)
     , token_stream_(3, Token{})
     , schema_(schema)
-    , ifcroot_type_(schema->declaration_by_name("IfcRoot"))
     , progress_(0)
 {
     init_locale();
@@ -1546,14 +1451,14 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition*
     storage_.references_to_resolve = &references_to_resolve_;
 }
 
-void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfStream* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) {
+void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) {
     // Initialize a "C" locale for locale-independent
     // number parsing. See comment above on line 41.
     init_locale();
 
     tokens = nullptr;
 
-    if (!s->valid) {
+    if (!s->size() || s->eof()) {
         // @todo set good on parent file
         good_ = file_open_status::READ_ERROR;
         return;
@@ -1608,7 +1513,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
 
         auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst)));
         instance->file_ = file;
-        instance->id_ = current_id;
+        instance->id_ = (uint32_t) current_id;
 
         if (instance->declaration().is(*ifcroot_type_)) {
             try {
@@ -1641,7 +1546,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
         }
 
         // byidentity_[instance->identity()] = instance;
-        byid_.insert({ current_id, instance });
+        byid_.insert({(uint32_t) current_id, instance });
 
         // @nb cannot assign to byid_;
         // byid_[current_id] = instance;
@@ -1693,10 +1598,10 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
             } else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(v)) {
                 byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst);
             }
-        } else if (auto* v = std::get_if<std::vector<reference_or_simple_type>>(&p.second)) {
+        } else if (auto* vv = std::get_if<std::vector<reference_or_simple_type>>(&p.second)) {
             aggregate_of_instance::ptr instances(new aggregate_of_instance);
-            instances->reserve(v->size());
+            instances->reserve(vv->size());
-            for (const auto& vi : *v) {
+            for (const auto& vi : *vv) {
                 if (auto* name = std::get_if<InstanceReference>(&vi)) {
                     auto it = byid_.find(*name);
                     if (it == byid_.end()) {
@@ -1726,9 +1631,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
             } else {
                 Logger::Error("Duplicate definition for instance reference");
             }
-        } else if (auto* v = std::get_if<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
+        } else if (auto* vvv = std::get_if<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
             aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance);
-            for (const auto& vi : *v) {
+            for (const auto& vi : *vvv) {
                 std::vector<IfcUtil::IfcBaseClass*> inner;
                 for (const auto& vii : vi) {
                     if (auto* name = std::get_if<InstanceReference>(&vii)) {
@@ -2003,15 +1908,15 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
             auto attr = entity->get_attribute_value(i);
             IfcUtil::ArgumentType attr_type = attr.type();
 
-            IfcParse::declaration* decl = 0;
+            IfcParse::declaration* potentially_length_measure_decl = 0;
             if (entity->declaration().as_entity() != nullptr) {
-                decl = 0;
+                potentially_length_measure_decl = 0;
                 const parameter_type* pt = entity->declaration().as_entity()->attribute_by_index(i)->type_of_attribute();
                 while (pt->as_aggregation_type() != nullptr) {
                     pt = pt->as_aggregation_type()->type_of_element();
                 }
                 if (pt->as_named_type() != nullptr) {
-                    decl = pt->as_named_type()->declared_type();
+                    potentially_length_measure_decl = pt->as_named_type()->declared_type();
                 }
             }
 
@@ -2050,7 +1955,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
                 }
                 
                 new_entity->set_attribute_value(i, new_instances);
-            } else if ((decl != nullptr) && decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) {
+            } else if ((potentially_length_measure_decl != nullptr) && potentially_length_measure_decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) {
                 if (boost::math::isnan(conversion_factor)) {
                     std::pair<IfcUtil::IfcBaseClass*, double> this_file_unit = {nullptr, 1.0};
                     std::pair<IfcUtil::IfcBaseClass*, double> other_file_unit = {nullptr, 1.0};
@@ -2283,8 +2188,6 @@ void IfcFile::process_deletion_(IfcUtil::IfcBaseClass* entity) {
 
     byid_.erase(entity->id());
 
-    const IfcParse::declaration* ty = &entity->declaration();
-
     remove_type_ref(entity);
 
     // entity_file_map is in place to prevent duplicate definitions with usage of add().
@@ -2891,7 +2794,7 @@ AttributeValue IfcEntityInstanceData::get_attribute_value(void* storage, const I
     if (storage_) {
         return AttributeValue(storage_, (uint8_t)index);
     } else {
-        return AttributeValue((IfcParse::impl::rocks_db_file_storage*)storage, identity, decl, index);
+        return AttributeValue((IfcParse::impl::rocks_db_file_storage*)storage, identity, decl, (uint8_t) index);
     }
 }
 

src/ifcparse/IfcParse.h

@@ -32,7 +32,7 @@
 #include "ifc_parse_api.h"
 #include "IfcBaseClass.h"
 #include "IfcCharacterDecoder.h"
-#include "IfcSpfStream.h"
+#include "FileReader.h"
 #include "macros.h"
 #include "storage.h"
 
@@ -103,27 +103,27 @@ class IFC_PARSE_API TokenFunc {
 // Functions for creating Tokens from an arbitary file offset
 // The first 4 bits are reserved for Tokens of type ()=,;$*
 //
-Token OperatorTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
+Token OperatorTokenPtr(IfcSpfLexer* tokens, size_t start, char data);
-Token GeneralTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
+Token GeneralTokenPtr(IfcSpfLexer* tokens, size_t start, const std::string& data);
 
-/// A stream of tokens to be read from a IfcSpfStream.
+/// A stream of tokens to be read from a FileReader.
 class IFC_PARSE_API IfcSpfLexer {
   private:
     IfcCharacterDecoder* decoder_;
-    unsigned int skipWhitespace() const;
+    size_t skipWhitespace() const;
-    unsigned int skipComment() const;
+    size_t skipComment() const;
 
   public:
     std::string& GetTempString() const {
         static my_thread_local std::string string;
         return string;
     }
-    IfcSpfStream* stream;
+    FileReader* stream;
     // IfcFile* file;
-    IfcSpfLexer(IfcSpfStream* stream);
+    IfcSpfLexer(FileReader* stream);
     Token Next();
     ~IfcSpfLexer();
-    void TokenString(unsigned int offset, std::string& result);
+    void TokenString(size_t offset, std::string& result);
 };
 
 IFC_PARSE_API aggregate_of_instance::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level = -1);

src/ifcparse/IfcSpfHeader.cpp

@@ -35,7 +35,7 @@ namespace {
             parse_context pc;
             storage->tokens->Next();
             storage->load(-1, nullptr, pc, -1);
-            return pc.construct(-1, *storage->references_to_resolve, nullptr, s, -1);
+            return pc.construct(boost::none, *storage->references_to_resolve, nullptr, s, -1);
         } else {
             // std::unreachable();
             return IfcEntityInstanceData(in_memory_attribute_storage(10));

src/ifcparse/IfcSpfStream.h

@@ -1,84 +0,0 @@
-/********************************************************************************
- *                                                                              *
- * This file is part of IfcOpenShell.                                           *
- *                                                                              *
- * IfcOpenShell is free software: you can redistribute it and/or modify         *
- * it under the terms of the Lesser GNU General Public License as published by  *
- * the Free Software Foundation, either version 3.0 of the License, or          *
- * (at your option) any later version.                                          *
- *                                                                              *
- * IfcOpenShell is distributed in the hope that it will be useful,              *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of               *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                 *
- * Lesser GNU General Public License for more details.                          *
- *                                                                              *
- * You should have received a copy of the Lesser GNU General Public License     *
- * along with this program. If not, see <http://www.gnu.org/licenses/>.         *
- *                                                                              *
- ********************************************************************************/
-
-/*********************************************************************************
- *                                                                               *
- * Reads a file and provides functions to access its                             *
- * contents randomly and character by character                                  *
- *                                                                               *
- ********************************************************************************/
-
-#ifndef IFCSPFSTREAM_H
-#define IFCSPFSTREAM_H
-
-#include "ifc_parse_api.h"
-
-#include <string>
-
-#ifdef USE_MMAP
-#include <boost/iostreams/device/mapped_file.hpp>
-#endif
-
-namespace IfcParse {
-/// The IfcSpfStream class represents a ISO 10303-21 IFC-SPF file in memory.
-/// The file is interpreted as a sequence of tokens which are lazily
-/// interpreted only when requested.
-class IFC_PARSE_API IfcSpfStream {
-  private:
-#ifdef USE_MMAP
-    boost::iostreams::mapped_file_source mfs;
-#endif
-    FILE* stream_;
-    const char* buffer_;
-    unsigned int ptr_;
-    unsigned int len_;
-
-  public:
-    bool valid;
-    bool eof;
-    unsigned int size;
-#ifdef USE_MMAP
-    IfcSpfStream(const std::string& path, bool mmap = false);
-#else
-    IfcSpfStream(const std::string& path);
-#endif
-    IfcSpfStream(std::istream& stream, int length);
-    IfcSpfStream(void* data, int length);
-    ~IfcSpfStream();
-    /// Returns the character at the cursor
-    char Peek();
-    /// Returns the character at specified offset
-    char Read(unsigned int offset);
-    /// Increment the file cursor and reads new page if necessary
-    void Inc();
-    void Close();
-    /// Moves the file cursor to an arbitrary offset in the file
-    void Seek(unsigned int offset);
-    /// Returns the cursor position
-    unsigned int Tell() const;
-
-    bool is_eof_at(unsigned int) const;
-    void increment_at(unsigned int&);
-    char peek_at(unsigned int);
-
-	operator bool() const { return valid && !eof; }
-};
-} // namespace IfcParse
-
-#endif

src/ifcparse/IfcUtil.cpp

@@ -76,8 +76,8 @@ void aggregate_of_instance::push(const aggregate_of_instance::ptr& instance) {
         }
     }
 }
-unsigned int aggregate_of_instance::size() const { return (unsigned int)list_.size(); }
+size_t aggregate_of_instance::size() const { return list_.size(); }
-void aggregate_of_instance::reserve(unsigned capacity) { list_.reserve((size_t)capacity); }
+void aggregate_of_instance::reserve(size_t capacity) { list_.reserve(capacity); }
 aggregate_of_instance::it aggregate_of_instance::begin() { return list_.begin(); }
 aggregate_of_instance::it aggregate_of_instance::end() { return list_.end(); }
 IfcUtil::IfcBaseClass* aggregate_of_instance::operator[](int i) {

src/ifcparse/aggregate_of_instance.h

@@ -48,8 +48,8 @@ class IFC_PARSE_API aggregate_of_instance {
     it begin();
     it end();
     IfcUtil::IfcBaseClass* operator[](int index);
-    unsigned int size() const;
+    size_t size() const;
-    void reserve(unsigned capacity);
+    void reserve(size_t capacity);
     bool contains(IfcUtil::IfcBaseClass*) const;
     
     template <class U>
@@ -81,7 +81,7 @@ class aggregate_of {
     }
     it begin() { return list_.begin(); }
     it end() { return list_.end(); }
-    unsigned int size() const { return (unsigned int)list_.size(); }
+    size_t size() const { return list_.size(); }
     aggregate_of_instance::ptr generalize() {
         aggregate_of_instance::ptr result(new aggregate_of_instance());
         for (it i = begin(); i != end(); ++i) {

src/ifcparse/storage.h

@@ -125,7 +125,7 @@ namespace IfcParse {
 
     class IfcFile;
     class IfcSpfLexer;
-    class IfcSpfStream;
+    class FileReader;
 
     enum TokenType {
         Token_NONE,
@@ -142,7 +142,7 @@ namespace IfcParse {
 
     struct Token {
         IfcSpfLexer* lexer; //TODO: remove it from here
-        unsigned startPos;
+        size_t startPos;
         TokenType type;
         union {
             char value_char;     //types: OPERATOR
@@ -154,7 +154,7 @@ namespace IfcParse {
             startPos(0),
             type(Token_NONE) {
         }
-        Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type)
+        Token(IfcSpfLexer* _lexer, size_t _startPos, TokenType _type)
             : lexer(_lexer),
             startPos(_startPos),
             type(_type) {
@@ -184,7 +184,7 @@ namespace IfcParse {
 
         void push(IfcUtil::IfcBaseClass* inst);
 
-        IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count=true);
+        IfcEntityInstanceData construct(boost::optional<size_t> name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count=true);
     };
 
     namespace impl {
@@ -195,7 +195,7 @@ namespace IfcParse {
             }
 
             IfcParse::IfcSpfLexer* tokens;
-            // IfcParse::IfcSpfStream* stream;
+            // IfcParse::FileReader* stream;
 
             // Either one of these needs to be set
             IfcParse::IfcFile* file;
@@ -260,7 +260,7 @@ namespace IfcParse {
             entities_by_ref_t byref_excl_;
             entity_instance_by_guid_t byguid_;
 
-            void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
+            void load(boost::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
             void try_read_semicolon() const;
 
             void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
@@ -268,7 +268,7 @@ namespace IfcParse {
 
             // @todo is this still used
             IfcEntityInstanceData read(unsigned int index);
-            void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
+            void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
 
             file_open_status good_ = file_open_status::SUCCESS;