Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-10 09:48:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Storage rework WIP

Browse Source
This commit is contained in:
Thomas Krijnen
2025-10-13 20:47:43 +02:00
parent 60d85b4259
commit 4d688170e0
15 changed files with 788 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/FileReader.cpp
+312
View File
@@ -0,0 +1,312 @@
#include "FileReader.h"
#include <algorithm>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <list>
#include <stdexcept>
#include <unordered_map>
#include <utility>
#include <vector>
#include <deque>
// Boost iostreams mmap
#include <boost/iostreams/device/mapped_file.hpp>
namespace {
#if defined(_WIN32)
inline void file_seek_abs(FILE* f, std::uint64_t off) {
if (_fseeki64(f, static_cast<long long>(off), SEEK_SET) != 0)
throw std::runtime_error("fseek failed");
}
#else
inline void file_seek_abs(FILE* f, std::uint64_t off) {
if (fseeko(f, static_cast<off_t>(off), SEEK_SET) != 0)
throw std::runtime_error("fseeko failed");
}
#endif
} // namespace
// ===================== Concrete backends =====================
using namespace IfcParse;
struct FullBufferImpl final : FileReader::Impl {
std::vector<char> buf_;
explicit FullBufferImpl(const std::string& fn) {
std::ifstream ifs(fn, std::ios::binary);
if (!ifs) throw std::runtime_error("Failed to open: " + fn);
ifs.seekg(0, std::ios::end);
const std::streamsize sz = ifs.tellg();
ifs.seekg(0, std::ios::beg);
buf_.resize(static_cast<size_t>(sz));
if (sz > 0 && !ifs.read(buf_.data(), sz)) {
throw std::runtime_error("Failed to read file into buffer");
}
}
size_t size() const override { return buf_.size(); }
char get(size_t pos) const override {
if (pos >= buf_.size()) throw std::out_of_range("get out of range");
return buf_[pos];
}
};
struct PagedFileImpl final : FileReader::Impl {
std::string fn_;
FILE* fp_ = nullptr;
size_t file_size_ = 0;
size_t page_size_ = 4096;
// LRU cache
size_t capacity_ = 8;
mutable std::list<size_t> lru_; // most recent at front
struct Entry {
FileReader::Page page;
std::list<size_t>::iterator it;
};
mutable std::unordered_map<size_t, Entry> map_;
PagedFileImpl(const std::string& fn, size_t page_size, size_t cap)
: fn_(fn), page_size_(std::max<size_t>(512, page_size)), capacity_(std::max<size_t>(2, cap)) {
namespace fs = std::filesystem;
if (!fs::exists(fn_)) throw std::runtime_error("File not found: " + fn_);
file_size_ = static_cast<size_t>(fs::file_size(fn_));
fp_ = std::fopen(fn_.c_str(), "rb");
if (!fp_) throw std::runtime_error("Failed to fopen: " + fn_);
}
~PagedFileImpl() override {
if (fp_) std::fclose(fp_);
fp_ = nullptr;
}
size_t size() const override { return file_size_; }
char get(size_t pos) const override {
if (pos >= file_size_) throw std::out_of_range("get out of range");
const size_t pidx = pos / page_size_;
const FileReader::Page& p = fetch_page_(pidx);
const size_t off = pos % page_size_;
if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes");
// Opportunistic read-ahead for sequential scans
// if (off + 1 == p.data.size()) (void)try_prefetch_(pidx + 1);
return p.data[off];
}
private:
const FileReader::Page& fetch_page_(size_t idx) const {
auto it = map_.find(idx);
if (it != map_.end()) {
touch_(it);
return it->second.page;
}
// Load page from disk using persistent FILE*
FileReader::Page pg;
pg.data.resize(page_size_);
const size_t begin = idx * page_size_;
const size_t avail = std::min(page_size_, file_size_ - begin);
file_seek_abs(fp_, begin);
if (avail > 0) {
const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
if (nread != avail) throw std::runtime_error("Short fread on page");
}
pg.data.resize(avail); // trim to actual size
// Insert into LRU
if (map_.size() >= capacity_) evict_();
lru_.push_front(idx);
auto lit = lru_.begin();
auto [emplaced_it, ok] = map_.emplace(idx, Entry{ std::move(pg), lit });
(void)ok;
return emplaced_it->second.page;
}
/*
bool try_prefetch_(size_t idx) const {
if (idx * page_size_ >= file_size_) return false;
if (map_.find(idx) != map_.end()) return true;
if (map_.size() + 1 > capacity_) return false;
(void)fetch_page_(idx);
return true;
}
*/
void touch_(typename std::unordered_map<size_t, Entry>::iterator it) const {
lru_.erase(it->second.it);
lru_.push_front(it->first);
it->second.it = lru_.begin();
}
void evict_() const {
if (lru_.empty()) return;
const size_t victim = lru_.back();
lru_.pop_back();
map_.erase(victim);
}
};
struct MMapImpl final : FileReader::Impl {
boost::iostreams::mapped_file_source map_;
size_t size_ = 0;
explicit MMapImpl(const std::string& fn) {
namespace fs = std::filesystem;
if (!fs::exists(fn)) throw std::runtime_error("File not found: " + fn);
size_ = static_cast<size_t>(fs::file_size(fn));
if (size_ == 0) return; // empty file: map_ stays closed
map_.open(fn);
if (!map_.is_open()) throw std::runtime_error("Failed to open mapped_file_source");
size_ = static_cast<size_t>(map_.size());
}
size_t size() const override { return size_; }
char get(size_t pos) const override {
if (pos >= size_) throw std::out_of_range("get out of range");
return map_.data()[pos];
}
};
/// User-pushed sequential backend with an arbitrary-length queue of future pages.
/// We keep a deque of pages; when reads move forward, we drop fully-consumed
/// pages from the front to release memory.
struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentialImpl>, FileReader::Impl {
// Deque of pages, front is earliest in file.
std::deque<FileReader::Page> pages_;
// total bytes in dropped pages
size_t discarded_page_bytes_ = 0;
size_t size() const override {
size_t n = discarded_page_bytes_;
for (auto& pg : pages_) n += pg.data.size();
return n;
}
// Drop fully-consumed pages so pos is guaranteed to be within the first page
void drop_consumed_up_to(size_t pos) {
while (!pages_.empty()) {
if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
discarded_page_bytes_ += pages_.front().data.size();
pages_.pop_front();
} else {
break;
}
}
}
char get(size_t pos) const override {
auto self = const_cast<PushedSequentialImpl*>(this);
if (this->shared_from_this().use_count() == 2) {
// only drop pages when there is only one active client.
// NB this->shared_from_this() increases count by 1
self->drop_consumed_up_to(pos);
}
const size_t avail_end = size();
if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet");
pos -= discarded_page_bytes_;
size_t page_start = 0;
for (const auto& pg : pages_) {
if (pos < page_start + pg.data.size()) {
const size_t off = pos - page_start;
return pg.data[off];
} else {
page_start += pg.data.size();
}
}
throw std::out_of_range("pushed backend: internal inconsistency");
}
void push_next_page(const std::string& data) override {
FileReader::Page p; p.data.assign(data.data(), data.data() + data.size());
pages_.push_back(std::move(p));
}
};
// ===================== FileReader public API =====================
IfcParse::FileReader::FileReader(const std::string& fn)
: cursor_(0)
{
impl_ = std::make_shared<FullBufferImpl>(fn);
}
IfcParse::FileReader::FileReader(const std::string& fn, const mmap_tag&)
: cursor_(0)
{
impl_ = std::make_shared<MMapImpl>(fn);
}
IfcParse::FileReader::FileReader(const caller_fed_tag&)
: cursor_(0)
{
impl_ = std::make_shared<PushedSequentialImpl>();
}
IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&)
{
impl_ = std::make_shared<PushedSequentialImpl>();
impl_->push_next_page(content);
}
IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
: cursor_(0)
{
impl_ = std::make_shared<PagedFileImpl>(fn, page_size, page_capacity);
}
FileReader FileReader::clone() const {
FileReader c(*this);
c.cursor_ = this->cursor_;
return c;
}
void FileReader::seek(size_t pos) {
if (pos > impl_->size()) throw std::out_of_range("seek out of range");
cursor_ = pos;
}
size_t FileReader::tell() const { return cursor_; }
size_t FileReader::size() const { return impl_->size(); }
char FileReader::peek() const {
if (cursor_ >= impl_->size()) throw std::out_of_range("peek at EOF");
return impl_->get(cursor_);
}
void FileReader::increment(size_t n) {
if (cursor_ + n > impl_->size()) throw std::out_of_range("increment past EOF");
cursor_ += n;
}
void IfcParse::FileReader::push_next_page(const std::string& data)
{
impl_->push_next_page(data);
}
bool IfcParse::FileReader::eof() const
{
return cursor_ >= impl_->size();
}
char IfcParse::FileReader::read()
{
auto c = peek();
increment(1);
return c;
}
char IfcParse::FileReader::get(size_t offset) const
{
return impl_->get(offset);
}
src/ifcparse/FileReader.h
+180
View File
@@ -0,0 +1,180 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
/*********************************************************************************
* *
* Reads a file and provides functions to access its *
* contents randomly and character by character *
* *
********************************************************************************/
#ifndef IFCSPFSTREAM_H
#define IFCSPFSTREAM_H
#include "ifc_parse_api.h"
/*
#include <string>
#ifdef USE_MMAP
#include <boost/iostreams/device/mapped_file.hpp>
#endif
namespace IfcParse {
/// The FileReader class represents a ISO 10303-21 IFC-SPF file in memory.
/// The file is interpreted as a sequence of tokens which are lazily
/// interpreted only when requested.
class IFC_PARSE_API FileReader {
private:
#ifdef USE_MMAP
boost::iostreams::mapped_file_source mfs;
#endif
FILE* stream_;
const char* buffer_;
size_t ptr_;
size_t len_;
size_t buf_size_;
size_t ptr_offset_ = 0;
public:
bool valid;
bool eof;
size_t size;
FileReader(const std::string& path, bool mmap = false, size_t buf_size = 0);
FileReader(std::istream& stream, int length);
FileReader(void* data, int length);
~FileReader();
/// Returns the character at the cursor
char peek();
/// Returns the character at specified offset
char Read(size_t offset);
/// Increment the file cursor and reads new page if necessary
void increment();
void Close();
/// Moves the file cursor to an arbitrary offset in the file
void seek(size_t offset);
/// Returns the cursor position
size_t Tell() const;
bool is_eof_at(size_t) const;
void increment_at(size_t&);
char peek_at(size_t);
operator bool() const { return valid && !eof; }
};
} // namespace IfcParse
#endif
*/
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
#include <variant>
#include <stdexcept>
#include <vector>
namespace IfcParse {
/// \brief Read-only file accessor that supports four backends:
/// - full-buffer in RAM
/// - paged (LRU-cached with capacity)
/// - memory-mapped via boost::iostreams
/// - user-pushed sequential pages (caller feeds pages intended for streaming reads in WASM)
class IFC_PARSE_API FileReader {
public:
struct Page { std::vector<char> data; };
/// \brief Tag to choose memory-mapped backend via boost::iostreams::mapped_file_source.
struct mmap_tag {};
/// \brief Tag to choose user-pushed sequential pages.
struct caller_fed_tag {};
/// \brief Construct a FileReader.
/// \param fn File path.
/// \param maybe_mmaped_or_chunked
/// - std::nullopt: full-buffer mode (entire file loaded into memory)
/// - size_t: paged mode with this page size in bytes (e.g., 4096)
/// - mmap_tag: memory-mapped mode using boost::iostreams::mapped_file_source
/// \param page_cache_capacity Number of pages to keep in the LRU when in paged mode.
FileReader(const std::string& fn);
FileReader(const std::string& fn, const mmap_tag&);
FileReader(const caller_fed_tag&);
FileReader(const std::string& content, const caller_fed_tag&);
FileReader(const std::string& fn, size_t page_size, size_t page_capacity);
/// \brief Copy-construct a new reader sharing the underlying storage but with its own cursor.
FileReader clone() const;
/// \brief Seek to an absolute byte position.
/// \throws std::out_of_range if pos > size().
void seek(size_t pos);
/// \brief Return the current cursor position.
size_t tell() const;
/// \brief Total file size in bytes.
size_t size() const;
/// \brief Peek the byte at the current cursor.
/// \throws std::out_of_range at EOF.
char peek() const;
/// \brief Advance the cursor by n bytes (default 1).
/// \throws std::out_of_range if advancing crosses EOF.
void increment(size_t n = 1);
/// \brief Push the next sequential page (pushed backend only).
/// \param data Contents of the page.
/// \throws std::logic_error if the current backend is not pushed mode, or if a next page is already queued.
void push_next_page(const std::string& data);
/// \brief Returns true if the cursor is at or beyond the end of available data.
/// For the pushed backend, EOF means all pushed bytes have been consumed.
bool eof() const;
/// \brief Equivalent of peek() followed by increment(1)
char read();
/// \brief Equivalent of peek() followed by increment(1)
char get(size_t offset) const;
struct Impl {
virtual ~Impl() = default;
virtual size_t size() const = 0;
virtual char get(size_t pos) const = 0;
/// \brief Backend may support pushing pages; default throws.
virtual void push_next_page(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
}
};
private:
std::shared_ptr<Impl> impl_;
size_t cursor_ = 0;
};
} // namespace IfcParse
#endif
src/ifcparse/IfcCharacterDecoder.cpp
+18 -55
View File
@@ -29,7 +29,7 @@
#include "IfcCharacterDecoder.h"
#include "IfcException.h"
#include "IfcSpfStream.h"
#include "FileReader.h"
#include "IfcLogger.h"
#include <codecvt>
@@ -77,7 +77,7 @@
using namespace IfcParse;
IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::IfcSpfStream* stream) {
IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::FileReader* stream) {
stream_ = stream;
codepage_ = 0;
}
@@ -86,49 +86,9 @@ IfcCharacterDecoder::~IfcCharacterDecoder() {
}
namespace {
unsigned int reference_helper = 0;
std::string read_string(IfcParse::FileReader& stream_, IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
std::u32string builder_;
class pure_impure_helper {
private:
bool pure_;
IfcParse::IfcSpfStream* stream_;
unsigned int& pointer_;
std::u32string builder_;
char peek() {
if (pure_) {
return stream_->peek_at(pointer_);
}
return stream_->Peek();
}
unsigned int tell() {
if (pure_) {
return pointer_;
}
return stream_->Tell();
}
void increment() {
if (pure_) {
stream_->increment_at(pointer_);
} else {
stream_->Inc();
}
}
public:
pure_impure_helper(IfcParse::IfcSpfStream* stream)
: pure_(false),
stream_(stream),
pointer_(reference_helper) {}
pure_impure_helper(IfcParse::IfcSpfStream* stream, unsigned int& pointer)
: pure_(true),
stream_(stream),
pointer_(pointer) {}
std::string get(IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
unsigned int parse_state = 0;
builder_.clear();
builder_.push_back('\'');
@@ -137,7 +97,7 @@ class pure_impure_helper {
unsigned int hex = 0;
unsigned int hex_count = 0;
while ((current_char = peek()) != 0) {
while ((current_char = stream_.peek()) != 0) {
if (EXPECTS_CHARACTER(parse_state)) {
builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80));
parse_state = 0;
@@ -178,7 +138,7 @@ class pure_impure_helper {
} else if (IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state)) {
if (IS_LOWERCASE_HEX(current_char)) {
Logger::Warning("Lowercase hexadecimal character '" + std::string(1, current_char) +
"' found at offset " + std::to_string(pointer_) +
"' found at offset " + std::to_string(stream_.tell()) +
". It is recommended to use uppercase for hexadecimal.");
}
hex <<= 4;
@@ -202,12 +162,12 @@ class pure_impure_helper {
if (parse_state == APOSTROPHE && current_char != '\'') {
break;
}
throw IfcInvalidTokenException(tell(), current_char);
throw IfcInvalidTokenException(stream_.tell(), current_char);
} else {
parse_state = hex = hex_count = 0;
builder_.push_back(current_char);
}
increment();
stream_.increment();
}
builder_.push_back('\'');
@@ -248,22 +208,25 @@ class pure_impure_helper {
}
throw IfcParse::IfcException("Invalid conversion mode");
}
};
} // namespace
IfcCharacterDecoder::operator std::string() {
return pure_impure_helper(stream_).get(mode, substitution_character);
return read_string(*stream_, mode, substitution_character);
}
std::string IfcCharacterDecoder::get(unsigned int& ptr) {
return pure_impure_helper(stream_, ptr).get(mode, substitution_character);
std::string IfcCharacterDecoder::get(size_t& ptr) {
auto local_stream = *stream_;
local_stream.seek(ptr);
auto s = read_string(local_stream, mode, substitution_character);
ptr = local_stream.tell();
return s;
}
void IfcCharacterDecoder::skip() {
unsigned int parse_state = 0;
char current_char;
unsigned int hex_count = 0;
while ((current_char = stream_->Peek()) != 0) {
while ((current_char = stream_->peek()) != 0) {
if (EXPECTS_CHARACTER(parse_state)) {
parse_state = 0;
} else if (current_char == '\'' && (parse_state == 0U)) {
@@ -318,11 +281,11 @@ void IfcCharacterDecoder::skip() {
if (parse_state == APOSTROPHE && current_char != '\'') {
break;
}
throw IfcInvalidTokenException(stream_->Tell(), current_char);
throw IfcInvalidTokenException(stream_->tell(), current_char);
} else {
parse_state = hex_count = 0;
}
stream_->Inc();
stream_->increment();
}
}
src/ifcparse/IfcCharacterDecoder.h
+4 -4
View File
@@ -27,7 +27,7 @@
#ifndef IFCCHARACTERDECODER_H
#define IFCCHARACTERDECODER_H
#include "IfcSpfStream.h"
#include "FileReader.h"
#include <string>
@@ -41,7 +41,7 @@ namespace IfcParse {
class IFC_PARSE_API IfcCharacterDecoder {
private:
IfcParse::IfcSpfStream* stream_;
IfcParse::FileReader* stream_;
int codepage_;
public:
@@ -52,7 +52,7 @@ class IFC_PARSE_API IfcCharacterDecoder {
};
static ConversionMode mode;
static char substitution_character;
IfcCharacterDecoder(IfcParse::IfcSpfStream* stream);
IfcCharacterDecoder(IfcParse::FileReader* stream);
~IfcCharacterDecoder();
// Only advances the underlying token stream read pointer
// to the next token.
@@ -62,7 +62,7 @@ class IFC_PARSE_API IfcCharacterDecoder {
operator std::string();
// Gets a decoded string representation at the offset provided,
// does not mutate the underlying token stream read pointer.
std::string get(unsigned int&);
std::string get(size_t&);
};
} // namespace IfcParse
src/ifcparse/IfcEntityInstanceData.h
+1 -1
View File
@@ -155,7 +155,7 @@ struct TypeEncoder_t<parameter_pack<Types...>> {
using TypeEncoder = TypeEncoder_t<type_variant_parameter_pack>;
struct IFC_PARSE_API MutableAttributeValue {
int name_;
uint32_t name_;
uint8_t index_;
};
src/ifcparse/IfcException.h
+2 -2
View File
@@ -57,7 +57,7 @@ class IFC_PARSE_API IfcAttributeOutOfRangeException : public IfcException {
class IFC_PARSE_API IfcInvalidTokenException : public IfcException {
public:
IfcInvalidTokenException(
int token_start,
size_t token_start,
const std::string& token_string,
const std::string& expected_type)
: IfcException(
@@ -65,7 +65,7 @@ class IFC_PARSE_API IfcInvalidTokenException : public IfcException {
boost::lexical_cast<std::string>(token_start) +
" invalid " + expected_type) {}
IfcInvalidTokenException(
int token_start,
size_t token_start,
char character)
: IfcException(
std::string("Unexpected '") + std::string(1, character) + "' at offset " +
src/ifcparse/IfcFile.cpp
+16 -12
View File
@@ -56,7 +56,7 @@ namespace {
constexpr bool is_type_in_variant_v = is_type_in_variant<Variant, T>::value;
template <typename Fn>
void dispatch_token(int instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) {
void dispatch_token(boost::optional<size_t> instance_id, int attribute_id, IfcParse::Token t, IfcParse::declaration* decl, Fn fn) {
if (t.type == IfcParse::Token_BINARY) {
fn(IfcParse::TokenFunc::asBinary(t));
} else if (IfcParse::TokenFunc::isBool(t)) {
@@ -89,7 +89,7 @@ namespace {
}
template <size_t Depth, typename Fn>
void construct_(int instance_id, int attribute_id, IfcParse::parse_context& p, const IfcParse::aggregation_type* aggr, Fn fn) {
void construct_(boost::optional<size_t> instance_id, int attribute_id, IfcParse::parse_context& p, const IfcParse::aggregation_type* aggr, Fn fn) {
if (p.tokens_.empty()) {
// @todo instead of ugly if-else we could also default initialize the respective
// variant types below.
@@ -234,7 +234,7 @@ namespace {
}
}
IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count) {
IfcEntityInstanceData IfcParse::parse_context::construct(boost::optional<size_t> name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count) {
std::vector<const IfcParse::parameter_type*> parameter_types;
std::unique_ptr<IfcParse::named_type> transient_named_type;
@@ -259,7 +259,7 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
expected_size && *expected_size != tokens_.size())
{
size_t expected = expected_size ? *expected_size : parameter_types.size();
Logger::Warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + " for instance #" + std::to_string(name > 0 ? name : 0));
Logger::Warning("Expected " + std::to_string(expected) + " attribute values, found " + std::to_string(tokens_.size()) + (name ? std::string(" for instance #" + std::to_string(*name)) : std::string("")));
}
if (tokens_.empty()) {
@@ -289,12 +289,12 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::Token>) {
dispatch_token(name, index, v, param_type && param_type->as_named_type() ? param_type->as_named_type()->declared_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](auto v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::reference_or_simple_type>) {
if (name > 0) {
if (name) {
references_to_resolve.push_back(std::make_pair(
// @todo previously this was storage but apparently the
// pointer is not constant with the moving and temporary nature
// maybe it ought to be and in that case a pointer is more direct
MutableAttributeValue{ name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index },
MutableAttributeValue{ (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t) resolve_reference_index },
v
));
}
@@ -311,12 +311,12 @@ IfcEntityInstanceData IfcParse::parse_context::construct(int name, unresolved_re
}
construct_<0>(name, index, *v, pt ? pt->as_aggregation_type() : nullptr, [this, &storage, name, &references_to_resolve, index, resolve_reference_index](const auto& v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::vector<reference_or_simple_type>>) {
if (name > 0) {
references_to_resolve.push_back({ {name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
if (name) {
references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
}
} else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::vector<std::vector<reference_or_simple_type>>>) {
if (name > 0) {
references_to_resolve.push_back({ {name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
if (name) {
references_to_resolve.push_back({ { (uint32_t) *name, resolve_reference_index == -1 ? index : (uint8_t)resolve_reference_index }, v });
}
} else {
storage.set(index, v);
@@ -652,6 +652,8 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
}
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> IfcParse::InstanceStreamer::read_instance() {
// std::cout << "global: " << stream_->tell() << std::endl;
std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstanceData>> return_value;
if (header_ && yielded_header_instances_ < 3) {
@@ -679,7 +681,7 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstance
}
unsigned current_id = 0;
while (good_ && !lexer_->stream->eof && !current_id) {
while (good_ && !lexer_->stream->eof() && !current_id) {
if (token_stream_[0].type == IfcParse::Token_IDENTIFIER &&
token_stream_[1].type == IfcParse::Token_OPERATOR &&
token_stream_[1].value_char == '=' &&
@@ -734,11 +736,13 @@ std::optional<std::tuple<size_t, const IfcParse::declaration*, IfcEntityInstance
Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
}
if (!lexer_->stream->eof && next_token.type == Token_NONE) {
if (!lexer_->stream->eof() && next_token.type == Token_NONE) {
good_ = file_open_status::INVALID_SYNTAX;
break;
}
// std::cout << next_token.startPos << " " << TokenFunc::toString(next_token) << std::endl;
token_stream_.push_back(next_token);
}
src/ifcparse/IfcFile.h
+44 -9
View File
@@ -88,12 +88,11 @@ IFC_PARSE_API filetype guess_file_type(const std::string& fn);
class IFC_PARSE_API InstanceStreamer {
private:
IfcSpfStream* stream_;
FileReader* stream_;
IfcSpfLexer* lexer_;
IfcSpfHeader* header_;
boost::circular_buffer<Token> token_stream_;
const IfcParse::schema_definition* schema_;
const IfcParse::declaration* ifcroot_type_;
IfcParse::impl::in_memory_file_storage storage_;
IfcParse::file_open_status good_ = IfcParse::file_open_status::SUCCESS;
int progress_;
@@ -104,7 +103,7 @@ public:
bool coerce_attribute_count = true;
operator bool() const {
return good_ && !lexer_->stream->eof;
return good_ && !lexer_->stream->eof();
}
IfcParse::file_open_status status() const {
@@ -131,7 +130,13 @@ public:
return storage_.steal_instances();
}
InstanceStreamer(const std::string& fn);
bool has_semicolon() const;
void push_page(const std::string& page);
InstanceStreamer();
InstanceStreamer(const std::string& fn, bool mmap=false);
InstanceStreamer(void* data, int length);
@@ -201,14 +206,44 @@ private:
public:
#ifdef USE_MMAP
IfcFile(const std::string& path, bool mmap = false);
#else
IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false);
/// <summary>
/// Constructs an IfcFile object from a file path, optionally using memory-mapped I/O, only supports IFC-SPF files.
/// </summary>
/// <param name="path">UTF-8 file path to an IFC-SPF file</param>
/// <param name="mmap">Whether to use memory-mapped I/O</param>
IfcFile(const std::string& path, bool mmap);
#endif
/// <summary>
/// Constructs an IfcFile object from a file path, supports IFC-SPF and the IfcOpenShell-specific RocksDB format.
/// </summary>
/// <param name="path">UTF-8 file path to an IFC-SPF file or RocksDB database directory</param>
/// <param name="ty">File type of the path</param>
/// <param name="readonly">Whether to open in read-only mode, only supported on RocksDB databases</param>
IfcFile(const std::string& path, filetype ty=FT_AUTODETECT, bool readonly=false);
/// <summary>
/// Constructs an IfcFile object from a stream containing IFC-SPF data.
/// </summary>
IfcFile(std::istream& stream, int length);
/// <summary>
/// Constructs an IfcFile object from a memory buffer containing IFC-SPF data.
/// </summary>
IfcFile(void* data, int length);
IfcFile(IfcParse::IfcSpfStream* stream);
// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
/// <summary>
/// Constructs an IfcFile object from a given IFC SPF stream.
/// </summary>
/// <param name="stream">A pointer to an IfcParse::FileReader object representing the input IFC SPF data stream.</param>
IfcFile(IfcParse::FileReader* stream);
/// <summary>
/// Constructs an IfcFile object with the specified schema, file type, and file path.
/// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
/// </summary>
/// <param name="schema">Pointer to the schema definition to use. Defaults to the IFC4 schema if not specified.</param>
/// <param name="ty">The file type to use for the file. Defaults to FT_AUTODETECT.</param>
/// <param name="path">The file system path to the IFC file. Defaults to an empty string.</param>
IfcFile(const IfcParse::schema_definition* schema = IfcParse::schema_by_name("IFC4"), filetype ty = FT_AUTODETECT, const std::string& path = "");
~IfcFile();
src/ifcparse/IfcParse.cpp
+189 -286
View File
@@ -26,7 +26,7 @@
#include "IfcLogger.h"
#include "IfcSchema.h"
#include "IfcSIPrefix.h"
#include "IfcSpfStream.h"
#include "FileReader.h"
#include "utils.h"
#include <algorithm>
@@ -102,167 +102,7 @@ void init_locale() {
#endif
//
// Opens the file and gets the filesize
//
#ifdef USE_MMAP
IfcSpfStream::IfcSpfStream(const std::string& path, bool mmap)
#else
IfcSpfStream::IfcSpfStream(const std::string& path)
#endif
: stream_(0),
buffer_(0),
valid(false),
eof(false) {
#ifdef _MSC_VER
std::wstring fn_ws = IfcUtil::path::from_utf8(path);
const wchar_t* fn_wide = fn_ws.c_str();
#ifdef USE_MMAP
if (mmap) {
mfs = boost::iostreams::mapped_file_source(boost::filesystem::wpath(fn_wide));
} else {
#endif
stream_ = _wfopen(fn_wide, L"rb");
#ifdef USE_MMAP
}
#endif
#else
#ifdef USE_MMAP
if (mmap) {
mfs = boost::iostreams::mapped_file_source(path);
} else {
#endif
stream_ = fopen(path.c_str(), "rb");
#ifdef USE_MMAP
}
#endif
#endif
#ifdef USE_MMAP
if (mmap) {
if (!mfs.is_open()) {
return;
}
valid = true;
buffer_ = mfs.data();
ptr_ = 0;
len_ = mfs.size();
} else {
#endif
if (stream_ == NULL) {
return;
}
valid = true;
fseek(stream_, 0, SEEK_END);
size = (unsigned int)ftell(stream_);
rewind(stream_);
char* buffer_rw = new char[size];
len_ = (unsigned int)fread(buffer_rw, 1, size, stream_);
buffer_ = buffer_rw;
eof = len_ == 0;
ptr_ = 0;
fclose(stream_);
stream_ = nullptr;
#ifdef USE_MMAP
}
#endif
}
IfcSpfStream::IfcSpfStream(std::istream& stream, int length)
: stream_(0),
buffer_(0) {
eof = false;
size = length;
char* buffer_rw = new char[size];
stream.read(buffer_rw, size);
buffer_ = buffer_rw;
valid = stream.gcount() == size;
ptr_ = 0;
len_ = length;
}
IfcSpfStream::IfcSpfStream(void* data, int length)
: stream_(0),
buffer_(0) {
eof = false;
size = length;
buffer_ = (char*)data;
valid = true;
ptr_ = 0;
len_ = length;
}
IfcSpfStream::~IfcSpfStream() {
Close();
}
void IfcSpfStream::Close() {
#ifdef USE_MMAP
if (mfs.is_open()) {
mfs.close();
return;
}
#endif
delete[] buffer_;
if (stream_ != nullptr) {
fclose(stream_);
}
}
//
// Seeks an arbitrary position in the file
//
void IfcSpfStream::Seek(unsigned int offset) {
ptr_ = offset;
if (ptr_ >= len_) {
throw IfcException("Reading outside of file limits");
}
eof = false;
}
//
// Returns the character at the cursor
//
char IfcSpfStream::Peek() {
return buffer_[ptr_];
}
//
// Returns the character at specified offset
//
char IfcSpfStream::Read(unsigned int offset) {
return buffer_[offset];
}
//
// Returns the cursor position
//
unsigned int IfcSpfStream::Tell() const {
return ptr_;
}
//
// Increments cursor and reads new chunk if necessary
//
void IfcSpfStream::Inc() {
if (++ptr_ == len_) {
eof = true;
return;
}
const char current = IfcSpfStream::Peek();
if (current == '\n' || current == '\r') {
// NB this is recursive. It might as well be a loop.
IfcSpfStream::Inc();
}
}
IfcSpfLexer::IfcSpfLexer(IfcParse::IfcSpfStream* stream_) {
IfcSpfLexer::IfcSpfLexer(IfcParse::FileReader* stream_) {
stream = stream_;
decoder_ = new IfcCharacterDecoder(stream_);
}
@@ -271,12 +111,12 @@ IfcSpfLexer::~IfcSpfLexer() {
delete decoder_;
}
unsigned int IfcSpfLexer::skipWhitespace() const {
unsigned int index = 0;
while (!stream->eof) {
char character = stream->Peek();
size_t IfcSpfLexer::skipWhitespace() const {
size_t index = 0;
while (!stream->eof()) {
char character = stream->peek();
if ((character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
stream->Inc();
stream->increment();
++index;
} else {
break;
@@ -285,22 +125,22 @@ unsigned int IfcSpfLexer::skipWhitespace() const {
return index;
}
unsigned int IfcSpfLexer::skipComment() const {
char character = stream->Peek();
size_t IfcSpfLexer::skipComment() const {
char character = stream->peek();
if (character != '/') {
return 0;
}
stream->Inc();
character = stream->Peek();
stream->increment();
character = stream->peek();
if (character != '*') {
stream->Seek(stream->Tell() - 1);
stream->seek(stream->tell() - 1);
return 0;
}
unsigned int index = 2;
size_t index = 2;
char intermediate = 0;
while (!stream->eof) {
character = stream->Peek();
stream->Inc();
while (!stream->eof()) {
character = stream->peek();
stream->increment();
++index;
if (character == '/' && intermediate == '*') {
break;
@@ -315,19 +155,20 @@ unsigned int IfcSpfLexer::skipComment() const {
//
Token IfcSpfLexer::Next() {
if (stream->eof) {
if (stream->eof()) {
return Token{};
}
while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
}
if (stream->eof) {
if (stream->eof()) {
return Token{};
}
unsigned int pos = stream->Tell();
char character = stream->Peek();
auto& str = GetTempString();
auto pos = stream->tell();
char character = stream->read();
// If the cursor is at [()=,;$*] we know token consists of single char
if (character == '(' ||
@@ -336,69 +177,45 @@ Token IfcSpfLexer::Next() {
character == ',' ||
character == ';' ||
character == '$' ||
character == '*') {
stream->Inc();
return OperatorTokenPtr(this, pos, pos + 1);
character == '*')
{
return OperatorTokenPtr(this, pos, character);
}
int len = 0;
while (!stream->eof) {
// Read character and increment pointer if not starting a new token
character = stream->Peek();
if ((len != 0) && (character == '(' ||
character == ')' ||
character == '=' ||
character == ',' ||
character == ';' ||
character == '/')) {
break;
}
stream->Inc();
len++;
if (character == '\'') {
// If a string is encountered defer processing to the IfcCharacterDecoder
if (character == '\'') {
decoder_->skip();
str = *decoder_;
} else {
str.assign(&character, 1);
while (!stream->eof()) {
// Read character and increment pointer if not starting a new token
character = stream->peek();
if (character == '(' ||
character == ')' ||
character == '=' ||
character == ',' ||
character == ';' ||
character == '/') {
break;
}
str.push_back(character);
stream->increment();
}
}
Token t;
if (len != 0) {
t = GeneralTokenPtr(this, pos, stream->Tell());
} else {
t = Token{};
}
// std::wcout << "token: " << pos << " " << TokenFunc::asStringRef(t).c_str() << std::endl;
return t;
}
bool IfcSpfStream::is_eof_at(unsigned int local_ptr) const {
return local_ptr >= len_;
}
void IfcSpfStream::increment_at(unsigned int& local_ptr) {
if (++local_ptr == len_) {
return;
}
const char current = IfcSpfStream::peek_at(local_ptr);
if (current == '\n' || current == '\r') {
IfcSpfStream::increment_at(local_ptr);
}
}
char IfcSpfStream::peek_at(unsigned int local_ptr) {
return buffer_[local_ptr];
return GeneralTokenPtr(this, pos, str);
}
//
// Reads a std::string from the file at specified offset
// Omits whitespace and comments
//
void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
void IfcSpfLexer::TokenString(size_t offset, std::string& buffer) {
buffer.clear();
while (!stream->is_eof_at(offset)) {
char character = stream->peek_at(offset);
auto local_stream = *this->stream;
local_stream.seek(offset);
while (!local_stream.eof()) {
char character = local_stream.peek();
if (!buffer.empty() && (character == '(' ||
character == ')' ||
character == '=' ||
@@ -407,7 +224,7 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
character == '/')) {
break;
}
stream->increment_at(offset);
local_stream.increment();
if (character == ' ' ||
character == '\r' ||
character == '\n' ||
@@ -416,6 +233,7 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
}
if (character == '\'') {
// todo, make decoder use local offset ptr
auto offset = local_stream.tell();
buffer = decoder_->get(offset);
break;
}
@@ -424,10 +242,11 @@ void IfcSpfLexer::TokenString(unsigned int offset, std::string& buffer) {
}
//Note: according to STEP standard, there may be newlines in tokens
inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned end, std::string& oDestination) {
/*
inline void RemoveTokenSeparators(FileReader* stream, size_t start, size_t end, std::string& oDestination) {
oDestination.clear();
for (unsigned i = start; i < end; i++) {
char character = stream->Read(i);
char character = stream->get(i);
if (character == ' ' ||
character == '\r' ||
character == '\n' ||
@@ -437,6 +256,7 @@ inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned
oDestination += character;
}
}
*/
bool ParseInt(const char* pStart, int& val) {
char* pEnd;
@@ -481,22 +301,17 @@ bool ParseBool(const char* pStart, int& val) {
return true;
}
Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) {
char first = lexer->stream->Read(start);
Token token(lexer, start, end, Token_OPERATOR);
token.value_char = first;
Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, size_t start, char data) {
Token token(lexer, start, Token_OPERATOR);
token.value_char = data;
return token;
}
Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) {
Token token(lexer, start, end, Token_NONE);
//extract token into temp buffer (remove eol-s, no encoding changes)
std::string& tokenStr = lexer->GetTempString();
RemoveTokenSeparators(lexer->stream, start, end, tokenStr);
Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, size_t start, const std::string& tokenStr) {
Token token(lexer, start, Token_NONE);
//determine type of the token
char first = lexer->stream->Read(start);
const char& first = tokenStr.front();
if (first == '#') {
token.type = Token_IDENTIFIER;
if (!ParseInt(tokenStr.c_str() + 1, token.value_int)) {
@@ -521,6 +336,11 @@ Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end
token.type = Token_KEYWORD;
}
// todo STRING/BINARY/ENUM/KEYWORD not stored
// tokenStr is gone after this.
// Orrrr we could only free pages when after a full instance is formed because after that no
// references to prior token ranges exist anymore.
return token;
}
@@ -676,7 +496,25 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& token) {
std::string TokenFunc::toString(const Token& token) {
std::string result;
token.lexer->TokenString(token.startPos, result);
if (token.type == Token_OPERATOR) {
result.push_back(token.value_char);
} else if (token.type == Token_INT) {
result = std::to_string(token.value_int);
} else if (token.type == Token_BOOL) {
if (token.value_int == 1) {
result = ".T.";
} else if (token.value_int == 0) {
result = ".F.";
} else {
result = ".U.";
}
} else if (token.type == Token_FLOAT) {
std::ostringstream oss;
oss << std::setprecision(15) << token.value_double;
result = oss.str();
} else {
token.lexer->TokenString(token.startPos, result);
}
return result;
}
@@ -684,7 +522,7 @@ std::string TokenFunc::toString(const Token& token) {
// Reads the arguments from a list of token
// Aditionally, registers the ids (i.e. #[\d]+) in the inverse map
//
void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
void IfcParse::impl::in_memory_file_storage::load(boost::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
Token next = tokens->Next();
/*
@@ -708,8 +546,8 @@ void IfcParse::impl::in_memory_file_storage::load(unsigned entity_instance_name,
load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
} else {
return_value++;
if (TokenFunc::isIdentifier(next) && entity) {
register_inverse(entity_instance_name, entity, next.value_int, attribute_index == -1 ? attribute_index_within_data : attribute_index);
if (TokenFunc::isIdentifier(next) && entity && entity_instance_name) {
register_inverse(*entity_instance_name, entity, next.value_int, attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
}
if (TokenFunc::isKeyword(next)) {
@@ -758,10 +596,10 @@ IfcEntityInstanceData IfcParse::impl::in_memory_file_storage::read(unsigned int
}
void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const {
unsigned int old_offset = tokens->stream->Tell();
auto old_offset = tokens->stream->tell();
Token semilocon = tokens->Next();
if (!TokenFunc::isOperator(semilocon, ';')) {
tokens->stream->Seek(old_offset);
tokens->stream->seek(old_offset);
}
}
@@ -783,7 +621,7 @@ void IfcParse::impl::in_memory_file_storage::unregister_inverse(unsigned id_from
namespace {
template <typename T>
std::string to_string_fixed_width(const T& t, size_t w) {
std::string to_string_fixed_width(const T& t, size_t) {
// @todo currently inactive
std::ostringstream oss;
oss << /*std::setfill('0') << std::setw(w) <<*/ t;
@@ -1345,11 +1183,27 @@ IfcUtil::IfcBaseClass::set_attribute_value(const std::string& s, const T& t) {
//
#ifdef USE_MMAP
IfcFile::IfcFile(const std::string& fn, bool mmap) {
IfcSpfStream s(fn, mmap);
storage_ = impl::in_memory_file_storage{};
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s);
std::unique_ptr<FileReader> s;
if (mmap) {
s = std::make_unique<FileReader>(fn, FileReader::mmap_tag{});
} else {
s = std::make_unique<FileReader>(fn);
}
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&*s, schema_, max_id_);
if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
// @todo unify these names, it's already confusing enough as it stands
byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
}
ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
}
#else
#endif
IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
: schema_(nullptr)
, max_id_(0)
@@ -1360,11 +1214,11 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
ty = guess_file_type(path);
}
if (ty == FT_IFCSPF) {
IfcSpfStream s(path);
FileReader s(path);
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
if (good_ = std::get<impl::in_memory_file_storage>(storage_).good_) {
if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
// @todo unify these names, it's already confusing enough as it stands
byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
@@ -1398,13 +1252,18 @@ IfcFile::IfcFile(const std::string& path, filetype ty, bool readonly)
}
ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
}
#endif
IfcFile::IfcFile(std::istream& stream, int length)
: schema_(nullptr)
, max_id_(0)
{
IfcSpfStream s(stream, length);
FileReader s(FileReader::caller_fed_tag{});
std::string string_data;
string_data.resize(length);
stream.read(string_data.data(), length);
s.push_next_page(string_data);
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
@@ -1419,7 +1278,8 @@ IfcFile::IfcFile(void* data, int length)
: schema_(nullptr)
, max_id_(0)
{
IfcSpfStream s(data, length);
FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{});
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_);
good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
@@ -1430,7 +1290,7 @@ IfcFile::IfcFile(void* data, int length)
byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
}
IfcFile::IfcFile(IfcParse::IfcSpfStream* s)
IfcFile::IfcFile(IfcParse::FileReader* s)
: schema_(nullptr)
, max_id_(0)
{
@@ -1475,18 +1335,65 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
setDefaultHeaderValues();
}
IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn)
: stream_(new IfcSpfStream(fn))
bool IfcParse::InstanceStreamer::has_semicolon() const
{
auto local_stream = stream_->clone();
auto local_lexer = IfcSpfLexer(&local_stream);
Token t = local_lexer.Next();
while (t.type != Token_NONE) {
if (TokenFunc::isOperator(t, ';')) {
return true;
}
// probably the issue is that this moves the cursor past the page boundary on the local stream which also affects the global one
// 'freeze' the stream so that no pages are cleared
t = local_lexer.Next();
}
std::cout << "local: " << local_stream.tell() << " global: " << stream_->tell() << std::endl;
return false;
}
void IfcParse::InstanceStreamer::push_page(const std::string& page)
{
std::cout << "global: " << stream_->tell() << std::endl;
stream_->push_next_page(page);
if (good_ == file_open_status::NO_HEADER) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
try {
schema_ = IfcParse::schema_by_name(header_->file_schema()->schema_identifiers().front());
good_ = file_open_status::SUCCESS;
} catch (const IfcParse::IfcException&) {
}
}
storage_.file = nullptr;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
}
}
IfcParse::InstanceStreamer::InstanceStreamer()
: stream_(new FileReader(FileReader::caller_fed_tag{}))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0)
{
init_locale();
good_ = file_open_status::NO_HEADER;
}
IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap)
: stream_(mmap ? new FileReader(fn, FileReader::mmap_tag{}) : new FileReader(fn))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, ifcroot_type_(nullptr)
, progress_(0)
{
init_locale();
good_ = file_open_status::NO_HEADER;
if (*stream_) {
if (stream_->size() && !stream_->eof()) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
try {
@@ -1503,17 +1410,16 @@ IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn)
}
IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length)
: stream_(new IfcSpfStream(data, length))
: stream_(new FileReader(std::string((char*) data, length), FileReader::caller_fed_tag{}))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, ifcroot_type_(nullptr)
, progress_(0)
{
init_locale();
good_ = file_open_status::NO_HEADER;
if (*stream_) {
if (stream_->size() && !stream_->eof()) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema()->schema_identifiers().size() == 1) {
try {
@@ -1535,7 +1441,6 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition*
, header_(nullptr)
, token_stream_(3, Token{})
, schema_(schema)
, ifcroot_type_(schema->declaration_by_name("IfcRoot"))
, progress_(0)
{
init_locale();
@@ -1546,14 +1451,14 @@ IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition*
storage_.references_to_resolve = &references_to_resolve_;
}
void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfStream* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) {
void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id) {
// Initialize a "C" locale for locale-independent
// number parsing. See comment above on line 41.
init_locale();
tokens = nullptr;
if (!s->valid) {
if (!s->size() || s->eof()) {
// @todo set good on parent file
good_ = file_open_status::READ_ERROR;
return;
@@ -1608,7 +1513,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
auto instance = schema->instantiate(std::get<1>(*inst), std::move(std::get<2>(*inst)));
instance->file_ = file;
instance->id_ = current_id;
instance->id_ = (uint32_t) current_id;
if (instance->declaration().is(*ifcroot_type_)) {
try {
@@ -1641,7 +1546,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
}
// byidentity_[instance->identity()] = instance;
byid_.insert({ current_id, instance });
byid_.insert({(uint32_t) current_id, instance });
// @nb cannot assign to byid_;
// byid_[current_id] = instance;
@@ -1693,10 +1598,10 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
} else if (auto* inst = std::get_if<IfcUtil::IfcBaseClass*>(v)) {
byid_[p.first.name_]->data().set_attribute_value(nullptr, nullptr, 0, p.first.index_, *inst);
}
} else if (auto* v = std::get_if<std::vector<reference_or_simple_type>>(&p.second)) {
} else if (auto* vv = std::get_if<std::vector<reference_or_simple_type>>(&p.second)) {
aggregate_of_instance::ptr instances(new aggregate_of_instance);
instances->reserve(v->size());
for (const auto& vi : *v) {
instances->reserve(vv->size());
for (const auto& vi : *vv) {
if (auto* name = std::get_if<InstanceReference>(&vi)) {
auto it = byid_.find(*name);
if (it == byid_.end()) {
@@ -1726,9 +1631,9 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::IfcSpfSt
} else {
Logger::Error("Duplicate definition for instance reference");
}
} else if (auto* v = std::get_if<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
} else if (auto* vvv = std::get_if<std::vector<std::vector<reference_or_simple_type>>>(&p.second)) {
aggregate_of_aggregate_of_instance::ptr instances(new aggregate_of_aggregate_of_instance);
for (const auto& vi : *v) {
for (const auto& vi : *vvv) {
std::vector<IfcUtil::IfcBaseClass*> inner;
for (const auto& vii : vi) {
if (auto* name = std::get_if<InstanceReference>(&vii)) {
@@ -2003,15 +1908,15 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
auto attr = entity->get_attribute_value(i);
IfcUtil::ArgumentType attr_type = attr.type();
IfcParse::declaration* decl = 0;
IfcParse::declaration* potentially_length_measure_decl = 0;
if (entity->declaration().as_entity() != nullptr) {
decl = 0;
potentially_length_measure_decl = 0;
const parameter_type* pt = entity->declaration().as_entity()->attribute_by_index(i)->type_of_attribute();
while (pt->as_aggregation_type() != nullptr) {
pt = pt->as_aggregation_type()->type_of_element();
}
if (pt->as_named_type() != nullptr) {
decl = pt->as_named_type()->declared_type();
potentially_length_measure_decl = pt->as_named_type()->declared_type();
}
}
@@ -2050,7 +1955,7 @@ IfcUtil::IfcBaseClass* IfcFile::addEntity(IfcUtil::IfcBaseClass* entity, int id)
}
new_entity->set_attribute_value(i, new_instances);
} else if ((decl != nullptr) && decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) {
} else if ((potentially_length_measure_decl != nullptr) && potentially_length_measure_decl->is(*schema()->declaration_by_name("IfcLengthMeasure"))) {
if (boost::math::isnan(conversion_factor)) {
std::pair<IfcUtil::IfcBaseClass*, double> this_file_unit = {nullptr, 1.0};
std::pair<IfcUtil::IfcBaseClass*, double> other_file_unit = {nullptr, 1.0};
@@ -2283,8 +2188,6 @@ void IfcFile::process_deletion_(IfcUtil::IfcBaseClass* entity) {
byid_.erase(entity->id());
const IfcParse::declaration* ty = &entity->declaration();
remove_type_ref(entity);
// entity_file_map is in place to prevent duplicate definitions with usage of add().
@@ -2891,7 +2794,7 @@ AttributeValue IfcEntityInstanceData::get_attribute_value(void* storage, const I
if (storage_) {
return AttributeValue(storage_, (uint8_t)index);
} else {
return AttributeValue((IfcParse::impl::rocks_db_file_storage*)storage, identity, decl, index);
return AttributeValue((IfcParse::impl::rocks_db_file_storage*)storage, identity, decl, (uint8_t) index);
}
}
src/ifcparse/IfcParse.h
+9 -9
View File
@@ -32,7 +32,7 @@
#include "ifc_parse_api.h"
#include "IfcBaseClass.h"
#include "IfcCharacterDecoder.h"
#include "IfcSpfStream.h"
#include "FileReader.h"
#include "macros.h"
#include "storage.h"
@@ -103,27 +103,27 @@ class IFC_PARSE_API TokenFunc {
// Functions for creating Tokens from an arbitary file offset
// The first 4 bits are reserved for Tokens of type ()=,;$*
//
Token OperatorTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
Token GeneralTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
Token OperatorTokenPtr(IfcSpfLexer* tokens, size_t start, char data);
Token GeneralTokenPtr(IfcSpfLexer* tokens, size_t start, const std::string& data);
/// A stream of tokens to be read from a IfcSpfStream.
/// A stream of tokens to be read from a FileReader.
class IFC_PARSE_API IfcSpfLexer {
private:
IfcCharacterDecoder* decoder_;
unsigned int skipWhitespace() const;
unsigned int skipComment() const;
size_t skipWhitespace() const;
size_t skipComment() const;
public:
std::string& GetTempString() const {
static my_thread_local std::string string;
return string;
}
IfcSpfStream* stream;
FileReader* stream;
// IfcFile* file;
IfcSpfLexer(IfcSpfStream* stream);
IfcSpfLexer(FileReader* stream);
Token Next();
~IfcSpfLexer();
void TokenString(unsigned int offset, std::string& result);
void TokenString(size_t offset, std::string& result);
};
IFC_PARSE_API aggregate_of_instance::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level = -1);
src/ifcparse/IfcSpfHeader.cpp
+1 -1
View File
@@ -35,7 +35,7 @@ namespace {
parse_context pc;
storage->tokens->Next();
storage->load(-1, nullptr, pc, -1);
return pc.construct(-1, *storage->references_to_resolve, nullptr, s, -1);
return pc.construct(boost::none, *storage->references_to_resolve, nullptr, s, -1);
} else {
// std::unreachable();
return IfcEntityInstanceData(in_memory_attribute_storage(10));
src/ifcparse/IfcSpfStream.h
-84
View File
@@ -1,84 +0,0 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
/*********************************************************************************
* *
* Reads a file and provides functions to access its *
* contents randomly and character by character *
* *
********************************************************************************/
#ifndef IFCSPFSTREAM_H
#define IFCSPFSTREAM_H
#include "ifc_parse_api.h"
#include <string>
#ifdef USE_MMAP
#include <boost/iostreams/device/mapped_file.hpp>
#endif
namespace IfcParse {
/// The IfcSpfStream class represents a ISO 10303-21 IFC-SPF file in memory.
/// The file is interpreted as a sequence of tokens which are lazily
/// interpreted only when requested.
class IFC_PARSE_API IfcSpfStream {
private:
#ifdef USE_MMAP
boost::iostreams::mapped_file_source mfs;
#endif
FILE* stream_;
const char* buffer_;
unsigned int ptr_;
unsigned int len_;
public:
bool valid;
bool eof;
unsigned int size;
#ifdef USE_MMAP
IfcSpfStream(const std::string& path, bool mmap = false);
#else
IfcSpfStream(const std::string& path);
#endif
IfcSpfStream(std::istream& stream, int length);
IfcSpfStream(void* data, int length);
~IfcSpfStream();
/// Returns the character at the cursor
char Peek();
/// Returns the character at specified offset
char Read(unsigned int offset);
/// Increment the file cursor and reads new page if necessary
void Inc();
void Close();
/// Moves the file cursor to an arbitrary offset in the file
void Seek(unsigned int offset);
/// Returns the cursor position
unsigned int Tell() const;
bool is_eof_at(unsigned int) const;
void increment_at(unsigned int&);
char peek_at(unsigned int);
operator bool() const { return valid && !eof; }
};
} // namespace IfcParse
#endif
src/ifcparse/IfcUtil.cpp
+2 -2
View File
@@ -76,8 +76,8 @@ void aggregate_of_instance::push(const aggregate_of_instance::ptr& instance) {
}
}
}
unsigned int aggregate_of_instance::size() const { return (unsigned int)list_.size(); }
void aggregate_of_instance::reserve(unsigned capacity) { list_.reserve((size_t)capacity); }
size_t aggregate_of_instance::size() const { return list_.size(); }
void aggregate_of_instance::reserve(size_t capacity) { list_.reserve(capacity); }
aggregate_of_instance::it aggregate_of_instance::begin() { return list_.begin(); }
aggregate_of_instance::it aggregate_of_instance::end() { return list_.end(); }
IfcUtil::IfcBaseClass* aggregate_of_instance::operator[](int i) {
src/ifcparse/aggregate_of_instance.h
+3 -3
View File
@@ -48,8 +48,8 @@ class IFC_PARSE_API aggregate_of_instance {
it begin();
it end();
IfcUtil::IfcBaseClass* operator[](int index);
unsigned int size() const;
void reserve(unsigned capacity);
size_t size() const;
void reserve(size_t capacity);
bool contains(IfcUtil::IfcBaseClass*) const;
template <class U>
@@ -81,7 +81,7 @@ class aggregate_of {
}
it begin() { return list_.begin(); }
it end() { return list_.end(); }
unsigned int size() const { return (unsigned int)list_.size(); }
size_t size() const { return list_.size(); }
aggregate_of_instance::ptr generalize() {
aggregate_of_instance::ptr result(new aggregate_of_instance());
for (it i = begin(); i != end(); ++i) {
src/ifcparse/storage.h
+7 -7
View File
@@ -125,7 +125,7 @@ namespace IfcParse {
class IfcFile;
class IfcSpfLexer;
class IfcSpfStream;
class FileReader;
enum TokenType {
Token_NONE,
@@ -142,7 +142,7 @@ namespace IfcParse {
struct Token {
IfcSpfLexer* lexer; //TODO: remove it from here
unsigned startPos;
size_t startPos;
TokenType type;
union {
char value_char; //types: OPERATOR
@@ -154,7 +154,7 @@ namespace IfcParse {
startPos(0),
type(Token_NONE) {
}
Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned /*_endPos*/, TokenType _type)
Token(IfcSpfLexer* _lexer, size_t _startPos, TokenType _type)
: lexer(_lexer),
startPos(_startPos),
type(_type) {
@@ -184,7 +184,7 @@ namespace IfcParse {
void push(IfcUtil::IfcBaseClass* inst);
IfcEntityInstanceData construct(int name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count=true);
IfcEntityInstanceData construct(boost::optional<size_t> name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, boost::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count=true);
};
namespace impl {
@@ -195,7 +195,7 @@ namespace IfcParse {
}
IfcParse::IfcSpfLexer* tokens;
// IfcParse::IfcSpfStream* stream;
// IfcParse::FileReader* stream;
// Either one of these needs to be set
IfcParse::IfcFile* file;
@@ -260,7 +260,7 @@ namespace IfcParse {
entities_by_ref_t byref_excl_;
entity_instance_by_guid_t byguid_;
void load(unsigned entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void load(boost::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
@@ -268,7 +268,7 @@ namespace IfcParse {
// @todo is this still used
IfcEntityInstanceData read(unsigned int index);
void read_from_stream(IfcParse::IfcSpfStream* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id);
file_open_status good_ = file_open_status::SUCCESS;