Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-09 17:31:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Try some things: (a) fewer allocations - parse context pool; lexer string pool (b) SWAR process multiple chars at once in keywords/enums/strs/stc.

Browse Source
This commit is contained in:
Thomas Krijnen
2026-03-27 20:45:13 +01:00
parent fe6e9d86ae
commit 603cedc487
14 changed files with 1266 additions and 860 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcconvert/IfcConvert.cpp
+2 -2
View File
@@ -522,9 +522,9 @@ int main(int argc, char** argv) {
#ifdef HAVE_ICU
if (!unicode_mode.empty()) {
if (unicode_mode == "utf8") {
IfcParse::IfcCharacterDecoder::mode = IfcParse::IfcCharacterDecoder::UTF8;
IfcParse::RuntimeIfcCharacterDecoder::mode = IfcParse::RuntimeIfcCharacterDecoder::UTF8;
} else if (unicode_mode == "escape") {
IfcParse::IfcCharacterDecoder::mode = IfcParse::IfcCharacterDecoder::JSON;
IfcParse::RuntimeIfcCharacterDecoder::mode = IfcParse::RuntimeIfcCharacterDecoder::ESCAPE;
} else {
cerr_ << "[Error] Invalid value for --unicode" << std::endl;
print_options(serializer_options);
src/ifcparse/FileReader.cpp
+291 -252
View File
@@ -1,18 +1,13 @@
#include "FileReader.h"
#include <algorithm>
#include <cstring>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <list>
#include <stdexcept>
#include <unordered_map>
#include <utility>
#include <vector>
#include <deque>
#ifdef USE_MMAP
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/filesystem/path.hpp>
#endif
@@ -21,306 +16,350 @@
namespace {
#if defined(_WIN32)
inline void file_seek_abs(FILE* f, std::uint64_t off) {
if (_fseeki64(f, static_cast<long long>(off), SEEK_SET) != 0)
throw std::runtime_error("fseek failed");
inline void file_seek_abs(FILE* f, std::uint64_t off) {
if (_fseeki64(f, static_cast<long long>(off), SEEK_SET) != 0) {
throw std::runtime_error("fseek failed");
}
}
#else
inline void file_seek_abs(FILE* f, std::uint64_t off) {
if (fseeko(f, static_cast<off_t>(off), SEEK_SET) != 0)
throw std::runtime_error("fseeko failed");
inline void file_seek_abs(FILE* f, std::uint64_t off) {
if (fseeko(f, static_cast<off_t>(off), SEEK_SET) != 0) {
throw std::runtime_error("fseeko failed");
}
}
#endif
} // namespace
using namespace IfcParse;
struct FullBufferImpl final : FileReader::Impl {
std::vector<char> buf_;
explicit FullBufferImpl(const std::string& fn) {
namespace {
template <typename T, typename Fn>
T gather_value(Fn&& fn) {
char bytes[sizeof(T)];
for (size_t i = 0; i < sizeof(bytes); ++i) {
bytes[i] = fn(i);
}
T value;
std::memcpy(&value, bytes, sizeof(value));
return value;
}
} // namespace
FullBufferImpl::FullBufferImpl(const std::string& fn) {
#ifdef _MSC_VER
std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
const wchar_t* fn_wide = fn_ws.c_str();
auto stream = _wfopen(fn_wide, L"rb");
std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
const wchar_t* fn_wide = fn_ws.c_str();
auto stream = _wfopen(fn_wide, L"rb");
#else
auto stream = fopen(fn.c_str(), "rb");
auto stream = fopen(fn.c_str(), "rb");
#endif
fseek(stream, 0, SEEK_END);
buf_.resize((size_t)ftell(stream));
rewind(stream);
buf_.resize((size_t) fread(buf_.data(), 1, buf_.capacity(), stream));
fclose(stream);
if (!stream) {
throw std::runtime_error("Failed to open file");
}
size_t size() const override { return buf_.size(); }
char get(size_t pos) const override {
if (pos >= buf_.size()) throw std::out_of_range("get out of range");
return buf_[pos];
fseek(stream, 0, SEEK_END);
size_ = (size_t)ftell(stream);
buf_.resize(size_);
rewind(stream);
buf_.resize((size_t)fread(buf_.data(), 1, buf_.capacity(), stream));
fclose(stream);
}
size_t FullBufferImpl::size() const { return size_; }
char FullBufferImpl::get(size_t pos) const {
if (pos >= buf_.size()) {
throw std::out_of_range("get out of range");
}
};
return buf_[pos];
}
struct PagedFileImpl final : FileReader::Impl {
std::string fn_;
FILE* fp_ = nullptr;
size_t file_size_ = 0;
size_t page_size_ = 4096;
uint64_t FullBufferImpl::get_u64(size_t pos) const {
if (pos + sizeof(uint64_t) > buf_.size()) {
throw std::out_of_range("get_u64 out of range");
}
uint64_t value;
std::memcpy(&value, buf_.data() + pos, sizeof(value));
return value;
}
// LRU cache
size_t capacity_ = 8;
mutable std::list<size_t> lru_;
struct Entry {
FileReader::Page page;
std::list<size_t>::iterator it;
};
mutable std::unordered_map<size_t, Entry> map_;
uint32_t FullBufferImpl::get_u32(size_t pos) const {
if (pos + sizeof(uint32_t) > buf_.size()) {
throw std::out_of_range("get_u32 out of range");
}
uint32_t value;
std::memcpy(&value, buf_.data() + pos, sizeof(value));
return value;
}
PagedFileImpl(const std::string& fn, size_t page_size, size_t cap)
: fn_(fn), page_size_(std::max<size_t>(512, page_size)), capacity_(std::max<size_t>(2, cap))
{
void FullBufferImpl::pushNextPage(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
}
void FullBufferImpl::dropPages(size_t) {
}
PagedFileImpl::PagedFileImpl(const std::string& fn, size_t page_size, size_t cap)
: fn_(fn)
, page_size_(std::max<size_t>(512, page_size))
, capacity_(std::max<size_t>(2, cap)) {
#ifdef _MSC_VER
std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
const wchar_t* fn_wide = fn_ws.c_str();
fp_ = _wfopen(fn_wide, L"rb");
std::wstring fn_ws = IfcUtil::path::from_utf8(fn);
const wchar_t* fn_wide = fn_ws.c_str();
fp_ = _wfopen(fn_wide, L"rb");
#else
fp_ = fopen(fn.c_str(), "rb");
fp_ = fopen(fn.c_str(), "rb");
#endif
fseek(fp_, 0, SEEK_END);
file_size_ = (size_t)ftell(fp_);
rewind(fp_);
if (!fp_) {
throw std::runtime_error("Failed to open file");
}
fseek(fp_, 0, SEEK_END);
file_size_ = (size_t)ftell(fp_);
rewind(fp_);
}
PagedFileImpl::~PagedFileImpl() {
if (fp_) {
std::fclose(fp_);
}
fp_ = nullptr;
}
size_t PagedFileImpl::size() const { return file_size_; }
char PagedFileImpl::get(size_t pos) const {
if (pos >= file_size_) {
throw std::out_of_range("get out of range");
}
const size_t pidx = pos / page_size_;
const FileReaderPage& p = fetchPage_(pidx);
const size_t off = pos % page_size_;
if (off >= p.data.size()) {
throw std::out_of_range("offset beyond valid page bytes");
}
return p.data[off];
}
uint64_t PagedFileImpl::get_u64(size_t pos) const {
if (pos + sizeof(uint64_t) > file_size_) {
throw std::out_of_range("get_u64 out of range");
}
~PagedFileImpl() override {
if (fp_) std::fclose(fp_);
fp_ = nullptr;
const size_t pidx = pos / page_size_;
const FileReaderPage& p = fetchPage_(pidx);
const size_t off = pos % page_size_;
if (off + sizeof(uint64_t) <= p.data.size()) {
uint64_t value;
std::memcpy(&value, p.data.data() + off, sizeof(value));
return value;
}
size_t size() const override { return file_size_; }
return gather_value<uint64_t>([&](size_t i) { return get(pos + i); });
}
char get(size_t pos) const override {
if (pos >= file_size_) throw std::out_of_range("get out of range");
const size_t pidx = pos / page_size_;
const FileReader::Page& p = fetchPage_(pidx);
const size_t off = pos % page_size_;
if (off >= p.data.size()) throw std::out_of_range("offset beyond valid page bytes");
return p.data[off];
uint32_t PagedFileImpl::get_u32(size_t pos) const {
if (pos + sizeof(uint32_t) > file_size_) {
throw std::out_of_range("get_u32 out of range");
}
private:
const FileReader::Page& fetchPage_(size_t idx) const {
auto it = map_.find(idx);
if (it != map_.end()) {
touch_(it);
return it->second.page;
const size_t pidx = pos / page_size_;
const FileReaderPage& p = fetchPage_(pidx);
const size_t off = pos % page_size_;
if (off + sizeof(uint32_t) <= p.data.size()) {
uint32_t value;
std::memcpy(&value, p.data.data() + off, sizeof(value));
return value;
}
return gather_value<uint32_t>([&](size_t i) { return get(pos + i); });
}
void PagedFileImpl::pushNextPage(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
}
void PagedFileImpl::dropPages(size_t) {
}
const FileReaderPage& PagedFileImpl::fetchPage_(size_t idx) const {
auto it = map_.find(idx);
if (it != map_.end()) {
touch_(it);
return it->second.page;
}
FileReaderPage pg;
pg.data.resize(page_size_);
const size_t begin = idx * page_size_;
const size_t avail = std::min(page_size_, file_size_ - begin);
file_seek_abs(fp_, begin);
if (avail > 0) {
const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
if (nread != avail) {
throw std::runtime_error("Short fread on page");
}
// Load page from disk using persistent FILE*
FileReader::Page pg;
pg.data.resize(page_size_);
const size_t begin = idx * page_size_;
const size_t avail = std::min(page_size_, file_size_ - begin);
file_seek_abs(fp_, begin);
if (avail > 0) {
const size_t nread = std::fread(pg.data.data(), 1, avail, fp_);
if (nread != avail) throw std::runtime_error("Short fread on page");
}
// trim to actual size
pg.data.resize(avail);
// Insert into LRU
if (map_.size() >= capacity_) evict_();
lru_.push_front(idx);
auto lit = lru_.begin();
auto [emplaced_it, ok] = map_.emplace(idx, Entry{ std::move(pg), lit });
(void)ok;
return emplaced_it->second.page;
}
pg.data.resize(avail);
void touch_(typename std::unordered_map<size_t, Entry>::iterator it) const {
lru_.erase(it->second.it);
lru_.push_front(it->first);
it->second.it = lru_.begin();
if (map_.size() >= capacity_) {
evict_();
}
lru_.push_front(idx);
auto lit = lru_.begin();
auto [emplaced_it, ok] = map_.emplace(idx, Entry{std::move(pg), lit});
(void)ok;
return emplaced_it->second.page;
}
void evict_() const {
if (lru_.empty()) return;
const size_t victim = lru_.back();
lru_.pop_back();
map_.erase(victim);
void PagedFileImpl::touch_(std::unordered_map<size_t, Entry>::iterator it) const {
lru_.erase(it->second.it);
lru_.push_front(it->first);
it->second.it = lru_.begin();
}
void PagedFileImpl::evict_() const {
if (lru_.empty()) {
return;
}
};
const size_t victim = lru_.back();
lru_.pop_back();
map_.erase(victim);
}
#ifdef USE_MMAP
struct MMapImpl final : FileReader::Impl {
boost::iostreams::mapped_file_source map_;
size_t size_ = 0;
explicit MMapImpl(const std::string& fn) {
map_.open(boost::filesystem::path(IfcUtil::path::from_utf8(fn)));
if (!map_.is_open()) throw std::runtime_error("Failed to open mapped_file_source");
size_ = static_cast<size_t>(map_.size());
MMapImpl::MMapImpl(const std::string& fn) {
map_.open(boost::filesystem::path(IfcUtil::path::from_utf8(fn)));
if (!map_.is_open()) {
throw std::runtime_error("Failed to open mapped_file_source");
}
size_ = static_cast<size_t>(map_.size());
}
size_t size() const override { return size_; }
size_t MMapImpl::size() const { return size_; }
char get(size_t pos) const override {
if (pos >= size_) throw std::out_of_range("get out of range");
return map_.data()[pos];
char MMapImpl::get(size_t pos) const {
if (pos >= size_) {
throw std::out_of_range("get out of range");
}
};
return map_.data()[pos];
}
uint64_t MMapImpl::get_u64(size_t pos) const {
if (pos + sizeof(uint64_t) > size_) {
throw std::out_of_range("get_u64 out of range");
}
uint64_t value;
std::memcpy(&value, map_.data() + pos, sizeof(value));
return value;
}
uint32_t MMapImpl::get_u32(size_t pos) const {
if (pos + sizeof(uint32_t) > size_) {
throw std::out_of_range("get_u32 out of range");
}
uint32_t value;
std::memcpy(&value, map_.data() + pos, sizeof(value));
return value;
}
void MMapImpl::pushNextPage(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
}
void MMapImpl::dropPages(size_t) {
}
#endif
/// User-pushed sequential backend with an arbitrary-length queue of future pages.
/// We keep a deque of pages; when reads move forward, we drop fully-consumed
/// pages from the front to release memory.
struct PushedSequentialImpl final : std::enable_shared_from_this<PushedSequentialImpl>, FileReader::Impl {
// Deque of pages, front is earliest in file.
std::deque<FileReader::Page> pages_;
// total bytes in dropped pages
size_t discarded_page_bytes_ = 0;
size_t PushedSequentialImpl::size() const {
size_t n = discarded_page_bytes_;
for (const auto& pg : pages_) {
n += pg.data.size();
}
return n;
}
size_t size() const override {
size_t n = discarded_page_bytes_;
for (auto& pg : pages_) n += pg.data.size();
return n;
void PushedSequentialImpl::dropPages(size_t pos) {
while (!pages_.empty()) {
if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
discarded_page_bytes_ += pages_.front().data.size();
pages_.pop_front();
} else {
break;
}
}
}
char PushedSequentialImpl::get(size_t pos) const {
const size_t avail_end = size();
if (pos >= avail_end) {
throw std::out_of_range("pushed backend: position not committed yet");
}
// Drop fully-consumed pages so pos is guaranteed to be within the first page
void dropPages(size_t pos) override {
while (!pages_.empty()) {
if (pos - discarded_page_bytes_ >= pages_.front().data.size()) {
discarded_page_bytes_ += pages_.front().data.size();
pages_.pop_front();
} else {
break;
pos -= discarded_page_bytes_;
size_t page_start = 0;
for (const auto& pg : pages_) {
if (pos < page_start + pg.data.size()) {
const size_t off = pos - page_start;
return pg.data[off];
}
page_start += pg.data.size();
}
throw std::out_of_range("pushed backend: internal inconsistency");
}
uint64_t PushedSequentialImpl::get_u64(size_t pos) const {
if (pos + sizeof(uint64_t) > size()) {
throw std::out_of_range("get_u64 out of range");
}
size_t relative_pos = pos - discarded_page_bytes_;
size_t page_start = 0;
for (const auto& pg : pages_) {
if (relative_pos < page_start + pg.data.size()) {
const size_t off = relative_pos - page_start;
if (off + sizeof(uint64_t) <= pg.data.size()) {
uint64_t value;
std::memcpy(&value, pg.data.data() + off, sizeof(value));
return value;
}
break;
}
page_start += pg.data.size();
}
char get(size_t pos) const override {
/*
auto self = const_cast<PushedSequentialImpl*>(this);
// We do not do this automatically because all variable width tokens:
// ENUM/STRING/BINARY/KEYWORD are stored as file offsets until a full
// entity instance is finalized.
if (this->shared_from_this().use_count() == 2) {
// only drop pages when there is only one active client.
// NB this->shared_from_this() increases count by 1
self->drop_consumed_up_to(pos);
}
*/
return gather_value<uint64_t>([&](size_t i) { return get(pos + i); });
}
const size_t avail_end = size();
if (pos >= avail_end) throw std::out_of_range("pushed backend: position not committed yet");
uint32_t PushedSequentialImpl::get_u32(size_t pos) const {
if (pos + sizeof(uint32_t) > size()) {
throw std::out_of_range("get_u32 out of range");
}
pos -= discarded_page_bytes_;
size_t page_start = 0;
for (const auto& pg : pages_) {
if (pos < page_start + pg.data.size()) {
const size_t off = pos - page_start;
return pg.data[off];
} else {
page_start += pg.data.size();
size_t relative_pos = pos - discarded_page_bytes_;
size_t page_start = 0;
for (const auto& pg : pages_) {
if (relative_pos < page_start + pg.data.size()) {
const size_t off = relative_pos - page_start;
if (off + sizeof(uint32_t) <= pg.data.size()) {
uint32_t value;
std::memcpy(&value, pg.data.data() + off, sizeof(value));
return value;
}
break;
}
throw std::out_of_range("pushed backend: internal inconsistency");
page_start += pg.data.size();
}
void pushNextPage(const std::string& data) override {
FileReader::Page p; p.data.assign(data.data(), data.data() + data.size());
pages_.push_back(std::move(p));
}
};
IfcParse::FileReader::FileReader(const std::string& fn)
: cursor_(0)
{
impl_ = std::make_shared<FullBufferImpl>(fn);
return gather_value<uint32_t>([&](size_t i) { return get(pos + i); });
}
IfcParse::FileReader::FileReader(const std::string& fn, const mmap_tag&)
: cursor_(0)
{
#ifdef USE_MMAP
impl_ = std::make_shared<MMapImpl>(fn);
#else
(void)fn;
throw std::runtime_error("IfcParse::FileReader: mmap_tag specified but library not compiled with USE_MMAP");
#endif
}
IfcParse::FileReader::FileReader(const caller_fed_tag&)
: cursor_(0)
{
impl_ = std::make_shared<PushedSequentialImpl>();
}
IfcParse::FileReader::FileReader(const std::string& content, const caller_fed_tag&)
{
impl_ = std::make_shared<PushedSequentialImpl>();
impl_->pushNextPage(content);
}
IfcParse::FileReader::FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
: cursor_(0)
{
impl_ = std::make_shared<PagedFileImpl>(fn, page_size, page_capacity);
}
FileReader FileReader::clone() const {
FileReader c(*this);
c.cursor_ = this->cursor_;
return c;
}
void FileReader::seek(size_t pos) {
if (pos > impl_->size()) throw std::out_of_range("seek out of range");
cursor_ = pos;
}
size_t FileReader::tell() const { return cursor_; }
size_t FileReader::size() const { return impl_->size(); }
char FileReader::peek() const {
if (cursor_ >= impl_->size()) throw std::out_of_range("peek at EOF");
return impl_->get(cursor_);
}
void FileReader::increment(size_t n) {
if (cursor_ + n > impl_->size()) throw std::out_of_range("increment past EOF");
cursor_ += n;
}
void IfcParse::FileReader::pushNextPage(const std::string& data)
{
impl_->pushNextPage(data);
}
void IfcParse::FileReader::dropPages()
{
impl_->dropPages(0);
}
void IfcParse::FileReader::dropPages(size_t up_to_pos)
{
impl_->dropPages(up_to_pos);
}
bool IfcParse::FileReader::eof() const
{
return cursor_ >= impl_->size();
}
char IfcParse::FileReader::read()
{
auto c = peek();
increment(1);
return c;
}
char IfcParse::FileReader::get(size_t offset) const
{
return impl_->get(offset);
void PushedSequentialImpl::pushNextPage(const std::string& data) {
FileReaderPage p;
p.data.assign(data.data(), data.data() + data.size());
pages_.push_back(std::move(p));
}
src/ifcparse/FileReader.h
+222 -82
View File
@@ -1,4 +1,4 @@
/********************************************************************************
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
@@ -30,104 +30,244 @@
#include "ifc_parse_api.h"
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <deque>
#include <list>
#include <memory>
#include <optional>
#include <string>
#include <variant>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
#ifdef USE_MMAP
#include <boost/iostreams/device/mapped_file.hpp>
#endif
namespace IfcParse {
/// \brief Read-only file accessor that supports four backends:
/// - full-buffer in RAM
/// - paged (LRU-cached with capacity)
/// - memory-mapped via boost::iostreams
/// - user-pushed sequential pages (caller feeds pages intended for streaming reads in WASM)
class IFC_PARSE_API FileReader {
struct FileReaderPage {
std::vector<char> data;
};
struct caller_fed_tag {};
template <typename>
inline constexpr bool file_reader_dependent_false_v = false;
class IFC_PARSE_API FullBufferImpl;
class IFC_PARSE_API PagedFileImpl;
#ifdef USE_MMAP
class IFC_PARSE_API MMapImpl;
#endif
class IFC_PARSE_API PushedSequentialImpl;
template <typename Impl>
class FileReader {
public:
struct Page { std::vector<char> data; };
using impl_type = Impl;
using Page = FileReaderPage;
/// \brief Tag to choose memory-mapped backend via boost::iostreams::mapped_file_source.
struct mmap_tag {};
/// \brief Tag to choose user-pushed sequential pages.
struct caller_fed_tag {};
FileReader() = default;
/// \brief Construct a FileReader.
/// \param fn File path.
/// \param maybe_mmaped_or_chunked
/// - std::nullopt: full-buffer mode (entire file loaded into memory)
/// - size_t: paged mode with this page size in bytes (e.g., 4096)
/// - mmap_tag: memory-mapped mode using boost::iostreams::mapped_file_source
/// \param page_cache_capacity Number of pages to keep in the LRU when in paged mode.
FileReader(const std::string& fn);
FileReader(const std::string& fn, const mmap_tag&);
FileReader(const caller_fed_tag&);
FileReader(const std::string& content, const caller_fed_tag&);
FileReader(const std::string& fn, size_t page_size, size_t page_capacity);
/// \brief Copy-construct a new reader sharing the underlying storage but with its own cursor.
FileReader clone() const;
/// \brief Seek to an absolute byte position.
/// \throws std::out_of_range if pos > size().
void seek(size_t pos);
/// \brief Return the current cursor position.
size_t tell() const;
/// \brief Total file size in bytes.
size_t size() const;
/// \brief Peek the byte at the current cursor.
/// \throws std::out_of_range at EOF.
char peek() const;
/// \brief Advance the cursor by n bytes (default 1).
/// \throws std::out_of_range if advancing crosses EOF.
void increment(size_t n = 1);
/// \brief Push the next sequential page (pushed backend only).
/// \param data Contents of the page.
/// \throws std::logic_error if the current backend is not in pushed mode
void pushNextPage(const std::string& data);
/// \brief Drops pages up to cursor position or provided offset. Does nothing when current backend is not in pushed mode
/// \param up_to_pos Pages with an end offset before up_to_pos are dropped from memory
void dropPages();
void dropPages(size_t up_to_pos);
/// \brief Returns true if the cursor is at or beyond the end of available data.
/// For the pushed backend, EOF means all pushed bytes have been consumed.
bool eof() const;
/// \brief Equivalent of peek() followed by increment(1)
char read();
/// \brief Equivalent of peek() followed by increment(1)
char get(size_t offset) const;
struct Impl {
virtual ~Impl() = default;
virtual size_t size() const = 0;
virtual char get(size_t pos) const = 0;
/// \brief Backend may support pushing pages; default throws.
virtual void pushNextPage(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
explicit FileReader(const std::string& fn)
: cursor_(0) {
if constexpr (std::is_same_v<Impl, FullBufferImpl>
#ifdef USE_MMAP
|| std::is_same_v<Impl, MMapImpl>
#endif
) {
impl_ = std::make_shared<Impl>(fn);
} else {
static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
}
virtual void dropPages(size_t) {
// empty on purpose
}
explicit FileReader(const caller_fed_tag&)
: cursor_(0) {
if constexpr (std::is_same_v<Impl, PushedSequentialImpl>) {
impl_ = std::make_shared<Impl>();
} else {
static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
}
};
}
FileReader(const std::string& content, const caller_fed_tag&)
: FileReader(caller_fed_tag{}) {
if constexpr (std::is_same_v<Impl, PushedSequentialImpl>) {
impl_->pushNextPage(content);
} else {
static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
}
}
FileReader(const std::string& fn, size_t page_size, size_t page_capacity)
: cursor_(0) {
if constexpr (std::is_same_v<Impl, PagedFileImpl>) {
impl_ = std::make_shared<Impl>(fn, page_size, page_capacity);
} else {
static_assert(file_reader_dependent_false_v<Impl>, "This FileReader constructor is not supported for the selected backend");
}
}
FileReader clone() const {
FileReader c(*this);
c.cursor_ = cursor_;
return c;
}
void seek(size_t pos) {
if (pos > size()) {
throw std::out_of_range("seek out of range");
}
cursor_ = pos;
}
size_t tell() const { return cursor_; }
size_t size() const { return impl_->size(); }
size_t remaining() const { return size() - cursor_; }
char peek() const {
if (cursor_ >= size()) {
throw std::out_of_range("peek at EOF");
}
return impl_->get(cursor_);
}
uint64_t peek_u64() const {
if (remaining() < sizeof(uint64_t)) {
throw std::out_of_range("peek_u64 at EOF");
}
return impl_->get_u64(cursor_);
}
uint32_t peek_u32() const {
if (remaining() < sizeof(uint32_t)) {
throw std::out_of_range("peek_u32 at EOF");
}
return impl_->get_u32(cursor_);
}
void increment(size_t n = 1) {
if (cursor_ + n > size()) {
throw std::out_of_range("increment past EOF");
}
cursor_ += n;
}
void pushNextPage(const std::string& data) {
impl_->pushNextPage(data);
}
void dropPages() {
impl_->dropPages(0);
}
void dropPages(size_t up_to_pos) {
impl_->dropPages(up_to_pos);
}
bool eof() const {
return cursor_ >= size();
}
char read() {
auto c = peek();
increment(1);
return c;
}
char get(size_t offset) const {
return impl_->get(offset);
}
private:
std::shared_ptr<Impl> impl_;
size_t cursor_ = 0;
};
class IFC_PARSE_API FullBufferImpl {
public:
explicit FullBufferImpl(const std::string& fn);
size_t size() const;
char get(size_t pos) const;
uint32_t get_u32(size_t pos) const;
uint64_t get_u64(size_t pos) const;
void pushNextPage(const std::string& data);
void dropPages(size_t pos);
private:
std::vector<char> buf_;
size_t size_;
};
class IFC_PARSE_API PagedFileImpl {
public:
struct Entry {
FileReaderPage page;
std::list<size_t>::iterator it;
};
PagedFileImpl(const std::string& fn, size_t page_size, size_t cap);
~PagedFileImpl();
size_t size() const;
char get(size_t pos) const;
uint32_t get_u32(size_t pos) const;
uint64_t get_u64(size_t pos) const;
void pushNextPage(const std::string& data);
void dropPages(size_t pos);
private:
const FileReaderPage& fetchPage_(size_t idx) const;
void touch_(std::unordered_map<size_t, Entry>::iterator it) const;
void evict_() const;
std::string fn_;
FILE* fp_ = nullptr;
size_t file_size_ = 0;
size_t page_size_ = 4096;
size_t capacity_ = 8;
mutable std::list<size_t> lru_;
mutable std::unordered_map<size_t, Entry> map_;
};
#ifdef USE_MMAP
class IFC_PARSE_API MMapImpl {
public:
explicit MMapImpl(const std::string& fn);
size_t size() const;
char get(size_t pos) const;
uint32_t get_u32(size_t pos) const;
uint64_t get_u64(size_t pos) const;
void pushNextPage(const std::string& data);
void dropPages(size_t pos);
private:
boost::iostreams::mapped_file_source map_;
size_t size_ = 0;
};
#endif
class IFC_PARSE_API PushedSequentialImpl {
public:
size_t size() const;
char get(size_t pos) const;
uint32_t get_u32(size_t pos) const;
uint64_t get_u64(size_t pos) const;
void pushNextPage(const std::string& data);
void dropPages(size_t pos);
private:
std::deque<FileReaderPage> pages_;
size_t discarded_page_bytes_ = 0;
};
} // namespace IfcParse
#endif
src/ifcparse/IfcCharacterDecoder.cpp
+110 -17
View File
@@ -77,18 +77,81 @@
using namespace IfcParse;
IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::FileReader* stream) {
stream_ = stream;
codepage_ = 0;
namespace SWAR {
constexpr uint32_t ONES32 = 0x01010101u;
constexpr uint32_t HIGHS32 = 0x80808080u;
constexpr uint64_t ONES64 = 0x0101010101010101ull;
constexpr uint64_t HIGHS64 = 0x8080808080808080ull;
constexpr uint32_t splat32(unsigned char c) {
return ONES32 * c;
}
IfcCharacterDecoder::~IfcCharacterDecoder() {
constexpr uint64_t splat64(unsigned char c) {
return ONES64 * c;
}
inline uint32_t has_zero_byte(uint32_t x) {
return (x - ONES32) & ~x & HIGHS32;
}
inline uint64_t has_zero_byte(uint64_t x) {
return (x - ONES64) & ~x & HIGHS64;
}
inline uint32_t eq_mask(uint32_t x, uint32_t c) {
return has_zero_byte(x ^ c);
}
inline uint64_t eq_mask(uint64_t x, uint64_t c) {
return has_zero_byte(x ^ c);
}
namespace chars {
constexpr uint32_t apostrophe32 = splat32('\'');
constexpr uint32_t solidus32 = splat32('\\');
constexpr uint64_t apostrophe64 = splat64('\'');
constexpr uint64_t solidus64 = splat64('\\');
} // namespace chars
inline uint32_t has_special_char(uint32_t x) {
return has_zero_byte(x) |
eq_mask(x, chars::apostrophe32) |
eq_mask(x, chars::solidus32) |
(x & HIGHS32);
}
inline uint64_t has_special_char(uint64_t x) {
return has_zero_byte(x) |
eq_mask(x, chars::apostrophe64) |
eq_mask(x, chars::solidus64) |
(x & HIGHS64);
}
inline void append_ascii(std::u32string& builder, const char* bytes, size_t count) {
builder.reserve(builder.size() + count);
for (size_t i = 0; i < count; ++i) {
builder.push_back(static_cast<unsigned char>(bytes[i]));
}
}
} // namespace SWAR
template <typename Reader>
IfcCharacterDecoder<Reader>::IfcCharacterDecoder(Reader* stream) {
stream_ = stream;
codepage_ = 0;
builder_.reserve(1024);
}
template <typename Reader>
IfcCharacterDecoder<Reader>::~IfcCharacterDecoder() {
}
namespace {
std::string read_string(IfcParse::FileReader& stream_, IfcParse::IfcCharacterDecoder::ConversionMode mode, char substitution_character) {
std::u32string builder_;
template <typename Reader>
std::string read_string(std::u32string& builder_, Reader& stream_, typename IfcParse::IfcCharacterDecoder<Reader>::ConversionMode mode, char substitution_character) {
unsigned int parse_state = 0;
builder_.clear();
// builder_.push_back('\'');
@@ -97,7 +160,31 @@ namespace {
unsigned int hex = 0;
unsigned int hex_count = 0;
while ((current_char = stream_.peek()) != 0) {
while (!stream_.eof()) {
if (parse_state == 0U) {
if (stream_.remaining() >= 8) {
uint64_t x = stream_.peek_u64();
if (SWAR::has_special_char(x) == 0) {
SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 8);
stream_.increment(8);
continue;
}
}
if (stream_.remaining() >= 4) {
uint32_t x = stream_.peek_u32();
if (SWAR::has_special_char(x) == 0) {
SWAR::append_ascii(builder_, reinterpret_cast<const char*>(&x), 4);
stream_.increment(4);
continue;
}
}
}
current_char = stream_.peek();
if (current_char == 0) {
break;
}
if (EXPECTS_CHARACTER(parse_state)) {
builder_.push_back(IfcUtil::convert_codepage(codepage, current_char + 0x80));
parse_state = 0;
@@ -171,7 +258,7 @@ namespace {
}
// builder_.push_back('\'');
if (mode == IfcParse::IfcCharacterDecoder::UTF8) {
if (mode == IfcParse::IfcCharacterDecoder<Reader>::UTF8) {
if (builder_.empty()) {
static std::string empty;
return empty;
@@ -183,7 +270,7 @@ namespace {
}
return IfcUtil::convert_utf8(builder_);
}
if (mode == IfcParse::IfcCharacterDecoder::SUBSTITUTE) {
if (mode == IfcParse::IfcCharacterDecoder<Reader>::SUBSTITUTE) {
std::string result;
result.reserve(builder_.size());
std::transform(builder_.begin(), builder_.end(), std::back_inserter(result), [&substitution_character](std::u32string::value_type character) {
@@ -194,7 +281,7 @@ namespace {
});
return result;
}
if (mode == IfcParse::IfcCharacterDecoder::ESCAPE) {
if (mode == IfcParse::IfcCharacterDecoder<Reader>::ESCAPE) {
std::stringstream stream;
stream << std::hex << std::setw(4) << std::setfill('0');
std::for_each(builder_.begin(), builder_.end(), [&stream](std::u32string::value_type character) {
@@ -210,20 +297,26 @@ namespace {
}
} // namespace
IfcCharacterDecoder::operator std::string() {
return read_string(*stream_, mode, substitution_character);
template <typename Reader>
IfcCharacterDecoder<Reader>::operator std::string() {
return read_string(builder_, *stream_, mode, substitution_character);
}
std::string IfcCharacterDecoder::get(size_t& ptr) {
template <typename Reader>
std::string IfcCharacterDecoder<Reader>::get(size_t& ptr) {
auto local_stream = *stream_;
local_stream.seek(ptr);
auto s = read_string(local_stream, mode, substitution_character);
auto s = read_string(builder_, local_stream, mode, substitution_character);
ptr = local_stream.tell();
return s;
}
IfcCharacterDecoder::ConversionMode IfcCharacterDecoder::mode = IfcCharacterDecoder::UTF8;
char IfcCharacterDecoder::substitution_character = '_';
template class IfcCharacterDecoder<FileReader<FullBufferImpl>>;
template class IfcCharacterDecoder<FileReader<PagedFileImpl>>;
template class IfcCharacterDecoder<FileReader<PushedSequentialImpl>>;
#ifdef USE_MMAP
template class IfcCharacterDecoder<MMapFileReader>;
#endif
IfcCharacterEncoder::IfcCharacterEncoder(const std::string& input)
: str_(IfcUtil::convert_utf8(input)) {}
src/ifcparse/IfcCharacterDecoder.h
+7 -4
View File
@@ -39,10 +39,12 @@ IFC_PARSE_API std::u32string convert_utf8(const std::string& string);
namespace IfcParse {
template <typename Reader>
class IFC_PARSE_API IfcCharacterDecoder {
private:
IfcParse::FileReader* stream_;
Reader* stream_;
int codepage_;
std::u32string builder_;
public:
enum ConversionMode {
@@ -50,9 +52,10 @@ class IFC_PARSE_API IfcCharacterDecoder {
UTF8,
ESCAPE
};
static ConversionMode mode;
static char substitution_character;
IfcCharacterDecoder(IfcParse::FileReader* stream);
inline static ConversionMode mode = UTF8;
inline static char substitution_character = '_';
IfcCharacterDecoder(Reader* stream);
~IfcCharacterDecoder();
// Gets a decoded string representation at the token stream
// read pointer and advances the underlying token stream.
src/ifcparse/IfcFile.cpp
+6 -125
View File
@@ -21,9 +21,9 @@ IfcParse::parse_context::~parse_context() {
}
IfcParse::parse_context& IfcParse::parse_context::push() {
auto* pc = new IfcParse::parse_context;
tokens_.push_back(pc);
return *pc;
auto child = pool_->make();
tokens_.emplace_back(child);
return *child;
}
void IfcParse::parse_context::push(Token t) {
@@ -220,7 +220,7 @@ namespace {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::Token>) {
// @todo get aggregate of enumeration
dispatch_token(instance_id, attribute_id, v, aggr && aggr->type_of_element()->as_named_type() ? aggr->type_of_element()->as_named_type()->declared_type() : nullptr, append_to_aggregate_storage);
} else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context*>) {
} else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context_handle>) {
// nested list
if constexpr (Depth < 3) {
construct_<Depth + 1>(instance_id, attribute_id, *v, nullptr, append_to_aggregate_storage);
@@ -245,7 +245,7 @@ std::shared_ptr<InstanceData> IfcParse::parse_context::construct(IfcParse::IfcFi
transient_named_type.reset(new IfcParse::named_type(const_cast<IfcParse::declaration*>(decl)));
parameter_types = { &*transient_named_type };
} else if ((decl != nullptr) && (decl->as_entity() != nullptr)) {
auto entity_attrs = decl->as_entity()->all_attributes();
const auto& entity_attrs = decl->as_entity()->all_attributes();
std::transform(
entity_attrs.begin(),
entity_attrs.end(),
@@ -307,7 +307,7 @@ std::shared_ptr<InstanceData> IfcParse::parse_context::construct(IfcParse::IfcFi
storage.set(index, v);
}
});
} else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context*>) {
} else if constexpr (std::is_same_v<std::decay_t<decltype(v)>, IfcParse::parse_context_handle>) {
const auto *pt = param_type;
if (pt) {
while (pt->as_named_type() && pt->as_named_type()->declared_type()->as_type_declaration()) {
@@ -642,125 +642,6 @@ IfcParse::filetype IfcParse::guess_file_type(const std::string& fn) {
}
}
void IfcParse::InstanceStreamer::bypassTypes(const std::set<std::string>& type_names) {
for (auto& name : type_names) {
try {
types_to_bypass_.push_back(schema_->declaration_by_name(name));
} catch (const IfcException&) {
continue;
}
}
}
std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> IfcParse::InstanceStreamer::readInstance() {
std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> return_value;
if (header_ && yielded_header_instances_ < 3) {
if (yielded_header_instances_ == 0) {
return_value.emplace(
0,
&header_->file_description().declaration(),
header_->file_description().data_weak().lock()
);
} else if (yielded_header_instances_ == 1) {
return_value.emplace(
0,
&header_->file_name().declaration(),
header_->file_name().data_weak().lock()
);
} else if (yielded_header_instances_ == 2) {
return_value.emplace(
0,
&header_->file_schema().declaration(),
header_->file_schema().data_weak().lock()
);
}
yielded_header_instances_ += 1;
return return_value;
}
unsigned current_id = 0;
while (good_ && !lexer_->stream->eof() && !current_id) {
if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER &&
token_stream_[1].type == IfcParse::Token::Token_OPERATOR &&
token_stream_[1].value_char == '=' &&
token_stream_[2].type == IfcParse::Token::Token_KEYWORD) {
current_id = token_stream_[0].as_identifier();
const IfcParse::declaration* entity_type;
try {
entity_type = schema_->declaration_by_name(token_stream_[2].as_string());
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos));
current_id = 0;
goto advance;
}
if (entity_type->as_entity() == nullptr) {
Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos));
goto advance;
}
for (auto& ty : types_to_bypass_) {
if (entity_type->is(*ty)) {
bypassed_instances_.push_back(current_id);
// Why is this a conditional clause in the loop?
current_id = 0;
goto advance;
}
}
parse_context ps;
lexer_->Next();
try {
storage_.load(current_id, entity_type->as_entity(), ps, -1);
} catch (const IfcInvalidTokenException& e) {
good_ = file_open_status::INVALID_SYNTAX;
Logger::Error(e);
break;
}
/// @todo Printing to stdout in a library class feels weird. Maybe move the progress prints to the client code?
// Update the status after every 1000 instances parsed
if (((++progress_) % 1000) == 0) {
std::stringstream ss;
ss << "\r#" << current_id;
Logger::Status(ss.str(), false);
}
auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count);
return_value.emplace(
(size_t)current_id,
entity_type,
data
);
}
advance:
Token next_token;
try {
next_token = lexer_->Next();
} catch (const IfcException& e) {
Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated");
} catch (...) {
Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
}
if (!lexer_->stream->eof() && !next_token) {
good_ = file_open_status::INVALID_SYNTAX;
break;
}
token_stream_.push_back(next_token);
}
// Free pages in front of cursor when variable-width tokens are materialized into entity instance data objects
(stream_ ? stream_ : (lexer_)->stream)->dropPages();
lexer_->resetPool();
return return_value;
}
express::Base IfcParse::impl::rocks_db_file_storage::create(const IfcParse::declaration* decl, int id) {
return express::Base{};
/*
src/ifcparse/IfcFile.h
+19 -17
View File
@@ -34,6 +34,7 @@
#include <boost/circular_buffer.hpp>
#include <iterator>
#include <map>
#include <memory>
#ifdef IFOPSH_WITH_ROCKSDB
@@ -87,10 +88,13 @@ enum filetype {
IFC_PARSE_API filetype guess_file_type(const std::string& fn);
template <typename Reader = FileReader<FullBufferImpl>>
class IFC_PARSE_API InstanceStreamer {
private:
FileReader* stream_;
IfcSpfLexer* lexer_;
std::unique_ptr<Reader> owned_stream_;
Reader* stream_;
std::unique_ptr<IfcSpfLexer<Reader>> lexer_;
std::unique_ptr<IfcSpfHeader> owned_header_;
IfcSpfHeader* header_;
IfcParse::IfcFile* owner_;
boost::circular_buffer<Token> token_stream_;
@@ -100,14 +104,18 @@ private:
int progress_;
IfcParse::unresolved_references references_to_resolve_;
int yielded_header_instances_ = 0;
bool yield_header_instances_ = true;
std::vector<const declaration*> types_to_bypass_;
std::vector<unsigned> bypassed_instances_;
void initialize_header();
IfcSpfHeader& ensure_header();
public:
bool coerce_attribute_count = true;
operator bool() const {
return good_ && !lexer_->stream->eof();
return good_ && lexer_ && !lexer_->stream->eof();
}
IfcParse::file_open_status status() const {
@@ -151,17 +159,17 @@ private:
InstanceStreamer(void* data, int length, IfcParse::IfcFile* f = nullptr);
InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer, IfcParse::IfcFile* f = nullptr);
InstanceStreamer(Reader* stream, IfcParse::IfcFile* f = nullptr);
void bypassTypes(const std::set<std::string>& type_names);
~InstanceStreamer() {
delete stream_;
if (stream_) {
delete lexer_;
}
delete header_;
}
void yieldHeaderInstances(bool value) { yield_header_instances_ = value; }
const IfcParse::schema_definition* schema() const { return schema_; }
const IfcSpfHeader* header() const { return header_; }
~InstanceStreamer() = default;
std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> readInstance();
};
@@ -249,12 +257,6 @@ public:
/// </summary>
IfcFile(void* data, int length);
/// <summary>
/// Constructs an IfcFile object from a given IFC SPF stream.
/// </summary>
/// <param name="stream">A pointer to an IfcParse::FileReader object representing the input IFC SPF data stream.</param>
IfcFile(IfcParse::FileReader* stream);
/// <summary>
/// Constructs an IfcFile object with the specified schema, file type, and file path.
/// @nb path is only used in rocksdb mode, for spf file is in-memory only until write() is called
src/ifcparse/IfcParse.cpp
+489 -177
View File
@@ -102,16 +102,19 @@ void init_locale() {
#endif
IfcSpfLexer::IfcSpfLexer(IfcParse::FileReader* stream_) {
template <typename Reader>
IfcSpfLexer<Reader>::IfcSpfLexer(Reader* stream_) {
stream = stream_;
decoder_ = new IfcCharacterDecoder(stream_);
decoder_ = new IfcCharacterDecoder<Reader>(stream_);
}
IfcSpfLexer::~IfcSpfLexer() {
template <typename Reader>
IfcSpfLexer<Reader>::~IfcSpfLexer() {
delete decoder_;
}
size_t IfcSpfLexer::skipWhitespace() const {
template <typename Reader>
size_t IfcSpfLexer<Reader>::skipWhitespace() const {
size_t index = 0;
while (!stream->eof()) {
char character = stream->peek();
@@ -125,7 +128,8 @@ size_t IfcSpfLexer::skipWhitespace() const {
return index;
}
size_t IfcSpfLexer::skipComment() const {
template <typename Reader>
size_t IfcSpfLexer<Reader>::skipComment() const {
if (stream->eof()) {
return 0;
}
@@ -153,7 +157,8 @@ size_t IfcSpfLexer::skipComment() const {
return index;
}
std::string& IfcSpfLexer::getTempString() const {
template <typename Reader>
std::string& IfcSpfLexer<Reader>::getTempString() const {
const size_t idx = pool_index++;
const size_t slice = idx >> 4;
const size_t offset = idx & 0xF;
@@ -161,6 +166,9 @@ std::string& IfcSpfLexer::getTempString() const {
while (stringpool_.size() <= slice) {
stringpool_.push_back(std::make_unique<std::array<std::string, 16>>());
}
// std::wcout << "Num contexts: " << idx << std::endl;
return (*stringpool_[slice])[offset];
}
@@ -192,17 +200,86 @@ bool parse_float_(const char* pStart, double& val) {
} // namespace
namespace SWAR {
constexpr uint32_t ONES32 = 0x01010101u;
constexpr uint32_t HIGHS32 = 0x80808080u;
constexpr uint64_t ONES = 0x0101010101010101ull;
constexpr uint64_t HIGHS = 0x8080808080808080ull;
constexpr uint64_t splat(unsigned char c) {
return ONES * c;
}
inline uint32_t has_zero_byte(uint32_t x) {
return (x - ONES32) & ~x & HIGHS32;
}
inline uint64_t has_zero_byte(uint64_t x) {
return (x - ONES) & ~x & HIGHS;
}
inline uint32_t eq_mask(uint32_t x, uint32_t c) {
return has_zero_byte(x ^ c);
}
inline uint64_t eq_mask(uint64_t x, uint64_t c) {
return has_zero_byte(x ^ c);
}
namespace chars {
constexpr uint64_t lpar = splat('(');
constexpr uint64_t rpar = splat(')');
constexpr uint64_t eq = splat('=');
constexpr uint64_t comma = splat(',');
constexpr uint64_t semi = splat(';');
constexpr uint64_t slash = splat('/');
constexpr uint64_t space = splat(' ');
constexpr uint64_t cr = splat('\r');
constexpr uint64_t lf = splat('\n');
constexpr uint64_t tab = splat('\t');
constexpr uint64_t quote = splat('"');
constexpr uint64_t dot = splat('.');
} // namespace chars
inline uint64_t has_special_char(uint64_t x) {
return eq_mask(x, chars::lpar) |
eq_mask(x, chars::rpar) |
eq_mask(x, chars::eq) |
eq_mask(x, chars::comma) |
eq_mask(x, chars::semi) |
eq_mask(x, chars::slash) |
eq_mask(x, chars::space) |
eq_mask(x, chars::cr) |
eq_mask(x, chars::lf) |
eq_mask(x, chars::tab) |
eq_mask(x, chars::quote) |
eq_mask(x, chars::dot);
}
inline uint32_t has_special_char(uint32_t x) {
return eq_mask(x, static_cast<uint32_t>(chars::lpar)) |
eq_mask(x, static_cast<uint32_t>(chars::rpar)) |
eq_mask(x, static_cast<uint32_t>(chars::eq)) |
eq_mask(x, static_cast<uint32_t>(chars::comma)) |
eq_mask(x, static_cast<uint32_t>(chars::semi)) |
eq_mask(x, static_cast<uint32_t>(chars::slash)) |
eq_mask(x, static_cast<uint32_t>(chars::space)) |
eq_mask(x, static_cast<uint32_t>(chars::cr)) |
eq_mask(x, static_cast<uint32_t>(chars::lf)) |
eq_mask(x, static_cast<uint32_t>(chars::tab)) |
eq_mask(x, static_cast<uint32_t>(chars::quote)) |
eq_mask(x, static_cast<uint32_t>(chars::dot));
}
}
//
// Returns the offset of the current Token and moves cursor to next
//
Token IfcSpfLexer::Next() {
if (stream->eof()) {
return Token{};
}
while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
}
template <typename Reader>
Token IfcSpfLexer<Reader>::Next() {
if (stream->eof()) {
return Token{};
@@ -211,6 +288,16 @@ Token IfcSpfLexer::Next() {
auto pos = stream->tell();
char character = stream->read();
if (character == '/' || character == ' ' || character == '\r' || character == '\n' || character == '\t') {
while ((skipWhitespace() != 0U) || (skipComment() != 0U)) {
}
if (stream->eof()) {
return Token{};
}
pos = stream->tell();
character = stream->read();
}
// If the cursor is at [()=,;$*] we know token consists of single char
if (character == '(' ||
character == ')' ||
@@ -246,8 +333,25 @@ Token IfcSpfLexer::Next() {
}
while (!stream->eof()) {
if (stream->remaining() >= 8) {
uint64_t x = stream->peek_u64();
if (SWAR::has_special_char(x) == 0) {
str.append(reinterpret_cast<const char*>(&x), 8);
stream->increment(8);
continue;
}
}
if (stream->remaining() >= 4) {
uint32_t x = stream->peek_u32();
if (SWAR::has_special_char(x) == 0) {
str.append(reinterpret_cast<const char*>(&x), 4);
stream->increment(4);
continue;
}
}
// Read character and increment pointer if not starting a new token
character = stream->peek();
char character = stream->peek();
if (character == '(' ||
character == ')' ||
character == '=' ||
@@ -257,7 +361,8 @@ Token IfcSpfLexer::Next() {
break;
}
if (!(character == ' ' || character == '\r' || character == '\n' || character == '\t')) {
if ((ttype == Token::Token_BINARY && character == '"') || (ttype == Token::Token_ENUMERATION && character == '.')) {
if ((ttype == Token::Token_BINARY && character == '"') ||
(ttype == Token::Token_ENUMERATION && character == '.')) {
// Skip
} else {
str.push_back(character);
@@ -300,6 +405,13 @@ Token IfcSpfLexer::Next() {
}
}
template class IfcSpfLexer<FileReader<FullBufferImpl>>;
template class IfcSpfLexer<FileReader<PagedFileImpl>>;
template class IfcSpfLexer<FileReader<PushedSequentialImpl>>;
#ifdef USE_MMAP
template class IfcSpfLexer<FileReader<MMapImpl>>;
#endif
bool Token::is_operator() {
return type == Token_OPERATOR;
}
@@ -458,7 +570,8 @@ std::string Token::to_string() {
// Reads the arguments from a list of token
// Aditionally, registers the ids (i.e. #[\d]+) in the inverse map
//
void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
template <typename Reader>
void IfcParse::impl::in_memory_file_storage::load(IfcParse::IfcSpfLexer<Reader>* tokens, std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context& context, int attribute_index) {
Token next = tokens->Next();
size_t attribute_index_within_data = 0;
@@ -473,7 +586,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
break;
} else if (next.is_operator('(')) {
return_value++;
load(entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
load(tokens, entity_instance_name, entity, context.push(), attribute_index == -1 ? (int) attribute_index_within_data : attribute_index);
} else {
return_value++;
if (next.is_identifier() && entity && entity_instance_name) {
@@ -483,7 +596,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
if (next.is_keyword()) {
try {
const auto* decl = (schema ? schema : file->schema())->declaration_by_name(next.as_string());
parse_context ps;
parse_context ps(&context_pool_);
tokens->Next();
// The only case we know where a defined type contains entity
// instance references is IfcPropertySetDefinitionSet. For
@@ -491,7 +604,7 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
// register inverses to the host entity (and not the defined
// type) and to be able to actually register the references in
// the 2nd pass.
load(entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index);
load(tokens, entity_instance_name, entity, ps, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index);
express::Base simple_type_instance(read_simple_type_instances.emplace_back(
ps.construct(file, entity_instance_name, *references_to_resolve, decl, std::nullopt, attribute_index == -1 ? (int)attribute_index_within_data : attribute_index))
);
@@ -510,7 +623,8 @@ void IfcParse::impl::in_memory_file_storage::load(std::optional<size_t> entity_i
}
}
void IfcParse::impl::in_memory_file_storage::try_read_semicolon() const {
template <typename Reader>
void IfcParse::impl::in_memory_file_storage::try_read_semicolon(IfcParse::IfcSpfLexer<Reader>* tokens) const {
auto old_offset = tokens->stream->tell();
Token semilocon = tokens->Next();
if (!semilocon.is_operator(';')) {
@@ -1094,16 +1208,16 @@ IfcFile::IfcFile(const std::string& fn, bool mmap) {
}
bool IfcParse::IfcFile::initialize(const std::string& fn, bool mmap) {
std::unique_ptr<FileReader> s;
if (mmap) {
s = std::make_unique<FileReader>(fn, FileReader::mmap_tag{});
MMapFileReader s(fn);
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
} else {
s = std::make_unique<FileReader>(fn);
FullBufferFileReader s(fn);
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
}
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&*s, schema_, max_id_, types_to_bypass_loading_);
if ((good_ = std::get<impl::in_memory_file_storage>(storage_).good_)) {
// @todo unify these names, it's already confusing enough as it stands
byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_read_);
@@ -1124,7 +1238,7 @@ bool IfcParse::IfcFile::initialize(const std::string& path, filetype ty, bool re
ty = guess_file_type(path);
}
if (ty == FT_IFCSPF) {
FileReader s(path);
FileReader<FullBufferImpl> s(path);
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
@@ -1181,7 +1295,7 @@ IfcFile::IfcFile(std::istream& stream, int length)
, max_id_(0)
, header_(new IfcParse::IfcSpfHeader(this))
{
FileReader s(FileReader::caller_fed_tag{});
FileReader<PushedSequentialImpl> s(caller_fed_tag{});
std::string string_data;
string_data.resize(length);
@@ -1203,7 +1317,7 @@ IfcFile::IfcFile(void* data, int length)
, max_id_(0),
header_(new IfcParse::IfcSpfHeader(this))
{
FileReader s(std::string((char*)data, length), FileReader::caller_fed_tag{});
FileReader<PushedSequentialImpl> s(std::string((char*)data, length), caller_fed_tag{});
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(&s, schema_, max_id_, types_to_bypass_loading_);
@@ -1215,20 +1329,6 @@ IfcFile::IfcFile(void* data, int length)
byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
}
IfcFile::IfcFile(IfcParse::FileReader* s)
: schema_(nullptr)
, max_id_(0)
{
storage_.emplace<1>(this);
std::get<impl::in_memory_file_storage>(storage_).read_from_stream(s, schema_, max_id_, types_to_bypass_loading_);
good_ = std::get<impl::in_memory_file_storage>(storage_).good_;
ifcroot_type_ = schema_ ? schema_->declaration_by_name("IfcRoot") : nullptr;
byid_ = decltype(byid_)(&std::get<impl::in_memory_file_storage>(storage_).byid_read_);
byref_excl_ = decltype(byref_excl_)(&std::get<impl::in_memory_file_storage>(storage_).byref_excl_);
byguid_ = decltype(byguid_)(&std::get<impl::in_memory_file_storage>(storage_).byguid_);
}
IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const std::string& path)
: schema_(schema)
, ifcroot_type_(schema_->declaration_by_name("IfcRoot"))
@@ -1260,9 +1360,127 @@ IfcFile::IfcFile(const IfcParse::schema_definition* schema, filetype ty, const s
setDefaultHeaderValues();
}
bool IfcParse::InstanceStreamer::hasSemicolon() const {
namespace {
template <typename Reader>
void read_terminal(IfcSpfLexer<Reader>& lexer, const std::string& term, bool trailing_semicolon) {
if (lexer.Next().as_string() != term) {
throw IfcException(std::string("Expected " + term));
}
if (trailing_semicolon) {
if (!lexer.Next().is_operator(';')) {
throw IfcException("Expected ;");
}
}
}
template <typename Reader>
std::shared_ptr<InstanceData> read_header_entity(
IfcParse::IfcFile* file,
IfcParse::impl::in_memory_file_storage& storage,
IfcSpfLexer<Reader>& lexer,
IfcParse::unresolved_references& references_to_resolve,
const IfcParse::entity& decl) {
parse_context pc(&storage.context_pool_);
lexer.Next();
storage.load(&lexer, std::nullopt, nullptr, pc, -1);
auto result = pc.construct(file, std::nullopt, references_to_resolve, &decl, decl.attribute_count(), -1);
storage.context_pool_.reset();
return result;
}
template <typename Reader>
void parse_header(
IfcParse::IfcSpfHeader& header,
IfcParse::impl::in_memory_file_storage& storage,
IfcSpfLexer<Reader>& lexer,
IfcParse::unresolved_references& references_to_resolve) {
static const char* const ISO_10303_21 = "ISO-10303-21";
static const char* const HEADER = "HEADER";
read_terminal(lexer, ISO_10303_21, true);
read_terminal(lexer, HEADER, true);
read_terminal(lexer, Header_section_schema::file_description::Class().name_uc(), false);
header.set_file_description(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_description::Class()));
if (!lexer.Next().is_operator(';')) {
throw IfcException("Expected ;");
}
read_terminal(lexer, Header_section_schema::file_name::Class().name_uc(), false);
header.set_file_name(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_name::Class()));
if (!lexer.Next().is_operator(';')) {
throw IfcException("Expected ;");
}
read_terminal(lexer, Header_section_schema::file_schema::Class().name_uc(), false);
header.set_file_schema(read_header_entity(header.file(), storage, lexer, references_to_resolve, Header_section_schema::file_schema::Class()));
if (!lexer.Next().is_operator(';')) {
throw IfcException("Expected ;");
}
}
template <typename Reader>
bool try_parse_header(
IfcParse::IfcSpfHeader& header,
IfcParse::impl::in_memory_file_storage& storage,
IfcSpfLexer<Reader>& lexer,
IfcParse::unresolved_references& references_to_resolve) {
try {
parse_header(header, storage, lexer, references_to_resolve);
return true;
} catch (const std::exception& e) {
storage.context_pool_.reset();
Logger::Error(e);
return false;
}
}
} // namespace
template <typename Reader>
IfcSpfHeader& IfcParse::InstanceStreamer<Reader>::ensure_header() {
if (header_) {
return *header_;
}
if (owner_ != nullptr) {
header_ = &owner_->header();
header_->file(owner_);
} else {
owned_header_ = std::make_unique<IfcSpfHeader>(owner_);
header_ = owned_header_.get();
}
return *header_;
}
template <typename Reader>
void IfcParse::InstanceStreamer<Reader>::initialize_header() {
storage_.file = owner_;
storage_.schema = schema_;
storage_.references_to_resolve = &references_to_resolve_;
if (!lexer_ || !stream_ || !stream_->size() || stream_->eof()) {
return;
}
auto& header = ensure_header();
if (try_parse_header(header, storage_, *lexer_, references_to_resolve_) && header.file_schema().schema_identifiers().size() == 1) {
try {
schema_ = IfcParse::schema_by_name(header.file_schema().schema_identifiers().front());
good_ = file_open_status::SUCCESS;
} catch (const IfcParse::IfcException&) {
}
}
storage_.schema = schema_;
}
template <typename Reader>
bool IfcParse::InstanceStreamer<Reader>::hasSemicolon() const {
auto local_stream = stream_->clone();
auto local_lexer = IfcSpfLexer(&local_stream);
auto local_lexer = IfcSpfLexer<Reader>(&local_stream);
Token t;
try {
t = local_lexer.Next();
@@ -1272,20 +1490,20 @@ bool IfcParse::InstanceStreamer::hasSemicolon() const {
while (t.type != Token::Token_NONE) {
if (t.is_operator(';')) {
return true;
}
}
try {
t = local_lexer.Next();
} catch (const std::out_of_range&) {
// This most likely happens when a page boundary is contained within a string
break;
}
}
return false;
return false;
}
size_t IfcParse::InstanceStreamer::semicolonCount() const {
template <typename Reader>
size_t IfcParse::InstanceStreamer<Reader>::semicolonCount() const {
auto local_stream = stream_->clone();
auto local_lexer = IfcSpfLexer(&local_stream);
auto local_lexer = IfcSpfLexer<Reader>(&local_stream);
Token t;
size_t count = 0;
try {
@@ -1300,145 +1518,248 @@ size_t IfcParse::InstanceStreamer::semicolonCount() const {
try {
t = local_lexer.Next();
} catch (const std::out_of_range&) {
// This most likely happens when a page boundary is contained within a string
break;
}
}
return count;
}
void IfcParse::InstanceStreamer::pushPage(const std::string& page)
{
template <typename Reader>
void IfcParse::InstanceStreamer<Reader>::pushPage(const std::string& page) {
stream_->pushNextPage(page);
if (good_ == file_open_status::NO_HEADER) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) {
try {
schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front());
good_ = file_open_status::SUCCESS;
} catch (const IfcParse::IfcException&) {
}
}
storage_.file = nullptr;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
initialize_header();
}
}
IfcParse::InstanceStreamer::InstanceStreamer(IfcParse::IfcFile* f)
: stream_(new FileReader(FileReader::caller_fed_tag{}))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0)
, owner_(f)
{
init_locale();
good_ = file_open_status::NO_HEADER;
storage_.file = f;
}
IfcParse::InstanceStreamer::InstanceStreamer(const std::string& fn, bool mmap, IfcParse::IfcFile* f)
: stream_(mmap ? new FileReader(fn, FileReader::mmap_tag{}) : new FileReader(fn))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0), owner_(f)
{
init_locale();
good_ = file_open_status::NO_HEADER;
if (stream_->size() && !stream_->eof()) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) {
try {
schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front());
good_ = file_open_status::SUCCESS;
} catch (const IfcParse::IfcException&) {
}
}
storage_.file = f;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
}
}
IfcParse::InstanceStreamer::InstanceStreamer(void* data, int length, IfcParse::IfcFile* f)
: stream_(new FileReader(std::string((char*) data, length), FileReader::caller_fed_tag{}))
, lexer_(new IfcSpfLexer(stream_))
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0)
, owner_(f)
{
init_locale();
good_ = file_open_status::NO_HEADER;
if (stream_->size() && !stream_->eof()) {
header_ = new IfcParse::IfcSpfHeader(lexer_);
if (header_->tryRead() && header_->file_schema().schema_identifiers().size() == 1) {
try {
schema_ = IfcParse::schema_by_name(header_->file_schema().schema_identifiers().front());
good_ = file_open_status::SUCCESS;
} catch (const IfcParse::IfcException&) {
}
}
storage_.file = f;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
}
}
IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer, IfcParse::IfcFile* f)
template <typename Reader>
IfcParse::InstanceStreamer<Reader>::InstanceStreamer(IfcParse::IfcFile* f)
: stream_(nullptr)
, lexer_(lexer)
, header_(nullptr)
, owner_(f)
, token_stream_(3, Token{})
, schema_(schema)
, schema_(nullptr)
, progress_(0)
, owner_(f)
{
init_locale();
if constexpr (std::is_same_v<Reader, FileReader<PushedSequentialImpl>>) {
owned_stream_ = std::make_unique<Reader>(caller_fed_tag{});
} else {
static_assert(file_reader_dependent_false_v<Reader>, "Default InstanceStreamer requires a pushed sequential reader");
}
stream_ = owned_stream_.get();
lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
good_ = file_open_status::NO_HEADER;
storage_.file = f;
storage_.schema = schema_;
storage_.tokens = lexer_;
storage_.references_to_resolve = &references_to_resolve_;
}
void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass) {
// Initialize a "C" locale for locale-independent
// number parsing. See comment above on line 41.
template <typename Reader>
IfcParse::InstanceStreamer<Reader>::InstanceStreamer(const std::string& fn, bool mmap, IfcParse::IfcFile* f)
: stream_(nullptr)
, header_(nullptr)
, owner_(f)
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0) {
init_locale();
tokens = nullptr;
if constexpr (std::is_same_v<Reader, FileReader<FullBufferImpl>>) {
(void)mmap;
owned_stream_ = std::make_unique<Reader>(fn);
#ifdef USE_MMAP
} else if constexpr (std::is_same_v<Reader, MMapFileReader>) {
(void)mmap;
owned_stream_ = std::make_unique<Reader>(fn);
#endif
} else {
static_assert(file_reader_dependent_false_v<Reader>, "Path-based InstanceStreamer requires a file-backed reader");
}
stream_ = owned_stream_.get();
lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
good_ = file_open_status::NO_HEADER;
initialize_header();
}
template <typename Reader>
IfcParse::InstanceStreamer<Reader>::InstanceStreamer(void* data, int length, IfcParse::IfcFile* f)
: stream_(nullptr)
, header_(nullptr)
, owner_(f)
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0)
{
init_locale();
if constexpr (std::is_same_v<Reader, FileReader<PushedSequentialImpl>>) {
owned_stream_ = std::make_unique<Reader>(std::string((char*)data, length), caller_fed_tag{});
} else {
static_assert(file_reader_dependent_false_v<Reader>, "Buffer-based InstanceStreamer requires a pushed sequential reader");
}
stream_ = owned_stream_.get();
lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
good_ = file_open_status::NO_HEADER;
initialize_header();
}
template <typename Reader>
IfcParse::InstanceStreamer<Reader>::InstanceStreamer(Reader* stream, IfcParse::IfcFile* f)
: stream_(stream)
, header_(nullptr)
, owner_(f)
, token_stream_(3, Token{})
, schema_(nullptr)
, progress_(0) {
init_locale();
lexer_ = std::make_unique<IfcSpfLexer<Reader>>(stream_);
good_ = file_open_status::NO_HEADER;
initialize_header();
}
template <typename Reader>
void IfcParse::InstanceStreamer<Reader>::bypassTypes(const std::set<std::string>& type_names) {
for (auto& name : type_names) {
try {
types_to_bypass_.push_back(schema_->declaration_by_name(name));
} catch (const IfcException&) {
continue;
}
}
}
template <typename Reader>
std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> IfcParse::InstanceStreamer<Reader>::readInstance() {
std::optional<std::tuple<size_t, const IfcParse::declaration*, std::shared_ptr<InstanceData>>> return_value;
if (yield_header_instances_ && header_ && yielded_header_instances_ < 3) {
if (yielded_header_instances_ == 0) {
return_value.emplace(
0,
&header_->file_description().declaration(),
header_->file_description().data_weak().lock());
} else if (yielded_header_instances_ == 1) {
return_value.emplace(
0,
&header_->file_name().declaration(),
header_->file_name().data_weak().lock());
} else if (yielded_header_instances_ == 2) {
return_value.emplace(
0,
&header_->file_schema().declaration(),
header_->file_schema().data_weak().lock());
}
yielded_header_instances_ += 1;
return return_value;
}
unsigned current_id = 0;
while (good_ && !lexer_->stream->eof() && !current_id) {
if (token_stream_[0].type == IfcParse::Token::Token_IDENTIFIER &&
token_stream_[1].type == IfcParse::Token::Token_OPERATOR &&
token_stream_[1].value_char == '=' &&
token_stream_[2].type == IfcParse::Token::Token_KEYWORD) {
current_id = token_stream_[0].as_identifier();
const IfcParse::declaration* entity_type;
try {
entity_type = schema_->declaration_by_name(token_stream_[2].as_string());
} catch (const IfcException& ex) {
Logger::Message(Logger::LOG_ERROR, std::string(ex.what()) + " at offset " + std::to_string(token_stream_[2].start_pos));
current_id = 0;
goto advance;
}
if (entity_type->as_entity() == nullptr) {
Logger::Message(Logger::LOG_ERROR, "Non entity type " + entity_type->name() + " at offset " + std::to_string(token_stream_[2].start_pos));
goto advance;
}
for (auto& ty : types_to_bypass_) {
if (entity_type->is(*ty)) {
bypassed_instances_.push_back(current_id);
current_id = 0;
goto advance;
}
}
parse_context ps(&storage_.context_pool_);
lexer_->Next();
try {
storage_.load(lexer_.get(), current_id, entity_type->as_entity(), ps, -1);
} catch (const IfcInvalidTokenException& e) {
good_ = file_open_status::INVALID_SYNTAX;
Logger::Error(e);
break;
}
if (((++progress_) % 1000) == 0) {
std::stringstream ss;
ss << "\r#" << current_id;
Logger::Status(ss.str(), false);
}
auto data = ps.construct(owner_, current_id, references_to_resolve_, entity_type, std::nullopt, -1, coerce_attribute_count);
storage_.context_pool_.reset();
return_value.emplace(
(size_t)current_id,
entity_type,
data);
}
advance:
Token next_token;
try {
next_token = lexer_->Next();
} catch (const IfcException& e) {
Logger::Message(Logger::LOG_ERROR, std::string(e.what()) + ". Parsing terminated");
} catch (...) {
Logger::Message(Logger::LOG_ERROR, "Parsing terminated");
}
if (!lexer_->stream->eof() && !next_token) {
good_ = file_open_status::INVALID_SYNTAX;
break;
}
token_stream_.push_back(next_token);
}
stream_->dropPages();
lexer_->resetPool();
return return_value;
}
template <typename Reader>
void IfcParse::impl::in_memory_file_storage::read_from_stream(Reader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass) {
init_locale();
schema = nullptr;
if (!s->size() || s->eof()) {
// @todo set good on parent file
good_ = file_open_status::READ_ERROR;
return;
}
tokens = new IfcSpfLexer(s);
std::vector<std::string> schemas;
file->header().file(file);
InstanceStreamer<Reader> streamer(s, file);
streamer.yieldHeaderInstances(false);
if (file->header().tryRead()) {
if (const auto* header = streamer.header()) {
try {
schemas = file->header().file_schema().schema_identifiers();
schemas = header->file_schema().schema_identifiers();
} catch (...) {
// Purposely empty catch block
}
} else {
good_ = file_open_status::NO_HEADER;
}
if (schemas.size() == 1) {
schema = streamer.schema();
if (schema == nullptr && schemas.size() == 1) {
try {
schema = IfcParse::schema_by_name(schemas.front());
} catch (const IfcParse::IfcException& e) {
@@ -1448,28 +1769,28 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
}
if (schema == nullptr) {
if (schemas.empty()) {
good_ = streamer.status();
} else {
good_ = file_open_status::UNSUPPORTED_SCHEMA;
}
Logger::Message(Logger::LOG_ERROR, "No support for file schema encountered (" + boost::algorithm::join(schemas, ", ") + ")");
return;
}
auto ifcroot_type_ = schema->declaration_by_name("IfcRoot");
InstanceStreamer streamer(schema, tokens, file);
streamer.bypassTypes(typed_to_bypass);
Logger::Status("Scanning file...");
while (streamer) {
auto inst = streamer.readInstance();
if (!inst) {
// No more instances to read
break;
}
}
auto current_id = std::get<0>(*inst);
express::Base instance(std::get<2>(*inst));
if (instance.declaration().is(*ifcroot_type_)) {
@@ -1487,10 +1808,7 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
}
const IfcParse::declaration* ty = &instance.declaration();
{
bytype_excl_[ty].push_back(instance);
}
bytype_excl_[ty].push_back(instance);
if (byid_.find(current_id) != byid_.end()) {
std::stringstream ss;
@@ -1499,27 +1817,15 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
}
byid_.insert({(uint32_t)current_id, std::get<2>(*inst)});
max_id = (std::max)(max_id, (unsigned int) current_id);
max_id = (std::max)(max_id, (unsigned int)current_id);
}
good_ = streamer.status();
byref_excl_ = streamer.inverses();
// Move the storage of simple type instances so that they are retained during the lifetime of the file
good_ = streamer.status();
byref_excl_ = streamer.inverses();
read_simple_type_instances = streamer.stealInstances();
// Set file ownership on simple type instances, so that when adding them to other files, proper copies are created
/*
// @todo double check whether file ownership is property set earlier on
for (auto& inst : read_simple_type_instances) {
inst->file_ = file;
}
*/
Logger::Status("\rDone scanning file ");
delete tokens;
if (good_ != file_open_status::SUCCESS) {
return;
}
@@ -1639,6 +1945,12 @@ void IfcParse::impl::in_memory_file_storage::read_from_stream(IfcParse::FileRead
Logger::Status("Done resolving references");
}
template void IfcParse::impl::in_memory_file_storage::read_from_stream(FileReader<FullBufferImpl>* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
template void IfcParse::impl::in_memory_file_storage::read_from_stream(FileReader<PushedSequentialImpl>* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
#ifdef USE_MMAP
template void IfcParse::impl::in_memory_file_storage::read_from_stream(MMapFileReader* s, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
#endif
void IfcFile::recalculate_id_counter() {
/*
// @todo
src/ifcparse/IfcParse.h
+4 -3
View File
@@ -48,9 +48,10 @@ extern const char *IFCOPENSHELL_VERSION;
namespace IfcParse {
/// A stream of tokens to be read from a FileReader.
template <typename Reader>
class IFC_PARSE_API IfcSpfLexer {
private:
IfcCharacterDecoder* decoder_;
IfcCharacterDecoder<Reader>* decoder_;
size_t skipWhitespace() const;
size_t skipComment() const;
@@ -68,9 +69,9 @@ class IFC_PARSE_API IfcSpfLexer {
}
}
FileReader* stream;
Reader* stream;
// IfcFile* file;
IfcSpfLexer(FileReader* stream);
IfcSpfLexer(Reader* stream);
Token Next();
~IfcSpfLexer();
// void TokenString(size_t offset, std::string& result);
src/ifcparse/IfcSchema.h
+12 -8
View File
@@ -28,6 +28,7 @@
#include <iterator>
#include <string>
#include <vector>
#include <optional>
// Forward declarations
class InstanceData;
@@ -297,6 +298,7 @@ class IFC_PARSE_API entity : public declaration {
std::vector<const entity*> subtypes_;
std::vector<const attribute*> attributes_;
mutable std::optional<std::vector<const attribute*>> all_attributes_;
std::vector<bool> derived_;
std::vector<const inverse_attribute*> inverse_attributes_;
@@ -354,15 +356,17 @@ class IFC_PARSE_API entity : public declaration {
const std::vector<const attribute*>& attributes() const { return attributes_; }
const std::vector<bool>& derived() const { return derived_; }
const std::vector<const attribute*> all_attributes() const {
std::vector<const attribute*> attrs;
attrs.reserve(derived_.size());
if (supertype_ != nullptr) {
const std::vector<const attribute*> supertype_attrs = supertype_->all_attributes();
std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs));
const std::vector<const attribute*>& all_attributes() const {
if (!all_attributes_) {
auto& attrs = all_attributes_.emplace();
attrs.reserve(derived_.size());
if (supertype_ != nullptr) {
const std::vector<const attribute*> supertype_attrs = supertype_->all_attributes();
std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs));
}
std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs));
}
std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs));
return attrs;
return *all_attributes_;
}
const std::vector<const inverse_attribute*> all_inverse_attributes() const {
src/ifcparse/IfcSpfHeader.cpp
+27 -129
View File
@@ -1,26 +1,6 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include "IfcSpfHeader.h"
#include "IfcFile.h"
#include "IfcLogger.h"
static const char* const ISO_10303_21 = "ISO-10303-21";
static const char* const HEADER = "HEADER";
@@ -30,116 +10,31 @@ static const char* const DATA = "DATA";
using namespace IfcParse;
namespace {
std::shared_ptr<InstanceData> read_from_spf_file(IfcParse::IfcFile* file, IfcParse::impl::in_memory_file_storage* storage, const IfcParse::entity* decl) {
if (storage != nullptr) {
parse_context pc;
storage->tokens->Next();
storage->load(-1, nullptr, pc, -1);
return pc.construct(file, std::nullopt, *storage->references_to_resolve, decl, decl->as_entity()->attribute_count(), -1);
} else {
// std::unreachable();
return nullptr;
}
}
} // namespace
void IfcSpfHeader::readSemicolon() {
if (storage_ != nullptr) {
if (!storage_->tokens->Next().is_operator(';')) {
throw IfcException(std::string("Expected ;"));
}
} else {
// std::unreachable();
}
}
void IfcSpfHeader::readTerminal(const std::string& term, Trail trail) {
if (storage_ != nullptr) {
if (storage_->tokens->Next().as_string() != term) {
throw IfcException(std::string("Expected " + term));
}
if (trail == TRAILING_SEMICOLON) {
readSemicolon();
}
} else {
// std::unreachable();
}
}
IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file)
: file_(file)
{
Header_section_schema::get_schema();
// @todo This might still not work in IfcFile's uninitialized mode
storage_ = std::visit([this](auto& m) -> decltype(storage_) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
return &m;
}
return nullptr;
std::shared_ptr<InstanceData> make_header_entity(IfcParse::IfcFile* file, const IfcParse::entity& decl) {
const bool in_memory = file == nullptr || std::visit([](auto& storage) {
return std::is_same_v<std::decay_t<decltype(storage)>, IfcParse::impl::in_memory_file_storage>;
}, file->storage_);
const bool in_memory = storage_ != nullptr;
if (in_memory) {
header_entities_[0] = std::make_shared<InstanceData>(file, &Header_section_schema::file_description::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_description::Class().attribute_count()));
header_entities_[1] = std::make_shared<InstanceData>(file, &Header_section_schema::file_name::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_name::Class().attribute_count()));
header_entities_[2] = std::make_shared<InstanceData>(file, &Header_section_schema::file_schema::Class(), 0, in_memory_attribute_storage(Header_section_schema::file_schema::Class().attribute_count()));
} else {
header_entities_[0] = std::make_shared<InstanceData>(file, &Header_section_schema::file_description::Class(), 0, rocks_db_attribute_storage{});
header_entities_[1] = std::make_shared<InstanceData>(file, &Header_section_schema::file_name::Class(), 0, rocks_db_attribute_storage{});
header_entities_[2] = std::make_shared<InstanceData>(file, &Header_section_schema::file_schema::Class(), 0, rocks_db_attribute_storage{});
return std::make_shared<InstanceData>(file, &decl, 0, in_memory_attribute_storage(decl.attribute_count()));
}
return std::make_shared<InstanceData>(file, &decl, 0, rocks_db_attribute_storage{});
}
IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcSpfLexer* lexer)
{
} // namespace
IfcParse::IfcSpfHeader::IfcSpfHeader(IfcParse::IfcFile* file)
: file_(file) {
Header_section_schema::get_schema();
storage_ = new impl::in_memory_file_storage;
storage_->tokens = lexer;
file_ = nullptr;
header_entities_[0] = make_header_entity(file_, Header_section_schema::file_description::Class());
header_entities_[1] = make_header_entity(file_, Header_section_schema::file_name::Class());
header_entities_[2] = make_header_entity(file_, Header_section_schema::file_schema::Class());
}
IfcParse::IfcSpfHeader::~IfcSpfHeader() {
}
void IfcSpfHeader::read() {
readTerminal(ISO_10303_21, TRAILING_SEMICOLON);
readTerminal(HEADER, TRAILING_SEMICOLON);
// | The header section of every exchange structure shall contain one
// | instance of each of the following entities: file_description, file_name,
// | and file_schema, and they shall appear in that order. Instances of
// | file_population, section_language and section_context may appear after
// | file_schema. If instances of user-defined header section entities are
// | present, they shall appear after the header section entity instances
// | defined in this section.
//
// ISO 10303-21 Second edition 2002-01-15 p. 16
readTerminal(Header_section_schema::file_description::Class().name_uc(), NONE);
header_entities_[0] = read_from_spf_file(file_, storage_, &Header_section_schema::file_description::Class());
readSemicolon();
readTerminal(Header_section_schema::file_name::Class().name_uc(), NONE);
header_entities_[1] = read_from_spf_file(file_, storage_, &Header_section_schema::file_name::Class());
readSemicolon();
readTerminal(Header_section_schema::file_schema::Class().name_uc(), NONE);
header_entities_[2] = read_from_spf_file(file_, storage_, &Header_section_schema::file_schema::Class());
readSemicolon();
}
bool IfcSpfHeader::tryRead() {
try {
read();
return true;
} catch (const std::exception& e) {
Logger::Error(e);
return false;
}
}
IfcParse::IfcSpfHeader::~IfcSpfHeader() = default;
void IfcSpfHeader::write(std::ostream& out) const {
out << ISO_10303_21 << ";"
@@ -163,18 +58,21 @@ void IfcSpfHeader::write(std::ostream& out) const {
void IfcParse::IfcSpfHeader::file(IfcParse::IfcFile* file) {
file_ = file;
if (file != nullptr) {
storage_ = std::visit([this](auto& m) -> decltype(storage_) {
if constexpr (std::is_same_v<std::decay_t<decltype(m)>, impl::in_memory_file_storage>) {
return &m;
}
return nullptr;
},
file_->storage_);
}
}
const Header_section_schema::file_description IfcParse::IfcSpfHeader::file_description() const {
void IfcParse::IfcSpfHeader::set_file_description(const std::shared_ptr<InstanceData>& data) {
header_entities_[0] = data;
}
void IfcParse::IfcSpfHeader::set_file_name(const std::shared_ptr<InstanceData>& data) {
header_entities_[1] = data;
}
void IfcParse::IfcSpfHeader::set_file_schema(const std::shared_ptr<InstanceData>& data) {
header_entities_[2] = data;
}
const Header_section_schema::file_description IfcParse::IfcSpfHeader::file_description() const {
return Header_section_schema::file_description(header_entities_[0]);
}
src/ifcparse/IfcSpfHeader.h
+4 -22
View File
@@ -23,7 +23,6 @@
#include "ifc_parse_api.h"
#include "InstanceData.h"
#include "Header_section_schema.h"
#include "storage.h"
namespace IfcParse {
@@ -32,39 +31,22 @@ class IfcFile;
class IFC_PARSE_API IfcSpfHeader {
private:
IfcFile* file_;
IfcParse::impl::in_memory_file_storage* storage_ = nullptr;
std::array<std::shared_ptr<InstanceData>, 3> header_entities_;
/*
mutable Header_section_schema::file_description* file_description_;
mutable Header_section_schema::file_name* file_name_;
mutable Header_section_schema::file_schema* file_schema_;
*/
void readSemicolon();
enum Trail {
TRAILING_SEMICOLON,
NONE
};
void readTerminal(const std::string& term, Trail trail);
public:
explicit IfcSpfHeader(IfcParse::IfcFile* file);
explicit IfcSpfHeader(IfcParse::IfcSpfLexer* lexer);
~IfcSpfHeader();
// IfcParse::IfcFile* file() { return file_; }
// void file(IfcParse::IfcFile* file);
void read();
bool tryRead();
void write(std::ostream& out) const;
IfcParse::IfcFile* file() { return file_; }
void file(IfcParse::IfcFile* file);
void set_file_description(const std::shared_ptr<InstanceData>& data);
void set_file_name(const std::shared_ptr<InstanceData>& data);
void set_file_schema(const std::shared_ptr<InstanceData>& data);
const Header_section_schema::file_description file_description() const;
const Header_section_schema::file_name file_name() const;
const Header_section_schema::file_schema file_schema() const;
src/ifcparse/storage.h
+66 -11
View File
@@ -131,8 +131,8 @@ namespace IfcParse {
typedef std::list<std::pair<MutableAttributeValue, std::variant<reference_or_simple_type, std::vector<reference_or_simple_type>, std::vector<std::vector<reference_or_simple_type>>>>> unresolved_references;
class IfcFile;
template <typename Reader>
class IfcSpfLexer;
class FileReader;
struct Token {
enum TokenType {
@@ -202,15 +202,36 @@ namespace IfcParse {
}
};
struct parse_context;
struct parse_context_pool;
struct parse_context_handle {
parse_context_pool* pool = nullptr;
uint32_t index = 0;
parse_context& get() const;
parse_context* operator->() const;
parse_context& operator*() const;
explicit operator bool() const { return pool != nullptr; }
};
struct parse_context {
std::list<
std::vector<
std::variant<
express::Base,
Token,
parse_context*
parse_context_handle
>> tokens_;
parse_context() {};
void reset() {
tokens_.clear();
}
parse_context_pool* pool_;
parse_context(parse_context_pool* pool) : pool_(pool) {
tokens_.reserve(16);
};
~parse_context();
parse_context(const parse_context&) = delete;
@@ -228,16 +249,47 @@ namespace IfcParse {
std::shared_ptr<InstanceData> construct(IfcParse::IfcFile* owner, std::optional<size_t> name, unresolved_references& references_to_resolve, const IfcParse::declaration* decl, std::optional<size_t> expected_size, int resolve_reference_index, bool coerce_attribute_count=true);
};
struct parse_context_pool {
std::vector<parse_context> nodes_;
uint32_t used_ = 0;
void reset() { used_ = 0; }
parse_context_handle make() {
if (used_ == nodes_.size()) {
nodes_.emplace_back(this);
}
auto idx = used_++;
nodes_[idx].reset();
return {this, idx};
}
parse_context& get(uint32_t index) {
return nodes_[index];
}
};
inline parse_context& parse_context_handle::get() const {
return pool->get(index);
}
inline parse_context* parse_context_handle::operator->() const {
return &pool->get(index);
}
inline parse_context& parse_context_handle::operator*() const {
return pool->get(index);
}
namespace impl {
struct IFC_PARSE_API in_memory_file_storage {
IfcParse::parse_context_pool context_pool_;
std::vector<std::shared_ptr<InstanceData>> read_simple_type_instances;
std::vector<std::shared_ptr<InstanceData>> steal_instances() {
return read_simple_type_instances;
}
IfcParse::IfcSpfLexer* tokens;
// IfcParse::FileReader* stream;
// Either one of these needs to be set
IfcParse::IfcFile* file;
const IfcParse::schema_definition* schema;
@@ -258,7 +310,7 @@ namespace IfcParse {
typedef std::map<inverse_attr_record, std::vector<uint32_t>> entities_by_ref_t;
typedef entity_instance_by_name_t::iterator iterator;
in_memory_file_storage(IfcParse::IfcFile* f = nullptr) : tokens(nullptr), file(f), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr<InstanceData>& d) { return express::Base(d); }) {};
in_memory_file_storage(IfcParse::IfcFile* f = nullptr) : file(f), schema(nullptr), byid_read_(&byid_, [this](const std::shared_ptr<InstanceData>& d) { return express::Base(d); }) {};
in_memory_file_storage(const in_memory_file_storage&) = delete;
in_memory_file_storage(const in_memory_file_storage&&) = delete;
@@ -303,13 +355,16 @@ namespace IfcParse {
entity_instance_by_guid_t byguid_;
entity_instance_by_name_t byid_read_;
void load(std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
void try_read_semicolon() const;
template <typename Reader>
void load(IfcParse::IfcSpfLexer<Reader>* tokens, std::optional<size_t> entity_instance_name, const IfcParse::entity* entity, parse_context&, int attribute_index = -1);
template <typename Reader>
void try_read_semicolon(IfcParse::IfcSpfLexer<Reader>* tokens) const;
void register_inverse(unsigned, const IfcParse::entity* from_entity, int inst_id, int attribute_index);
void unregister_inverse(unsigned, const IfcParse::entity* from_entity, const express::Base&, int attribute_index);
void read_from_stream(IfcParse::FileReader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
template <typename Reader>
void read_from_stream(Reader* stream, const IfcParse::schema_definition*& schema, unsigned int& max_id, const std::set<std::string>& typed_to_bypass);
file_open_status good_ = file_open_status::SUCCESS;
src/ifcwrap/IfcParseWrapper.i
+7 -11
View File
@@ -31,10 +31,8 @@
%ignore IfcParse::IfcFile::byref_excl_;
%ignore IfcParse::IfcFile::types_to_bypass_loading_;
%ignore IfcParse::InstanceStreamer::InstanceStreamer(const IfcParse::schema_definition* schema, IfcParse::IfcSpfLexer* lexer);
%ignore IfcParse::InstanceStreamer::readInstance;
%ignore IfcParse::InstanceStreamer::stealInstances;
%ignore IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>::readInstance;
%ignore IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>::stealInstances;
%ignore express::Entity;
%ignore express::Select;
@@ -54,9 +52,6 @@
%ignore IfcParse::FileSchema::FileSchema;
%ignore IfcParse::IfcFile::tokens;
%ignore IfcParse::IfcSpfHeader::IfcSpfHeader(IfcSpfLexer*);
%ignore IfcParse::IfcSpfHeader::lexer;
%ignore IfcParse::IfcSpfHeader::stream;
%ignore IfcParse::IfcSpfHeader::file_description;
%ignore IfcParse::IfcSpfHeader::file_name;
%ignore IfcParse::IfcSpfHeader::file_schema;
@@ -922,6 +917,7 @@ object = custom_base
%}
%include "../ifcparse/IfcFile.h"
%template(InstanceStreamer) IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>;
%pythoncode %{
### hack hack hack
@@ -977,10 +973,10 @@ object = _old_object
return f;
}
IfcParse::InstanceStreamer* stream_from_string(const std::string& data) {
IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>* stream_from_string(const std::string& data) {
char* copiedData = new char[data.length()];
memcpy(copiedData, data.c_str(), data.length());
return new IfcParse::InstanceStreamer((void *)copiedData, data.length());
return new IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>>((void *)copiedData, data.length());
}
const char* version() {
@@ -1247,7 +1243,7 @@ object = _old_object
}
%}
%extend IfcParse::InstanceStreamer {
%extend IfcParse::InstanceStreamer<IfcParse::FileReader<IfcParse::FullBufferImpl>> {
PyObject* readInstancePy(bool type_as_declaration_instance=false) {
auto simply_type_to_dictionary = [&](const express::Base& t) -> PyObject* {
const auto& nm = t.declaration().name();
@@ -1452,4 +1448,4 @@ object = _old_object
return d;
}
}