diff --git a/src/ifcparse/file_reader.cpp b/src/ifcparse/file_reader.cpp index fa739cb280..2040daa2e4 100644 --- a/src/ifcparse/file_reader.cpp +++ b/src/ifcparse/file_reader.cpp @@ -109,8 +109,6 @@ paged_file_impl::~paged_file_impl() { fp_ = nullptr; } -size_t paged_file_impl::size() const { return file_size_; } - char paged_file_impl::get(size_t pos) const { if (pos >= file_size_) { throw std::out_of_range("get out of range"); diff --git a/src/ifcparse/file_reader.h b/src/ifcparse/file_reader.h index 50322125be..e1beccb302 100644 --- a/src/ifcparse/file_reader.h +++ b/src/ifcparse/file_reader.h @@ -47,6 +47,14 @@ #include #endif +// The cursor accessors sit on the tokenizer's innermost loop, one call per +// byte; left to the compiler's heuristics some of them end up as calls. +#if defined(_MSC_VER) +#define IFC_READER_INLINE __forceinline +#else +#define IFC_READER_INLINE inline __attribute__((always_inline)) +#endif + namespace ifcopenshell { struct file_reader_page { @@ -134,9 +142,9 @@ public: size_t tell() const { return cursor_; } size_t size() const { return impl_->size(); } - size_t remaining() const { return size() - cursor_; } + IFC_READER_INLINE size_t remaining() const { return size() - cursor_; } - char peek() const { + IFC_READER_INLINE char peek() const { if (cursor_ >= size()) { throw std::out_of_range("peek at EOF"); } @@ -146,7 +154,7 @@ public: return impl_->get(cursor_); } - uint64_t peek_u64() const { + IFC_READER_INLINE uint64_t peek_u64() const { if (remaining() < sizeof(uint64_t)) { throw std::out_of_range("peek_u64 at EOF"); } @@ -158,7 +166,7 @@ public: return impl_->get_u64(cursor_); } - uint32_t peek_u32() const { + IFC_READER_INLINE uint32_t peek_u32() const { if (remaining() < sizeof(uint32_t)) { throw std::out_of_range("peek_u32 at EOF"); } @@ -170,7 +178,7 @@ public: return impl_->get_u32(cursor_); } - void increment(size_t count = 1) { + IFC_READER_INLINE void increment(size_t count = 1) { if (cursor_ + count > size()) { throw std::out_of_range("increment past EOF"); } @@ -189,17 +197,17 @@ public: impl_->drop_pages(up_to_position); } - bool eof() const { + IFC_READER_INLINE bool eof() const { return cursor_ >= size(); } - char read() { + IFC_READER_INLINE char read() { auto c = peek(); increment(1); return c; } - char get(size_t position) const { + IFC_READER_INLINE char get(size_t position) const { if (const char* p = cached_(position, 1)) { return *p; } @@ -221,11 +229,28 @@ private: // A pointer to `count` bytes at `position` if they lie in one page, // else nullptr. Always nullptr for a contiguous implementation, whose // get() is already direct. - const char* cached_(size_t position, size_t count) const { + IFC_READER_INLINE const char* cached_(size_t position, size_t count) const { if constexpr (std::is_same_v) { if (cached_data_ != nullptr && position >= cached_begin_ && position + count <= cached_end_ && cached_evictions_ == impl_->evictions()) { return cached_data_ + (position - cached_begin_); } + return cached_refresh_(position, count); + } else { + (void)position; + (void)count; + return nullptr; + } + } + + // The slow half of cached_(): fetches the page and re-points the cache. + // Kept out of line so the check above inlines into every peek. +#if defined(_MSC_VER) + __declspec(noinline) +#else + __attribute__((noinline)) +#endif + const char* cached_refresh_(size_t position, size_t count) const { + if constexpr (std::is_same_v) { const size_t page_size = impl_->page_size(); const size_t index = position / page_size; const auto page = impl_->page(index); @@ -305,7 +330,7 @@ public: // Incremented whenever a page leaves the cache, so a pointer into a // page can be checked for validity cheaply. size_t evictions() const { return evictions_; } - size_t size() const; + size_t size() const { return file_size_; } char get(size_t position) const; uint32_t get_u32(size_t position) const; uint64_t get_u64(size_t position) const; diff --git a/src/ifcparse/parse.cpp b/src/ifcparse/parse.cpp index 4c29cdd2ea..623e8cbc39 100644 --- a/src/ifcparse/parse.cpp +++ b/src/ifcparse/parse.cpp @@ -179,6 +179,14 @@ bool parse_num_(const char* pStart, size_t size, T& val) { } // namespace +// These helpers sit on the tokenizer's innermost loop; left to the +// compiler's heuristics they end up as calls, one per eight bytes. +#if defined(_MSC_VER) +#define IFC_SWAR_INLINE __forceinline +#else +#define IFC_SWAR_INLINE inline __attribute__((always_inline)) +#endif + namespace SWAR { constexpr uint32_t ONES32 = 0x01010101u; constexpr uint32_t HIGHS32 = 0x80808080u; @@ -189,19 +197,19 @@ constexpr uint64_t splat(unsigned char c) { return ONES * c; } -inline uint32_t has_zero_byte(uint32_t x) { +IFC_SWAR_INLINE uint32_t has_zero_byte(uint32_t x) { return (x - ONES32) & ~x & HIGHS32; } -inline uint64_t has_zero_byte(uint64_t x) { +IFC_SWAR_INLINE uint64_t has_zero_byte(uint64_t x) { return (x - ONES) & ~x & HIGHS; } -inline uint32_t eq_mask(uint32_t x, uint32_t c) { +IFC_SWAR_INLINE uint32_t eq_mask(uint32_t x, uint32_t c) { return has_zero_byte(x ^ c); } -inline uint64_t eq_mask(uint64_t x, uint64_t c) { +IFC_SWAR_INLINE uint64_t eq_mask(uint64_t x, uint64_t c) { return has_zero_byte(x ^ c); } @@ -223,7 +231,7 @@ constexpr uint64_t dot = splat('.'); } // namespace chars template -inline uint64_t has_special_char(uint64_t x) { +IFC_SWAR_INLINE uint64_t has_special_char(uint64_t x) { return eq_mask(x, chars::lpar) | eq_mask(x, chars::rpar) | eq_mask(x, chars::eq) | @@ -239,7 +247,7 @@ inline uint64_t has_special_char(uint64_t x) { } template -inline uint32_t has_special_char(uint32_t x) { +IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) { return eq_mask(x, static_cast(chars::lpar)) | eq_mask(x, static_cast(chars::rpar)) | eq_mask(x, static_cast(chars::eq)) |