diff --git a/src/ifcparse/file_reader.h b/src/ifcparse/file_reader.h index 37d1c9c440..b66dd4a422 100644 --- a/src/ifcparse/file_reader.h +++ b/src/ifcparse/file_reader.h @@ -183,6 +183,22 @@ public: } } + // The current contiguous bytes, valid until the shared page cache evicts + // them. An empty span asks callers to use the regular reader operations. + IFC_READER_INLINE std::pair span() const { + if (eof()) { + return {nullptr, 0}; + } + if constexpr (std::is_same_v) { + const char* data = cached_(cursor_, 1); + return {data, cached_end_ - cursor_}; + } else if constexpr (std::is_same_v) { + return {nullptr, 0}; + } else { + return {impl_->data() + cursor_, remaining()}; + } + } + IFC_READER_INLINE char peek() const { if (cursor_ >= size()) { throw std::out_of_range("peek at EOF"); diff --git a/src/ifcparse/spf_scan.h b/src/ifcparse/spf_scan.h index 61956deff7..8583d451f6 100644 --- a/src/ifcparse/spf_scan.h +++ b/src/ifcparse/spf_scan.h @@ -158,6 +158,10 @@ IFC_SWAR_INLINE uint32_t has_special_char(uint32_t x) { } +inline bool is_token_delimiter(char c) { + return c == '(' || c == ')' || c == '=' || c == ',' || c == ';' || c == '/'; +} + // One pass over the tokens from the cursor, handing each to the consumer // without building a token; each callback returns whether to go on. The // consumer's constexpr flags decide what is decoded: decode_strings (else a @@ -223,6 +227,26 @@ void spf_lexer::scan(Consumer& consumer) { continue; } + // Parse names directly when the complete spelling is in this span. + // Whitespace, page splits and invalid names use the normal token loop. + if (character == '#') { + const auto span = stream->span(); + if (span.second) { + const char* begin = span.first; + const char* end = begin + span.second; + const char* digits = begin + (*begin == '+'); + int value; + const auto parsed = std::from_chars(digits, end, value); + if (parsed.ec == std::errc() && parsed.ptr != end && is_token_delimiter(*parsed.ptr)) { + stream->increment(static_cast(parsed.ptr - begin)); + if (!consumer.identifier(pos, static_cast(value))) { + return; + } + continue; + } + } + } + auto ttype = token::Token_NONE; if (character == '"') { ttype = token::Token_BINARY; @@ -272,12 +296,7 @@ void spf_lexer::scan(Consumer& consumer) { // Read character and increment pointer if not starting a new token char c = stream->peek(); - if (c == '(' || - c == ')' || - c == '=' || - c == ',' || - c == ';' || - c == '/') { + if (is_token_delimiter(c)) { break; } if (!(c == ' ' || c == '\r' || c == '\n' || c == '\t')) { diff --git a/src/ifcparse/tests/test_ifcopenshell_parse.cpp b/src/ifcparse/tests/test_ifcopenshell_parse.cpp index 66783b1f43..eaf3365b76 100644 --- a/src/ifcparse/tests/test_ifcopenshell_parse.cpp +++ b/src/ifcparse/tests/test_ifcopenshell_parse.cpp @@ -302,12 +302,26 @@ struct recording_consumer { static constexpr bool decode_values = false; static constexpr bool keep_keywords = false; std::vector> seen; - bool operator_(size_t pos, char c) { seen.push_back({pos, c}); return true; } - bool identifier(size_t pos, uint32_t) { seen.push_back({pos, '#'}); return true; } - bool string(size_t pos, size_t) { seen.push_back({pos, '\''}); return true; } - bool literal(size_t pos) { seen.push_back({pos, 'L'}); return true; } + std::vector identifiers; + bool operator_(size_t pos, char c) { + seen.push_back({pos, c}); + return true; + } + bool identifier(size_t pos, uint32_t value) { + seen.push_back({pos, '#'}); + identifiers.push_back(value); + return true; + } + bool string(size_t pos, size_t) { + seen.push_back({pos, '\''}); + return true; + } + bool literal(size_t pos) { + seen.push_back({pos, 'L'}); + return true; + } }; -} +} // namespace TEST_CASE("The index token policy ends every token where the full policy does, without decoding", "[ifcparse]") { // Doubled quotes, a \S\' escape (an apostrophe as the page character, @@ -382,6 +396,62 @@ TEST_CASE("The index token policy ends every token where the full policy does, w CHECK(lexer.next().as_string() == "a\xc2\xa7" "b"); } +TEST_CASE("Scanning preserves identifiers and token boundaries across reader pages", "[ifcparse][scan]") { + const std::string data = + "#1=IFCEXAMPLE(#0,#+12,#-1,#2147483647,#+-2,#1 2,#000003," + "'it''s','a\\S\\'b','\\X2\\00E9\\X0\\c',/* #999 */ .T.,\"0A\",-1.5E-3,$,*,(IFCLABEL(''),#42));#9"; + ifcopenshell::file_reader reference(data, ifcopenshell::caller_fed_tag{}); + ifcopenshell::spf_lexer lexer(&reference); + recording_consumer expected; + while (auto tk = lexer.next()) { + expected.seen.push_back({tk.start_pos, tk.is_operator() ? tk.value_char : tk.is_identifier() ? '#' : tk.is_string() ? '\'' : 'L'}); + if (tk.is_identifier()) { + expected.identifiers.push_back(tk.as_identifier()); + } + lexer.reset_pool(); + } + CHECK(expected.identifiers == std::vector{1, 0, 12, UINT32_MAX, INT32_MAX, UINT32_MAX - 1, 12, 3, 42, 9}); + const auto check = [&](auto& source) { + ifcopenshell::spf_lexer> scanner(&source); + recording_consumer actual; + scanner.scan(actual); + CHECK(actual.seen == expected.seen); + CHECK(actual.identifiers == expected.identifiers); + CHECK(source.tell() == data.size()); + }; + reference.seek(0); + check(reference); + ifcopenshell::file_reader pushed(ifcopenshell::caller_fed_tag{}); + for (char c : data) { + pushed.push_next_page(std::string(1, c)); + } + check(pushed); + const auto path = std::filesystem::temp_directory_path() / "ifcopenshell_scan_pages_test.ifc"; + { + std::ofstream out(path, std::ios::binary); + out << data; + } + for (size_t page_size : {1, 2, 3, 7, 8, 9, 16, 64}) { + CAPTURE(page_size); + ifcopenshell::file_reader paged(path.string(), page_size, 1); + check(paged); + } + std::filesystem::remove(path); +} + +TEST_CASE("Identifier shortcuts retain invalid-token errors", "[ifcparse][scan]") { + for (const std::string data : {"#;", "#+;", "#2147483648;", "#-2147483649;", "#12a;", "#1.0;", "##2;"}) { + CAPTURE(data); + ifcopenshell::file_reader reader(data, ifcopenshell::caller_fed_tag{}); + ifcopenshell::spf_lexer lexer(&reader); + CHECK_THROWS_AS(lexer.next(), ifcopenshell::invalid_token_exception); + reader.seek(0); + lexer.reset_pool(); + recording_consumer consumer; + CHECK_THROWS_AS(lexer.scan(consumer), ifcopenshell::invalid_token_exception); + } +} + namespace { const char* const reference_resolution_spf = "ISO-10303-21;\n"