mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-09-21 12:13:43 +00:00
Dion Moult
9829ebf001
spf_lexer::next() becomes next<Policy>(). full_tokens, the default, is what the parser has always had. index_tokens is what the lazy index needs: a string is ended but not decoded, and a number, enumeration or binary comes back as Token_LITERAL with only its position; names, keywords and operators are read as before. Each policy compiles to its own loop from the one implementation, so there is no second tokenizer. character_decoder gains skip(): the same state machine as the conversion with the collection compiled out, so an escape such as \S\' (an apostrophe as the page character) ends the string at the same byte under both policies. A byte-level scan would have ended it early. Also fixes a comment that follows a token without whitespace, ",/* x */", which skip_comment() never saw because the slash had been consumed. TXG (58 MB), single thread: tokenizing the whole file 194 MB/s with full_tokens, 249 MB/s with index_tokens; through 64 KB pages 196 and 205 MB/s. The parse itself is unchanged. This commit was written by an AI coding tool and has not been verified by a human. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL
116 lines
4.6 KiB
C++
116 lines
4.6 KiB
C++
/********************************************************************************
|
|
* *
|
|
* This file is part of IfcOpenShell. *
|
|
* *
|
|
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
|
* it under the terms of the Lesser GNU General Public License as published by *
|
|
* the Free Software Foundation, either version 3.0 of the License, or *
|
|
* (at your option) any later version. *
|
|
* *
|
|
* IfcOpenShell is distributed in the hope that it will be useful, *
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
|
* Lesser GNU General Public License for more details. *
|
|
* *
|
|
* You should have received a copy of the Lesser GNU General Public License *
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
|
* *
|
|
********************************************************************************/
|
|
|
|
/********************************************************************************
|
|
* *
|
|
* This file provides functions for loading an IFC file into memory and access *
|
|
* its entities either by ID, by an IfcSchema::Type or by reference *
|
|
* *
|
|
********************************************************************************/
|
|
|
|
#ifndef IFCPARSE_H
|
|
#define IFCPARSE_H
|
|
|
|
#include "argument.h"
|
|
#include "ifc_parse_api.h"
|
|
#include "express.h"
|
|
#include "character_decoder.h"
|
|
#include "file_reader.h"
|
|
#include "macros.h"
|
|
#include "storage.h"
|
|
|
|
#include <boost/dynamic_bitset.hpp>
|
|
#include <memory>
|
|
#include <cstring>
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
|
|
extern IFC_PARSE_API const char *IFCOPENSHELL_VERSION;
|
|
|
|
namespace ifcopenshell {
|
|
|
|
IFC_PARSE_API std::string encode_spf_string(const std::string& value);
|
|
|
|
IFC_PARSE_API std::string decode_spf_string(const std::string& value);
|
|
|
|
/// What a pass over the tokens has to produce. The parser needs every
|
|
/// value; the lazy index only needs to know where the tokens are and which
|
|
/// of them are instance names, so it ends strings without decoding them and
|
|
/// passes over numbers, enumerations and binaries. The choice is a template
|
|
/// parameter of spf_lexer::next(), so each pass compiles to its own loop.
|
|
struct full_tokens {
|
|
static constexpr bool decode_strings = true;
|
|
static constexpr bool decode_values = true;
|
|
};
|
|
struct index_tokens {
|
|
static constexpr bool decode_strings = false;
|
|
static constexpr bool decode_values = false;
|
|
};
|
|
|
|
/// A stream of tokens to be read from a file_reader.
|
|
template <typename Reader>
|
|
class IFC_PARSE_API spf_lexer {
|
|
private:
|
|
character_decoder<Reader>* decoder_;
|
|
ifcopenshell::logger& logger_;
|
|
|
|
size_t skip_whitespace() const;
|
|
size_t skip_comment() const;
|
|
|
|
mutable std::vector<std::unique_ptr<std::array<std::string, 16>>> stringpool_;
|
|
mutable size_t pool_index = 0;
|
|
|
|
public:
|
|
|
|
spf_lexer(const spf_lexer&) = delete;
|
|
spf_lexer& operator=(const spf_lexer&) = delete;
|
|
|
|
std::string& get_temp_string() const;
|
|
void reset_pool() const {
|
|
pool_index = 0;
|
|
}
|
|
void pop_pool_entry() {
|
|
if (pool_index > 0) {
|
|
--pool_index;
|
|
}
|
|
}
|
|
|
|
Reader* stream;
|
|
// file* file;
|
|
spf_lexer(Reader* stream, ifcopenshell::logger& logger = ifcopenshell::logger::root());
|
|
// The next token. With index_tokens a string, number, enumeration or
|
|
// binary comes back as Token_LITERAL (Token_STRING for a string) with
|
|
// only its position; names, keywords and operators are always read.
|
|
template <typename Policy = full_tokens>
|
|
token next();
|
|
~spf_lexer();
|
|
// void TokenString(size_t offset, std::string& result);
|
|
};
|
|
|
|
IFC_PARSE_API std::vector<express::base> traverse(const express::base& instance, int max_depth = -1);
|
|
|
|
IFC_PARSE_API std::vector<express::base> traverse_breadth_first(const express::base& instance, int max_depth = -1);
|
|
} // namespace ifcopenshell
|
|
|
|
IFC_PARSE_API std::ostream& operator<<(std::ostream& stream, const ifcopenshell::file& file);
|
|
|
|
#endif
|