mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-09-22 21:12:22 +00:00
Dion Moult
9829ebf001
spf_lexer::next() becomes next<Policy>(). full_tokens, the default, is what the parser has always had. index_tokens is what the lazy index needs: a string is ended but not decoded, and a number, enumeration or binary comes back as Token_LITERAL with only its position; names, keywords and operators are read as before. Each policy compiles to its own loop from the one implementation, so there is no second tokenizer. character_decoder gains skip(): the same state machine as the conversion with the collection compiled out, so an escape such as \S\' (an apostrophe as the page character) ends the string at the same byte under both policies. A byte-level scan would have ended it early. Also fixes a comment that follows a token without whitespace, ",/* x */", which skip_comment() never saw because the slash had been consumed. TXG (58 MB), single thread: tokenizing the whole file 194 MB/s with full_tokens, 249 MB/s with index_tokens; through 64 KB pages 196 and 205 MB/s. The parse itself is unchanged. This commit was written by an AI coding tool and has not been verified by a human. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013wcN7XquTfUi4vsKQ4KchL
86 lines
3.5 KiB
C++
86 lines
3.5 KiB
C++
/********************************************************************************
|
|
* *
|
|
* This file is part of IfcOpenShell. *
|
|
* *
|
|
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
|
* it under the terms of the Lesser GNU General Public License as published by *
|
|
* the Free Software Foundation, either version 3.0 of the License, or *
|
|
* (at your option) any later version. *
|
|
* *
|
|
* IfcOpenShell is distributed in the hope that it will be useful, *
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
|
* Lesser GNU General Public License for more details. *
|
|
* *
|
|
* You should have received a copy of the Lesser GNU General Public License *
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
|
* *
|
|
********************************************************************************/
|
|
|
|
/********************************************************************************
|
|
* *
|
|
* Implementation of character decoding as described in ISO 10303-21 table 2 and *
|
|
* table 4 *
|
|
* *
|
|
********************************************************************************/
|
|
|
|
#ifndef IFCCHARACTERDECODER_H
|
|
#define IFCCHARACTERDECODER_H
|
|
|
|
#include "file_reader.h"
|
|
#include "logger.h"
|
|
|
|
#include <string>
|
|
|
|
namespace ifcopenshell {
|
|
IFC_PARSE_API std::u32string::value_type convert_codepage(int codepage, int index);
|
|
IFC_PARSE_API std::string convert_utf8(const std::u32string& string);
|
|
IFC_PARSE_API std::u32string convert_utf8(const std::string& string);
|
|
} // namespace ifcopenshell
|
|
|
|
namespace ifcopenshell {
|
|
|
|
template <typename Reader>
|
|
class IFC_PARSE_API character_decoder {
|
|
private:
|
|
Reader* stream_;
|
|
logger& logger_;
|
|
int codepage_;
|
|
std::u32string builder_;
|
|
|
|
public:
|
|
enum ConversionMode {
|
|
SUBSTITUTE,
|
|
UTF8,
|
|
ESCAPE
|
|
};
|
|
inline static ConversionMode mode = UTF8;
|
|
inline static char substitution_character = '_';
|
|
|
|
character_decoder(Reader* stream, logger& logger = ifcopenshell::logger::root());
|
|
~character_decoder();
|
|
// Gets a decoded string representation at the token stream
|
|
// read pointer and advances the underlying token stream.
|
|
operator std::string();
|
|
// Advances the token stream past the string at the read pointer
|
|
// without decoding it: the same state machine as the conversion, so
|
|
// escapes such as \S\' end the string at the same byte.
|
|
void skip();
|
|
// Gets a decoded string representation at the offset provided,
|
|
// does not mutate the underlying token stream read pointer.
|
|
std::string get(size_t& offset);
|
|
};
|
|
|
|
class IFC_PARSE_API character_encoder {
|
|
private:
|
|
std::u32string str_;
|
|
|
|
public:
|
|
character_encoder(const std::string& input);
|
|
operator std::string();
|
|
};
|
|
|
|
} // namespace ifcopenshell
|
|
|
|
#endif
|