Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-05 23:41:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcparse: combine UTF-16 surrogate pairs in \X2\ escapes

Browse Source 
ISO 10303-21 defines the values inside a \X2\ escape as UTF-16 code
units, so a character outside the BMP is necessarily written as a
surrogate pair. The decoder pushed each 4-hex value straight into the
u32string builder as if it were a codepoint, so a pair arrived as two
lone surrogates, which boost::locale::conv::utf_to_utf then discarded.
Non-BMP characters were silently dropped on read and lost again on
write.

Pair a high surrogate with the low surrogate that follows and emit the
combined codepoint. Lone surrogates keep their previous behaviour, and
the pending high surrogate is cleared when the \X0\ escape ends the
extended block.

Generated with the assistance of an AI coding tool.
This commit is contained in:
Petru Conduraru
2026-07-20 12:50:43 +03:00
parent 55a2430d71
commit afd7b81f0c
2 changed files with 43 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcopenshell-python/test/test_parse.py
+24
View File
@@ -1,3 +1,5 @@
import pytest
import ifcopenshell
@@ -32,3 +34,25 @@ def test_reference_to_undefined_owning_instance_simple_type():
data = "ISO-10303-21;HEADER;FILE_DESCRIPTION();FILE_NAME();FILE_SCHEMA(('IFC4'));#=IFCPROJECT((#))#4=IFCSIUNIT("
ifcopenshell.file.from_string(data)
print(ifcopenshell.get_log())
@pytest.mark.parametrize(
"encoded,expected",
[
("AA\\X2\\D83DDE00\\X0\\BB", "AA\U0001f600BB"),
("AA\\X4\\0001F600\\X0\\BB", "AA\U0001f600BB"),
("AA\\X2\\00E9\\X0\\BB", "AAéBB"),
("\\X2\\00E9D83DDE0000E9\\X0\\", "é\U0001f600é"),
("\\X2\\D83DDE00D83DDE01\\X0\\", "\U0001f600\U0001f601"),
("Bld \\X2\\5EFA7BC9\\X0\\ ok", "Bld 建築 ok"),
],
)
def test_x2_utf16_surrogate_pairs(encoded, expected):
data = (
"ISO-10303-21;HEADER;FILE_DESCRIPTION((''),'2;1');FILE_NAME('','',(''),(''),'','','');"
"FILE_SCHEMA(('IFC4'));ENDSEC;DATA;"
"#1=IFCPROJECT('0YvctVUKr0kugbFTf53O9L',$,'" + encoded + "',$,$,$,$,$,$);"
"ENDSEC;END-ISO-10303-21;"
)
f = ifcopenshell.file.from_string(data)
assert f.by_id(1).Name == expected
src/ifcparse/IfcCharacterDecoder.cpp
+19 -2
View File
@@ -98,6 +98,7 @@ namespace {
int codepage = 1;
unsigned int hex = 0;
unsigned int hex_count = 0;
unsigned int high_surrogate = 0;
while ((current_char = stream_.peek()) != 0) {
if (EXPECTS_CHARACTER(parse_state)) {
@@ -111,7 +112,7 @@ namespace {
if (((parse_state & ALPHABET_DEFINITION) != 0U) ||
((parse_state & IGNORED_DIRECTIVE) != 0U) ||
((parse_state & ENDEXTENDED_0) != 0U)) {
parse_state = hex = hex_count = 0;
parse_state = hex = hex_count = high_surrogate = 0;
} else if ((parse_state & ENCOUNTERED_HEX) != 0U) {
parse_state += THIRD_SOLIDUS;
parse_state -= ENCOUNTERED_HEX;
@@ -149,7 +150,23 @@ namespace {
if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) ||
(hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) ||
(hex_count == 8)) {
builder_.push_back(hex);
// \X2\ carries UTF-16 code units, so surrogate pairs form a single codepoint.
bool awaiting_low_surrogate = false;
auto codepoint = static_cast<std::u32string::value_type>(hex);
if (hex_count == 4) {
if (hex >= 0xd800 && hex <= 0xdbff) {
high_surrogate = hex;
awaiting_low_surrogate = true;
} else if ((high_surrogate != 0U) && hex >= 0xdc00 && hex <= 0xdfff) {
codepoint = 0x10000 + ((high_surrogate - 0xd800) << 10) + (hex - 0xdc00);
high_surrogate = 0;
} else {
high_surrogate = 0;
}
}
if (!awaiting_low_surrogate) {
builder_.push_back(codepoint);
}
if (hex_count == 2) {
parse_state = 0;
} else {