diff --git a/src/ifcopenshell-python/test/test_parse.py b/src/ifcopenshell-python/test/test_parse.py index d40101333e..ad2ae40f0b 100644 --- a/src/ifcopenshell-python/test/test_parse.py +++ b/src/ifcopenshell-python/test/test_parse.py @@ -1,3 +1,5 @@ +import pytest + import ifcopenshell @@ -32,3 +34,25 @@ def test_reference_to_undefined_owning_instance_simple_type(): data = "ISO-10303-21;HEADER;FILE_DESCRIPTION();FILE_NAME();FILE_SCHEMA(('IFC4'));#=IFCPROJECT((#))#4=IFCSIUNIT(" ifcopenshell.file.from_string(data) print(ifcopenshell.get_log()) + + +@pytest.mark.parametrize( + "encoded,expected", + [ + ("AA\\X2\\D83DDE00\\X0\\BB", "AA\U0001f600BB"), + ("AA\\X4\\0001F600\\X0\\BB", "AA\U0001f600BB"), + ("AA\\X2\\00E9\\X0\\BB", "AAéBB"), + ("\\X2\\00E9D83DDE0000E9\\X0\\", "é\U0001f600é"), + ("\\X2\\D83DDE00D83DDE01\\X0\\", "\U0001f600\U0001f601"), + ("Bld \\X2\\5EFA7BC9\\X0\\ ok", "Bld 建築 ok"), + ], +) +def test_x2_utf16_surrogate_pairs(encoded, expected): + data = ( + "ISO-10303-21;HEADER;FILE_DESCRIPTION((''),'2;1');FILE_NAME('','',(''),(''),'','','');" + "FILE_SCHEMA(('IFC4'));ENDSEC;DATA;" + "#1=IFCPROJECT('0YvctVUKr0kugbFTf53O9L',$,'" + encoded + "',$,$,$,$,$,$);" + "ENDSEC;END-ISO-10303-21;" + ) + f = ifcopenshell.file.from_string(data) + assert f.by_id(1).Name == expected diff --git a/src/ifcparse/IfcCharacterDecoder.cpp b/src/ifcparse/IfcCharacterDecoder.cpp index 019499a49e..6b8fd7562d 100644 --- a/src/ifcparse/IfcCharacterDecoder.cpp +++ b/src/ifcparse/IfcCharacterDecoder.cpp @@ -98,6 +98,7 @@ namespace { int codepage = 1; unsigned int hex = 0; unsigned int hex_count = 0; + unsigned int high_surrogate = 0; while ((current_char = stream_.peek()) != 0) { if (EXPECTS_CHARACTER(parse_state)) { @@ -111,7 +112,7 @@ namespace { if (((parse_state & ALPHABET_DEFINITION) != 0U) || ((parse_state & IGNORED_DIRECTIVE) != 0U) || ((parse_state & ENDEXTENDED_0) != 0U)) { - parse_state = hex = hex_count = 0; + parse_state = hex = hex_count = high_surrogate = 0; } else if ((parse_state & ENCOUNTERED_HEX) != 0U) { parse_state += THIRD_SOLIDUS; parse_state -= ENCOUNTERED_HEX; @@ -149,7 +150,23 @@ namespace { if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) || (hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) || (hex_count == 8)) { - builder_.push_back(hex); + // \X2\ carries UTF-16 code units, so surrogate pairs form a single codepoint. + bool awaiting_low_surrogate = false; + auto codepoint = static_cast(hex); + if (hex_count == 4) { + if (hex >= 0xd800 && hex <= 0xdbff) { + high_surrogate = hex; + awaiting_low_surrogate = true; + } else if ((high_surrogate != 0U) && hex >= 0xdc00 && hex <= 0xdfff) { + codepoint = 0x10000 + ((high_surrogate - 0xd800) << 10) + (hex - 0xdc00); + high_surrogate = 0; + } else { + high_surrogate = 0; + } + } + if (!awaiting_low_surrogate) { + builder_.push_back(codepoint); + } if (hex_count == 2) { parse_state = 0; } else {