From afd7b81f0c5c482d5c1ef5e2d1c94df948371de8 Mon Sep 17 00:00:00 2001
From: Petru Conduraru <petru@bimvoice.com>
Date: Mon, 20 Jul 2026 12:50:43 +0300
Subject: [PATCH] ifcparse: combine UTF-16 surrogate pairs in \X2\ escapes

ISO 10303-21 defines the values inside a \X2\ escape as UTF-16 code
units, so a character outside the BMP is necessarily written as a
surrogate pair. The decoder pushed each 4-hex value straight into the
u32string builder as if it were a codepoint, so a pair arrived as two
lone surrogates, which boost::locale::conv::utf_to_utf then discarded.
Non-BMP characters were silently dropped on read and lost again on
write.

Pair a high surrogate with the low surrogate that follows and emit the
combined codepoint. Lone surrogates keep their previous behaviour, and
the pending high surrogate is cleared when the \X0\ escape ends the
extended block.

Generated with the assistance of an AI coding tool.
---
 src/ifcopenshell-python/test/test_parse.py | 24 ++++++++++++++++++++++
 src/ifcparse/IfcCharacterDecoder.cpp       | 21 +++++++++++++++++--
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/src/ifcopenshell-python/test/test_parse.py b/src/ifcopenshell-python/test/test_parse.py
index d40101333e..ad2ae40f0b 100644
--- a/src/ifcopenshell-python/test/test_parse.py
+++ b/src/ifcopenshell-python/test/test_parse.py
@@ -1,3 +1,5 @@
+import pytest
+
 import ifcopenshell
 
 
@@ -32,3 +34,25 @@ def test_reference_to_undefined_owning_instance_simple_type():
     data = "ISO-10303-21;HEADER;FILE_DESCRIPTION();FILE_NAME();FILE_SCHEMA(('IFC4'));#=IFCPROJECT((#))#4=IFCSIUNIT("
     ifcopenshell.file.from_string(data)
     print(ifcopenshell.get_log())
+
+
+@pytest.mark.parametrize(
+    "encoded,expected",
+    [
+        ("AA\\X2\\D83DDE00\\X0\\BB", "AA\U0001f600BB"),
+        ("AA\\X4\\0001F600\\X0\\BB", "AA\U0001f600BB"),
+        ("AA\\X2\\00E9\\X0\\BB", "AAéBB"),
+        ("\\X2\\00E9D83DDE0000E9\\X0\\", "é\U0001f600é"),
+        ("\\X2\\D83DDE00D83DDE01\\X0\\", "\U0001f600\U0001f601"),
+        ("Bld \\X2\\5EFA7BC9\\X0\\ ok", "Bld 建築 ok"),
+    ],
+)
+def test_x2_utf16_surrogate_pairs(encoded, expected):
+    data = (
+        "ISO-10303-21;HEADER;FILE_DESCRIPTION((''),'2;1');FILE_NAME('','',(''),(''),'','','');"
+        "FILE_SCHEMA(('IFC4'));ENDSEC;DATA;"
+        "#1=IFCPROJECT('0YvctVUKr0kugbFTf53O9L',$,'" + encoded + "',$,$,$,$,$,$);"
+        "ENDSEC;END-ISO-10303-21;"
+    )
+    f = ifcopenshell.file.from_string(data)
+    assert f.by_id(1).Name == expected
diff --git a/src/ifcparse/IfcCharacterDecoder.cpp b/src/ifcparse/IfcCharacterDecoder.cpp
index 019499a49e..6b8fd7562d 100644
--- a/src/ifcparse/IfcCharacterDecoder.cpp
+++ b/src/ifcparse/IfcCharacterDecoder.cpp
@@ -98,6 +98,7 @@ namespace {
         int codepage = 1;
         unsigned int hex = 0;
         unsigned int hex_count = 0;
+        unsigned int high_surrogate = 0;
 
         while ((current_char = stream_.peek()) != 0) {
             if (EXPECTS_CHARACTER(parse_state)) {
@@ -111,7 +112,7 @@ namespace {
                 if (((parse_state & ALPHABET_DEFINITION) != 0U) ||
                     ((parse_state & IGNORED_DIRECTIVE) != 0U) ||
                     ((parse_state & ENDEXTENDED_0) != 0U)) {
-                    parse_state = hex = hex_count = 0;
+                    parse_state = hex = hex_count = high_surrogate = 0;
                 } else if ((parse_state & ENCOUNTERED_HEX) != 0U) {
                     parse_state += THIRD_SOLIDUS;
                     parse_state -= ENCOUNTERED_HEX;
@@ -149,7 +150,23 @@ namespace {
                 if ((hex_count == 2 && ((parse_state & EXTENDED2) == 0U)) ||
                     (hex_count == 4 && ((parse_state & EXTENDED4) == 0U)) ||
                     (hex_count == 8)) {
-                    builder_.push_back(hex);
+                    // \X2\ carries UTF-16 code units, so surrogate pairs form a single codepoint.
+                    bool awaiting_low_surrogate = false;
+                    auto codepoint = static_cast<std::u32string::value_type>(hex);
+                    if (hex_count == 4) {
+                        if (hex >= 0xd800 && hex <= 0xdbff) {
+                            high_surrogate = hex;
+                            awaiting_low_surrogate = true;
+                        } else if ((high_surrogate != 0U) && hex >= 0xdc00 && hex <= 0xdfff) {
+                            codepoint = 0x10000 + ((high_surrogate - 0xd800) << 10) + (hex - 0xdc00);
+                            high_surrogate = 0;
+                        } else {
+                            high_surrogate = 0;
+                        }
+                    }
+                    if (!awaiting_low_surrogate) {
+                        builder_.push_back(codepoint);
+                    }
                     if (hex_count == 2) {
                         parse_state = 0;
                     } else {