Reimplement sort / reverse / join function to format language, simplify text annotation variables, add tests

Previously, sort, reverse list, and join functionality was implemented
as special cases in Bonsai itself. Given that it has usecases
(especially in material lists, but any sort of list applies) I've moved
this function into the IOS formatting language.

The IOS formatting language previously wasn't capable of this, but the
awesome addition by @falken10vdl made the formatting language accept
queries inline, so that means it can handle lists. I also added tests
for all the new functions and expression syntax (+-*/ operators).

I simplified the code that gets the evaluated text literal - previously
it seems to call format() multiple times.

src/bonsai/bonsai/bim/module/drawing/data.py

@@ -365,15 +365,10 @@ class DecoratorData:
 
         for literal in literals:
             literal_value = literal.Literal
-            try:
-                eval_value = cls.evaluate_formatting_expressions(literal_value, product)
-                current_value = tool.Drawing.replace_text_literal_variables(eval_value, product)
-            except Exception:
-                current_value = literal_value
             literal_data = {
                 "Literal": literal_value,
                 "BoxAlignment": literal.BoxAlignment,
-                "CurrentValue": current_value,
+                "CurrentValue": tool.Drawing.replace_text_literal_variables(literal_value, product),
             }
             literals_data.append(literal_data)
 
@@ -399,21 +394,6 @@ class DecoratorData:
 
         return element
 
-    @classmethod
-    def evaluate_formatting_expressions(cls, text: str, element=None) -> str:
-        """Evaluate formatting expressions wrapped in backticks using ifcopenshell.util.selector.format, always passing element context"""
-        import re
-
-        def evaluate_expression(match):
-            try:
-                expression = match.group(1)
-                result = ifcopenshell.util.selector.format(expression, element)
-                return str(result)
-            except Exception as e:
-                return match.group(0)
-
-        return re.sub(r"``([^`]+)``", evaluate_expression, text)
-
     @classmethod
     def get_element_value_by_key(cls, element: ifcopenshell.entity_instance, key: str):
         """Get element value by its key using IfcOpenShell selector syntax"""

src/bonsai/bonsai/tool/drawing.py

@@ -1306,9 +1306,7 @@ class Drawing(bonsai.core.tool.Drawing):
         element = tool.Ifc.get_entity(obj)
         assert element
         # updating text font size in EPset_Annotation.Classes
-        print("we got", font_size, repr(font_size))
         font_size_str = next((key for key in FONT_SIZES if FONT_SIZES[key] == font_size), None)
-        print("so", font_size_str)
         classes = ifcopenshell.util.element.get_pset(element, "EPset_Annotation", "Classes")
         assert isinstance(classes, Union[str, None])
         classes_split = classes.split() if classes else []
@@ -2101,16 +2099,13 @@ class Drawing(bonsai.core.tool.Drawing):
         if not product:
             return text
 
-        for command in re.findall("``.*?``", text):
+        for command in re.findall("``.+?``", text):
             original_command = command
             command_content = command[2:-2]
-            if command_content is None or str(command_content).strip().lower() == "none":
+            try:
+                text = text.replace(original_command, ifcopenshell.util.selector.format(command_content, product))
+            except Exception:
                 text = text.replace(original_command, "")
-            else:
-                try:
-                    text = text.replace(original_command, ifcopenshell.util.selector.format(command_content, product))
-                except Exception:
-                    text = text.replace(original_command, "")
         for variable in re.findall("{{.*?}}", text):
             value = ifcopenshell.util.selector.get_element_value(product, variable[2:-2])
             if isinstance(value, (list, tuple)):

src/ifcopenshell-python/docs/ifcopenshell-python/selector_syntax.rst

@@ -252,6 +252,10 @@ nest formulas, for example ``concat(title("foo"), lower("Bar"))`` will produce
     "``number({{value}}[, {{decimal_separator}}[, {{thousands_separator}}]])``", "``number(1234.56, "","", ""."")``", "``1.234,56``", "Formats {{value}} with an optional custom {{decimal_separator}} and {{thousands_separator}}. The default separators are ``.`` and ``,``."
     "``metric_length({{value}}, {{precision}}, {{decimals}})``", "``metric_length(3.123, 0.1, 2)``", "``3.10``", "Rounds ``{{value}}`` to the nearest ``{{precision}}`` then displays using a certain amount of decimal places."
     "``imperial_length({{value}}, {{precision}}, {{input_unit}}, {{output_unit}}, {{suppress_zero_inches}})``", "``imperial_length(3.0, 4, ""foot"", ""foot"", true)`` OR ``imperial_length(3.0, 4, ""foot"", ""foot"", false)``", "``3'`` OR ``3' - 0""``", "The ``{{value}}`` may be specified either as ``foot`` or ``inch`` depending on ``{{input_unit}}``. The ``{{value}}`` is then rounded to the nearest ``1/{{precision}}`` inch, then formatted using fractional feet and inches if ``{{output_unit}}`` is set to ``foot``, or just inches if ``{{output_unit}}`` is set to ``inch``. When ``{{suppress_zero_inches}}`` is ``true`` (default), measurements with zero inches will omit the inch portion (e.g., ``3'`` instead of ``3' - 0""``)."
+    "``sort({{values}})``", "``sort({{mats.Name}})``", "``Name1, Name2``", "Sorts a list of items."
+    "``reverse({{values}})``", "``reverse({{mats.Name}})``", "``Name2, Name1``", "Reverses a list of items."
+    "``join({{separator}}, {{values}})``", "``join("-", {{mats.Name}})``", "``Name1-Name2``", "Joins a list of items with a custom separator. By default, all lists a rendered as comma separated."
+    "``{{value1}}[+-*/]{{value2}}``", "``{{z}}+3``", "``5``", "Does arithmetic. Typical operators such as +, -, \*, and / are allowed and can be mixed with other variables and formatting functions."
 
 When using queries in an IfcAnnotation tag surround with backticks. 
 Examples: 

src/ifcopenshell-python/ifcopenshell/util/selector.py

@@ -144,7 +144,7 @@ format_grammar = lark.Lark(
         | mul_div "*" function  -> multiply
         | mul_div "/" function  -> divide
     
-    function: round | number | int | format_length | lower | upper | title | concat | substr | variable | ESCAPED_STRING | SIGNED_NUMBER | "(" expression ")"
+    function: round | number | int | format_length | lower | upper | title | concat | substr | sort | reverse | join | variable | ESCAPED_STRING | SIGNED_NUMBER | "(" expression ")"
 
     variable: "{{" query_path "}}"
     query_path: /[^}]+/
@@ -160,6 +160,9 @@ format_grammar = lark.Lark(
     title: "title(" expression ")"
     concat: "concat(" expression ("," expression)* ")"
     substr: "substr(" expression "," SIGNED_INT ["," SIGNED_INT] ")"
+    sort: "sort(" expression ")"
+    reverse: "reverse(" expression ")"
+    join: "join(" ESCAPED_STRING "," expression ")"
     boolean: TRUE | FALSE
 
     TRUE: "true" | "True" | "TRUE"
@@ -201,6 +204,8 @@ class FormatTransformer(lark.Transformer):
         self.element = element
 
     def start(self, args):
+        if isinstance(args[0], (list, tuple)):
+            return ", ".join(args[0])
         return args[0]
 
     def expression(self, args):
@@ -208,18 +213,11 @@ class FormatTransformer(lark.Transformer):
 
     def variable(self, args):
         """Handle variable substitution like {{z}} or {{Pset_Wall.FireRating}}"""
-        if self.element is None:
-            return "0"  # Default value if no element context
-
-        query_path = args[0]
-        try:
-            value = get_element_value(self.element, query_path)
-            if value is None:
-                return "0"
-            # Convert to string for further processing
-            return str(value)
-        except:
-            return "0"  # Return default on error
+        if self.element:
+            try:
+                return get_element_value(self.element, args[0])
+            except:
+                pass
 
     def query_path(self, args):
         """Extract the query path from variable"""
@@ -301,6 +299,15 @@ class FormatTransformer(lark.Transformer):
         elif len(args) == 2:
             return str(args[0])[int(args[1]) :]
 
+    def sort(self, args):
+        return sorted(args[0])
+
+    def reverse(self, args):
+        return list(reversed(args[0]))
+
+    def join(self, args):
+        return args[0].join(args[1])
+
     def boolean(self, args):
         if not args:
             return True

src/ifcopenshell-python/test/util/test_selector.py

@@ -35,7 +35,7 @@ import ifcopenshell.util.selector as subject
 import test.bootstrap
 
 
-class TestFormat:
+class TestFormat(test.bootstrap.IFC4):
     def test_no_formatting(self):
         assert subject.format("123") == "123"
         assert subject.format('"123"') == "123"
@@ -78,6 +78,38 @@ class TestFormat:
         assert subject.format('imperial_length(3.0, 4, "foot", "foot", false)') == "3' - 0\""
         assert subject.format('imperial_length(3.0, 4, "foot", "foot", False)') == "3' - 0\""
 
+    def test_variable_formatting(self):
+        assert subject.format('{{undefined}}') is None
+        assert subject.format('upper({{undefined}})') == "NONE"
+        assert subject.format('int({{undefined}})') == "0"
+        element = ifcopenshell.api.root.create_entity(self.file, ifc_class="IfcWall")
+        assert subject.format('{{undefined}}', element) is None
+        assert subject.format('{{class}}', element) == "IfcWall"
+        assert subject.format('{{ class }}', element) == "IfcWall"
+        assert subject.format('upper({{ class }})', element) == "IFCWALL"
+
+    def test_list_formatting(self):
+        element = ifcopenshell.api.root.create_entity(self.file, ifc_class="IfcWall")
+        material = ifcopenshell.api.material.add_material(self.file, name="CON01")
+        material2 = ifcopenshell.api.material.add_material(self.file, name="CON03")
+        material3 = ifcopenshell.api.material.add_material(self.file, name="CON02")
+        material_set = ifcopenshell.api.material.add_material_set(self.file, set_type="IfcMaterialLayerSet")
+        layer = ifcopenshell.api.material.add_layer(self.file, layer_set=material_set, material=material)
+        layer = ifcopenshell.api.material.add_layer(self.file, layer_set=material_set, material=material2)
+        layer = ifcopenshell.api.material.add_layer(self.file, layer_set=material_set, material=material3)
+        ifcopenshell.api.material.assign_material(self.file, products=[element], material=material_set)
+        assert subject.format('{{materials.Name}}', element) == "CON01, CON03, CON02"
+        assert subject.format('sort({{materials.Name}})', element) == "CON01, CON02, CON03"
+        assert subject.format('reverse({{materials.Name}})', element) == "CON02, CON03, CON01"
+        assert subject.format('join("-", {{materials.Name}})', element) == "CON01-CON03-CON02"
+
+    def test_expressions(self):
+        assert subject.format('2+3') == "5"
+        assert subject.format('-2+3') == "1"
+        assert subject.format('2-3') == "-1"
+        assert subject.format('3*2') == "6"
+        assert subject.format('3/2') == "1.5"
+
 
 class TestGetElementValue(test.bootstrap.IFC4):
     def test_selecting_an_elements_class_or_id_using_a_query(self):