rules fixtures: check number of expected failures

src/ifcopenshell-python/test/fixtures/rules/generate.py

@@ -1,13 +1,21 @@
+import re
 import subprocess
 import sys
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal, TypeAlias
 
 import ifcopenshell
 
-directory = Path(__file__).parent
 
-Result: TypeAlias = Literal["fail", "pass"]
+@dataclass
+class FailObj:
+    expected_count: int = 1
+
+
+PassResult: TypeAlias = Literal["pass"]
+
+Result = FailObj | PassResult
 
 
 def normalize_header(f: ifcopenshell.file) -> None:
@@ -16,21 +24,40 @@ def normalize_header(f: ifcopenshell.file) -> None:
     f.header.file_name.originating_system = "IfcOpenShell"
 
 
-def pass_if(condition: bool) -> Result:
+def pass_if(condition: bool, *, errors_if_fail: int = 1) -> Result:
-    return "pass" if condition else "fail"
+    return "pass" if condition else FailObj(expected_count=errors_if_fail)
 
 
-def fail_if(condition: bool) -> Result:
+def fail_if(condition: bool, *, errors_if_fail: int = 1) -> Result:
-    return "fail" if condition else "pass"
+    return pass_if(not condition, errors_if_fail=errors_if_fail)
 
 
-def write_fixture(file: ifcopenshell.file, source: str, result: Result, path: str) -> None:
+def parse_result(fixture_path: Path) -> Result:
+    name = fixture_path.name
+    if name.startswith("fail-"):
+        expected_count = 1
+        if match := re.search(r"^fail-expected-(\d+)", name):
+            expected_count = int(match.group(1))
+        return FailObj(expected_count=expected_count)
+    if name.startswith("pass-"):
+        return "pass"
+    raise ValueError(f"Fixture filename must start with 'fail-' or 'pass-': {name}")
+
+
+def write_fixture(file: ifcopenshell.file, source: str, result: Result, stem: str) -> None:
     content = file.to_string()
     content += f"/* IFC fixture is generated by {Path(source).name} */\n"
-    Path(f"{result}-{path}.ifc").write_text(content)
+    match result:
+        case FailObj(expected_count=expected_count):
+            prefix = "fail" if expected_count == 1 else f"fail-expected-{expected_count}"
+        case _:
+            prefix = result
+    Path(f"{prefix}-{stem}.ifc").write_text(content)
 
 
 if __name__ == "__main__":
+    directory = Path(__file__).parent
+
     for path in directory.glob("*.ifc"):
         path.unlink()
 

src/ifcopenshell-python/test/fixtures/rules/generate_02.py

@@ -4,24 +4,24 @@ import ifcopenshell
 from generate import normalize_header, pass_if, write_fixture
 
 latitudes = [
-    (False, (-361, 0, 0)),
+    (False, (-361, 0, 0), 1),
-    (False, (-361, 0, 0, 0)),
+    (False, (-361, 0, 0, 0), 1),
-    (True, (-360, 0, 0, 0)),
+    (True, (-360, 0, 0, 0), 1),
-    (True, (0, 0, 0, 0)),
+    (True, (0, 0, 0, 0), 1),
-    (True, (359, 0, 0, 0)),
+    (True, (359, 0, 0, 0), 1),
-    (False, (360, 0, 0, 0)),
+    (False, (360, 0, 0, 0), 1),
-    (True, (30, 30, 30)),
+    (True, (30, 30, 30), 1),
-    (False, (1000, 1000, 1000)),
+    (False, (1000, 1000, 1000), 3),
-    (False, (0, 0, 1000)),
+    (False, (0, 0, 1000), 1),
-    (False, (0, 0, 1000, 0)),
+    (False, (0, 0, 1000, 0), 1),
-    (True, (0, 0, 0, 100000)),
+    (True, (0, 0, 0, 100000), 1),
-    (True, (1, 1, 1, 1)),
+    (True, (1, 1, 1, 1), 1),
-    (True, (-1, -1, -1, -1)),
+    (True, (-1, -1, -1, -1), 1),
-    (False, (-1, -1, 1, 1)),
+    (False, (-1, -1, 1, 1), 1),
-    (False, (1, -1, -1, -1)),
+    (False, (1, -1, -1, -1), 1),
 ]
 
-for i, (is_valid, lat) in enumerate(latitudes):
+for i, (is_valid, lat, errors_if_fail) in enumerate(latitudes):
     f = ifcopenshell.file(schema="IFC2X3")
     p = f.createIfcPerson(Id="tfk", GivenName="Thomas")
     o = f.createIfcOrganization(Name="AECgeeks")
@@ -47,4 +47,4 @@ for i, (is_valid, lat) in enumerate(latitudes):
     )
     f.createIfcRelAggregates(ifcopenshell.guid.new(), ownerhist, None, None, proj, [site])
     normalize_header(f)
-    write_fixture(f, __file__, pass_if(is_valid), f"site-latitude-{i}-ifc2x3")
+    write_fixture(f, __file__, pass_if(is_valid, errors_if_fail=errors_if_fail), f"site-latitude-{i}-ifc2x3")

src/ifcopenshell-python/test/fixtures/rules/generate_10.py

@@ -16,10 +16,15 @@ for d, w in itertools.product(depths, widths):
 
     f = ifcopenshell.file(schema="IFC2X3")
 
-    valid = (Girth < (Depth / 2.0)) and ((WallThickness < Width / 2.0) and (WallThickness < Depth / 2.0))
+    girth_valid = Girth < (Depth / 2.0)
+    wallthickness_valid = (WallThickness < Width / 2.0) and (WallThickness < Depth / 2.0)
+    valid = girth_valid and wallthickness_valid
+    errors_if_fail = int(not girth_valid) + int(not wallthickness_valid)
 
     inst = f.createIfcCShapeProfileDef(
         "AREA", None, f.createIfcAxis2Placement2D(f.createIfcCartesianPoint((0.0, 0.0))), **D
     )
     normalize_header(f)
-    write_fixture(f, __file__, pass_if(valid), f"cshape-profile-width-{w}-depth-{d}-ifc2x3")
+    write_fixture(
+        f, __file__, pass_if(valid, errors_if_fail=errors_if_fail), f"cshape-profile-width-{w}-depth-{d}-ifc2x3"
+    )

src/ifcopenshell-python/test/test_rules.py

@@ -1,34 +1,33 @@
-import glob
+from pathlib import Path
-import os
 
 import pytest
 import tabulate
 
 import ifcopenshell.express.rule_executor
 import ifcopenshell.validate
+from .fixtures.rules.generate import FailObj, Result, parse_result
 
 
-def pytest_generate_tests(metafunc):
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
-    if "filename" not in metafunc.fixturenames:
+    if "filepath" not in metafunc.fixturenames:
         return
     rule = metafunc.config.getoption("--rule")
-    filenames = [
+    filepaths = [fp for fp in (Path(__file__).parent / "fixtures/rules").glob("*.ifc") if not rule or rule in fp.name]
-        fn
+    metafunc.parametrize(
-        for fn in glob.glob(os.path.join(os.path.dirname(__file__), "fixtures/rules/*.ifc"))
+        "filepath,expected_result",
-        if not rule or rule in os.path.basename(fn)
+        [(fp, parse_result(fp)) for fp in filepaths],
-    ]
+        ids=[fp.name for fp in filepaths],
-    metafunc.parametrize("filename", filenames, ids=[os.path.basename(fn) for fn in filenames])
+    )
 
 
-def test_file(filename):
+def test_file(filepath: Path, expected_result: Result):
-    base = os.path.basename(filename)
+    file = ifcopenshell.open(filepath)
-    file = ifcopenshell.open(filename)
     logger = ifcopenshell.validate.json_logger()
     ifcopenshell.express.rule_executor.run(file, logger)
     results = logger.statements
 
     print()
-    print(base)
+    print(filepath.name)
     print()
     print(f"{len(results)} errors")
 
@@ -42,10 +41,11 @@ def test_file(filename):
             )
         )
 
-    if base.startswith("fail-"):
+    match expected_result:
-        assert len(results) > 0
+        case FailObj(expected_count=expected_count):
-    if base.startswith("pass-"):
+            assert len(results) == expected_count
-        assert len(results) == 0
+        case "pass":
+            assert len(results) == 0
 
 
 if __name__ == "__main__":