Added ifc4x3 json schema and it's parser

API for docs works the same way as it worked for ifc2x3 and ifc4. Besides changing `doc.py` and adding `.json` schemas added `ifc4x3dev_scrape_data_for_docs.py` (it's kind of hacky now and relies on `server.py` code from the further mentioned repo) that's used to parse some entities descriptions from https://github.com/buildingSMART/IFC4.3.x-development for later use in `doc.py`.
2026-08-10 09:48:32 +00:00 · 2023-02-14 16:39:27 +05:00
parent a6fa6201f7
commit 476ab506d0
5 changed files with 27926 additions and 7 deletions
@@ -1,5 +1,5 @@
 # IfcOpenShell - IFC toolkit and geometry engine
-# Copyright (C) 2022 @Andrej730
+# Copyright (C) 2022, 2023 @Andrej730
 #
 # This file is part of IfcOpenShell.
 #
@@ -21,6 +21,7 @@ from pathlib import Path
 from pprint import pprint
 import copy
 import ifcopenshell
+import ifcopenshell.util.attribute

 try:
    import glob
@@ -28,8 +29,11 @@ try:
    import requests
    import urllib.parse
    from markdown import markdown
-    from bs4 import BeautifulSoup
-    from bs4 import MarkupResemblesLocatorWarning
+    from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
+    import zipfile
+    from lxml import etree
+    import re
+    import shutil
 except:
    pass  # Only necessary if you're using it to generate the docs database

@@ -38,6 +42,23 @@ BASE_MODULE_PATH = Path(__file__).parent
 IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc2.3.0.1"
 IFC4_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc4.0.2.1"

+IFC4x3_HTML_LOCATION = BASE_MODULE_PATH / "IFC4.3-html"
+IFC4x3_DEV_LOCATION = BASE_MODULE_PATH / "IFC4.3.x-development"
+IFC4x3_SPEC_URL_TEMPLATE = "https://ifc43-docs.standards.buildingsmart.org/IFC/RELEASE/IFC4x3/HTML/lexical/%s.htm"
+
+# entities schema
+# entity -> description, spec_url, attributes[], predefined_types[]
+
+# types schema
+# type -> description, spec_url
+
+# properties schema
+# pset/qset -> description, spec_url, properties[]
+# property -> description, children[]
+# child -> description
+# note: in IFC4x3 there is no children[] for properties
+
+
 SCHEMA_FILES = {
    "IFC2X3": {
        "entities": BASE_MODULE_PATH / "schema/ifc2x3_entities.json",
@@ -49,10 +70,15 @@ SCHEMA_FILES = {
        "properties": BASE_MODULE_PATH / "schema/ifc4_properties.json",
        "types": BASE_MODULE_PATH / "schema/ifc4_types.json",
    },
+    "IFC4X3": {
+        "entities": BASE_MODULE_PATH / "schema/ifc4x3_entities.json",
+        "properties": BASE_MODULE_PATH / "schema/ifc4x3_properties.json",
+        "types": BASE_MODULE_PATH / "schema/ifc4x3_types.json",
+    },
 }

 db = None
-schema_by_name = {"IFC2X3": None, "IFC4": None}
+schema_by_name = {"IFC2X3": None, "IFC4": None, "IFC4X3": None}


 def get_db(version):
@@ -89,7 +115,7 @@ def get_entity_doc(version, entity_name, recursive=True):
        ifc_schema = get_schema_by_name(version)
        ifc_entity = ifc_schema.declaration_by_name(entity_name)
        ifc_supertype = ifc_entity.supertype()
-        if ifc_entity.supertype():
+        if ifc_supertype:
            parent_entity = get_entity_doc(version, ifc_supertype.name(), recursive=True)
            if "attributes" not in entity:
                entity["attributes"] = dict()
@@ -134,7 +160,37 @@ def get_type_doc(version, ifc_type):
        return db["types"].get(ifc_type)


+def get_inverse_attributes(el):
+    inverse_attrs = []
+    for a in el.all_inverse_attributes():
+        attribute_type = a.attribute_reference().type_of_attribute()
+        if isinstance(attribute_type, ifcopenshell.ifcopenshell_wrapper.aggregation_type):
+            attribute_type = attribute_type.type_of_element()
+        attribute_type = attribute_type.declared_type()
+
+        if not isinstance(attribute_type, ifcopenshell.ifcopenshell_wrapper.entity):
+            attr_types = [t.name() for t in attribute_type.select_list()]
+        else:
+            attr_types = [attribute_type.name()]
+
+        if el.name() in attr_types:
+            inverse_attrs.append(a)
+    return inverse_attrs
+
+
 class DocExtractor:
+    def clean_highlighted_words(self, text):
+        text = re.sub(r"\b_([a-zA-Z0-9]+)_\b", r"\1", text)
+        text = re.sub(r"\*\*([a-zA-Z0-9]+)\*\*", r"\1", text)
+        return text
+
+    def clean_description(self, description):
+        description = description.replace("\n", " ")
+        description = description.replace("\u00a0", " ")
+        description = description.split("HISTORY:", 1)[0]
+        description = description.strip()
+        return description
+
    def extract_ifc2x3(self):
        print("Parsing data for Ifc2.3.0.1")
        if not IFC2x3_DOCS_LOCATION.is_dir():
@@ -373,7 +429,7 @@ class DocExtractor:
                if len(tags) != 1:
                    print(
                        f"WARNING. Found more properties inside property xml, "
-                        f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
+                        f"only the first one was parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
                    )
                property_name = tags[0]["name"]

@@ -711,7 +767,7 @@ class DocExtractor:
                if len(tags) != 1:
                    print(
                        f"WARNING. Found more properties inside property xml, "
-                        f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
+                        f"only the first one was parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
                    )
                property_name = tags[0]["name"]

@@ -783,36 +839,211 @@ class DocExtractor:
            print(f"{len(types_dict)} ifc types parsed")
            json.dump(types_dict, fo, sort_keys=True, indent=4)

+    def extract_ifc4x3(self):
+        print("Parsing data for Ifc4.3.0.1")
+        if not IFC4x3_DEV_LOCATION.is_dir():
+            raise Exception(
+                f'Specs development repository for Ifc4.3.0.1 expected to be in folder "{IFC4x3_DEV_LOCATION.resolve()}\\"\n'
+                "For doc extraction please either setup docs as described above \n"
+                "or change IFC4x3_DEV_LOCATION in doc.py accordingly."
+                "You can download docs from the repository: \n"
+                "https://github.com/buildingSMART/IFC4.3.x-development"
+            )
+        if not IFC4x3_HTML_LOCATION.is_dir():
+            raise Exception(
+                f'Formal release for Ifc4.3.0.1 expected to be in folder "{IFC4x3_HTML_LOCATION.resolve()}\\"\n'
+                "For doc extraction please either setup docs as described above \n"
+                "or change IFC4x3_HTML_LOCATION in doc.py accordingly."
+                "You can download docs from the repository: \n"
+                "https://github.com/buildingsmart/ifc4.3-html"
+            )
+        dev_code_path = IFC4x3_DEV_LOCATION / "code"
+        description_json_path = dev_code_path / "entities_description.json"
+        if not description_json_path.is_file():
+            shutil.copy(
+                BASE_MODULE_PATH / "ifc4x3dev_scrape_data_for_docs.py",
+                dev_code_path / "ifc4x3dev_scrape_data_for_docs.py",
+            )
+            raise Exception(
+                f'The entities description data expected to be located in \n"{description_json_path.resolve()}.\n'
+                f"To generate it `ifc4x3dev_scrape_data_for_docs.py` will be copied from current folder to \n{dev_code_path}\n"
+                "and you'll need to run in from `/code` folder.\nThis script will use development `server.py` "
+                "module to extract entities descriptions.\n\n"
+                "Before running it make sure you run `create_resources.sh` from `/code` folder first.\n"
+                "You'll need to complete at least 3 commands from `create_resources.sh`:\n"
+                "   py extract_concepts_from_xmi.py ../schemas/IFC.xml\n"
+                "   py to_pset.py ../schemas/IFC.xml psd\n"
+                "   py parse_xmi.py ../schemas/IFC.xml"
+            )
+
+        self.extract_ifc4x3_entities()
+        self.extract_ifc4x3_property_sets()
+
+    def extract_ifc4x3_entities(self):
+        with open(IFC4x3_DEV_LOCATION / "code/entities_description.json", "r") as fi:
+            entities_description = json.load(fi)
+
+        entities_dict = dict()
+        types_dict = dict()
+        schema = ifcopenshell.ifcopenshell_wrapper.schema_by_name("IFC4X3")
+
+        for entity in schema.declarations():
+            entity_name = entity.name()
+
+            entity_data = dict()
+            entity_data["spec_url"] = IFC4x3_SPEC_URL_TEMPLATE % entity_name
+
+            if entity_name not in entities_description:
+                print(
+                    f"WARNING. Entity {entity_name} is not present in data parsed from DEV DOCUMENTATION "
+                    "even though it's present in ifcopenshell schema. It's description will be left empty."
+                )
+                description = ""
+            else:
+                description = self.clean_highlighted_words(entities_description[entity_name]["description"])
+            entity_data["description"] = description
+
+            # types = type_declaration + enumeration_type + select_type
+            if not isinstance(entity, ifcopenshell.ifcopenshell_wrapper.entity):
+                types_dict[entity_name] = entity_data
+                continue
+
+            # entities processing
+            # assign attributes / predef types data
+            parsed_attributes_data = entities_description[entity_name]["attributes"]
+            parsed_predefined_types_data = entities_description[entity_name]["predefined_types"]
+            attributes_data = dict()
+            predefined_types = dict()
+
+            # iterate over forward and inverse entity attributes
+            # TODO: more eloquent way to get inverse attributes of the declaration?
+            for a in list(entity.attributes()) + get_inverse_attributes(entity):
+                attr_name = a.name()
+                # predefined types
+                if attr_name == "PredefinedType":
+                    for v in ifcopenshell.util.attribute.get_enum_items(a):
+                        if v not in parsed_predefined_types_data:
+                            print(
+                                f"WARNING. Predefined type {v} (of entity {entity_name}) is not present in data parsed from DEV DOCUMENTATION "
+                                "even though it's present in ifcopenshell schema. It's description will be left empty."
+                            )
+                            description = ""
+                        else:
+                            description = self.clean_description(parsed_predefined_types_data[v])
+                        predefined_types[v] = description
+                    continue
+                
+                # attributes
+                if attr_name not in parsed_attributes_data:
+                    print(
+                        f"WARNING. Attribute {attr_name} (of entity {entity_name}) is not present in data parsed from DEV DOCUMENTATION "
+                        "even though it's present in ifcopenshell schema. It's description will be left empty."
+                    )
+                    description = ""
+                else:
+                    description = self.clean_description(parsed_attributes_data[attr_name])
+                attributes_data[attr_name] = description
+
+            if attributes_data:
+                entity_data["attributes"] = attributes_data
+            if predefined_types:
+                entity_data["predefined_types"] = predefined_types
+
+            entities_dict[entity_name] = entity_data
+
+        # export entities data
+        with open(BASE_MODULE_PATH / "schema/ifc4x3_entities.json", "w", encoding="utf-8") as fo:
+            print(f"{len(entities_dict)} entities parsed")
+            json.dump(entities_dict, fo, sort_keys=True, indent=4)
+
+        # export entities data
+        with open(BASE_MODULE_PATH / "schema/ifc4x3_types.json", "w", encoding="utf-8") as fo:
+            print(f"{len(types_dict)} ifc types parsed")
+            json.dump(types_dict, fo, sort_keys=True, indent=4)
+
+    def extract_ifc4x3_property_sets(self):
+        pset_data_zip = IFC4x3_HTML_LOCATION / "IFC/RELEASE/IFC4x3/HTML/annex-a-psd.zip"
+        pset_data_location = BASE_MODULE_PATH / "temp/annex-a-psd"
+        with zipfile.ZipFile(pset_data_zip, "r") as fi_zip:
+            fi_zip.extractall(pset_data_location)
+
+        property_sets_dict = dict()
+
+        for pset_path in glob.iglob(f"{pset_data_location}/*.xml"):
+            pset_path = Path(pset_path)
+            pset_name = pset_path.stem
+
+            # pset / qset
+            pset_type = True if pset_name.split("_")[0] == "Pset" else False
+
+            pset_data = dict()
+            pset_data["spec_url"] = IFC4x3_SPEC_URL_TEMPLATE % pset_name
+
+            with open(pset_path, "r", encoding="utf-8") as fi:
+                root_xml = etree.fromstring(fi.read())
+
+            description = root_xml.find("Definition").text
+            pset_data["description"] = self.clean_description(description)
+
+            # parsing pset/qset properties data
+            prop_data = dict()
+            search_tag = "PropertyDef" if pset_type else "QtoDef"
+            props = root_xml.find(search_tag + "s").findall(search_tag)
+            for prop in props:
+                prop_name = prop.find("Name").text
+                prop_description = prop.find("Definition").text
+                if not prop_description:  # it could be just `<Definition/>`
+                    prop_description = ""
+                prop_description = self.clean_description(prop_description)
+                prop_data[prop_name] = {"description": prop_description}
+
+            pset_data["properties"] = prop_data
+            property_sets_dict[pset_name] = pset_data
+
+        # export property sets data
+        with open(BASE_MODULE_PATH / "schema/ifc4x3_properties.json", "w", encoding="utf-8") as fo:
+            print(f"{len(property_sets_dict)} property sets parsed")
+            json.dump(property_sets_dict, fo, sort_keys=True, indent=4)
+
+        shutil.rmtree(pset_data_location)
+

 def run_doc_api_examples():
    print("Entities (with parent entities attributes included):")
    print(get_entity_doc("IFC2X3", "IfcWindow"))
    print(get_entity_doc("IFC4", "IfcWindow"))
+    print(get_entity_doc("IFC4X3", "IfcWindow"))

    print("Entity attributes (with parent entities attributes included):")
    print(get_attribute_doc("IFC2X3", "IfcWindow", "OwnerHistory"))
    print(get_attribute_doc("IFC4", "IfcWindow", "OwnerHistory"))
+    print(get_attribute_doc("IFC4X3", "IfcWindow", "OwnerHistory"))

    print("Entity predefined types:")
    print(get_predefined_type_doc("IFC2X3", "IfcControllerType", "FLOATING"))
    print(get_predefined_type_doc("IFC4", "IfcControllerType", "FLOATING"))
+    print(get_predefined_type_doc("IFC4X3", "IfcControllerType", "FLOATING"))

    print("Propety sets:")
    print(get_property_set_doc("IFC2X3", "Pset_ZoneCommon"))
    print(get_property_set_doc("IFC4", "Pset_ZoneCommon"))
+    print(get_property_set_doc("IFC4X3", "Pset_ZoneCommon"))

    print("Propety sets attributes:")
    print(get_property_doc("IFC2X3", "Pset_ZoneCommon", "Category"))
    print(get_property_doc("IFC4", "Pset_ZoneCommon", "NetPlannedArea"))
+    print(get_property_doc("IFC4X3", "Pset_ZoneCommon", "NetPlannedArea"))

    print("Types:")
    print(get_type_doc("IFC2X3", "IfcIsothermalMoistureCapacityMeasure"))
    print(get_type_doc("IFC4", "IfcDuration"))
+    print(get_type_doc("IFC4X3", "IfcDuration"))


 if __name__ == "__main__":
    extractor = DocExtractor()
    extractor.extract_ifc2x3()
    extractor.extract_ifc4()
+    extractor.extract_ifc4x3()

    # run_doc_api_examples()