Added ifc4x3 json schema and it's parser

API for docs works the same way as it worked for ifc2x3 and ifc4.

Besides changing `doc.py` and adding `.json` schemas added `ifc4x3dev_scrape_data_for_docs.py` (it's kind of hacky now and relies on `server.py` code from the further mentioned repo) that's used to parse some entities descriptions from https://github.com/buildingSMART/IFC4.3.x-development for later use in `doc.py`.

src/ifcopenshell-python/ifcopenshell/util/doc.py

@@ -1,5 +1,5 @@
 # IfcOpenShell - IFC toolkit and geometry engine
-# Copyright (C) 2022 @Andrej730
+# Copyright (C) 2022, 2023 @Andrej730
 #
 # This file is part of IfcOpenShell.
 #
@@ -21,6 +21,7 @@ from pathlib import Path
 from pprint import pprint
 import copy
 import ifcopenshell
+import ifcopenshell.util.attribute
 
 try:
     import glob
@@ -28,8 +29,11 @@ try:
     import requests
     import urllib.parse
     from markdown import markdown
-    from bs4 import BeautifulSoup
-    from bs4 import MarkupResemblesLocatorWarning
+    from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
+    import zipfile
+    from lxml import etree
+    import re
+    import shutil
 except:
     pass  # Only necessary if you're using it to generate the docs database
 
@@ -38,6 +42,23 @@ BASE_MODULE_PATH = Path(__file__).parent
 IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc2.3.0.1"
 IFC4_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc4.0.2.1"
 
+IFC4x3_HTML_LOCATION = BASE_MODULE_PATH / "IFC4.3-html"
+IFC4x3_DEV_LOCATION = BASE_MODULE_PATH / "IFC4.3.x-development"
+IFC4x3_SPEC_URL_TEMPLATE = "https://ifc43-docs.standards.buildingsmart.org/IFC/RELEASE/IFC4x3/HTML/lexical/%s.htm"
+
+# entities schema
+# entity -> description, spec_url, attributes[], predefined_types[]
+
+# types schema
+# type -> description, spec_url
+
+# properties schema
+# pset/qset -> description, spec_url, properties[]
+# property -> description, children[]
+# child -> description
+# note: in IFC4x3 there is no children[] for properties
+
+
 SCHEMA_FILES = {
     "IFC2X3": {
         "entities": BASE_MODULE_PATH / "schema/ifc2x3_entities.json",
@@ -49,10 +70,15 @@ SCHEMA_FILES = {
         "properties": BASE_MODULE_PATH / "schema/ifc4_properties.json",
         "types": BASE_MODULE_PATH / "schema/ifc4_types.json",
     },
+    "IFC4X3": {
+        "entities": BASE_MODULE_PATH / "schema/ifc4x3_entities.json",
+        "properties": BASE_MODULE_PATH / "schema/ifc4x3_properties.json",
+        "types": BASE_MODULE_PATH / "schema/ifc4x3_types.json",
+    },
 }
 
 db = None
-schema_by_name = {"IFC2X3": None, "IFC4": None}
+schema_by_name = {"IFC2X3": None, "IFC4": None, "IFC4X3": None}
 
 
 def get_db(version):
@@ -89,7 +115,7 @@ def get_entity_doc(version, entity_name, recursive=True):
         ifc_schema = get_schema_by_name(version)
         ifc_entity = ifc_schema.declaration_by_name(entity_name)
         ifc_supertype = ifc_entity.supertype()
-        if ifc_entity.supertype():
+        if ifc_supertype:
             parent_entity = get_entity_doc(version, ifc_supertype.name(), recursive=True)
             if "attributes" not in entity:
                 entity["attributes"] = dict()
@@ -134,7 +160,37 @@ def get_type_doc(version, ifc_type):
         return db["types"].get(ifc_type)
 
 
+def get_inverse_attributes(el):
+    inverse_attrs = []
+    for a in el.all_inverse_attributes():
+        attribute_type = a.attribute_reference().type_of_attribute()
+        if isinstance(attribute_type, ifcopenshell.ifcopenshell_wrapper.aggregation_type):
+            attribute_type = attribute_type.type_of_element()
+        attribute_type = attribute_type.declared_type()
+
+        if not isinstance(attribute_type, ifcopenshell.ifcopenshell_wrapper.entity):
+            attr_types = [t.name() for t in attribute_type.select_list()]
+        else:
+            attr_types = [attribute_type.name()]
+
+        if el.name() in attr_types:
+            inverse_attrs.append(a)
+    return inverse_attrs
+
+
 class DocExtractor:
+    def clean_highlighted_words(self, text):
+        text = re.sub(r"\b_([a-zA-Z0-9]+)_\b", r"\1", text)
+        text = re.sub(r"\*\*([a-zA-Z0-9]+)\*\*", r"\1", text)
+        return text
+
+    def clean_description(self, description):
+        description = description.replace("\n", " ")
+        description = description.replace("\u00a0", " ")
+        description = description.split("HISTORY:", 1)[0]
+        description = description.strip()
+        return description
+
     def extract_ifc2x3(self):
         print("Parsing data for Ifc2.3.0.1")
         if not IFC2x3_DOCS_LOCATION.is_dir():
@@ -373,7 +429,7 @@ class DocExtractor:
                 if len(tags) != 1:
                     print(
                         f"WARNING. Found more properties inside property xml, "
-                        f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
+                        f"only the first one was parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
                     )
                 property_name = tags[0]["name"]
 
@@ -711,7 +767,7 @@ class DocExtractor:
                 if len(tags) != 1:
                     print(
                         f"WARNING. Found more properties inside property xml, "
-                        f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
+                        f"only the first one was parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
                     )
                 property_name = tags[0]["name"]
 
@@ -783,36 +839,211 @@ class DocExtractor:
             print(f"{len(types_dict)} ifc types parsed")
             json.dump(types_dict, fo, sort_keys=True, indent=4)
 
+    def extract_ifc4x3(self):
+        print("Parsing data for Ifc4.3.0.1")
+        if not IFC4x3_DEV_LOCATION.is_dir():
+            raise Exception(
+                f'Specs development repository for Ifc4.3.0.1 expected to be in folder "{IFC4x3_DEV_LOCATION.resolve()}\\"\n'
+                "For doc extraction please either setup docs as described above \n"
+                "or change IFC4x3_DEV_LOCATION in doc.py accordingly."
+                "You can download docs from the repository: \n"
+                "https://github.com/buildingSMART/IFC4.3.x-development"
+            )
+        if not IFC4x3_HTML_LOCATION.is_dir():
+            raise Exception(
+                f'Formal release for Ifc4.3.0.1 expected to be in folder "{IFC4x3_HTML_LOCATION.resolve()}\\"\n'
+                "For doc extraction please either setup docs as described above \n"
+                "or change IFC4x3_HTML_LOCATION in doc.py accordingly."
+                "You can download docs from the repository: \n"
+                "https://github.com/buildingsmart/ifc4.3-html"
+            )
+        dev_code_path = IFC4x3_DEV_LOCATION / "code"
+        description_json_path = dev_code_path / "entities_description.json"
+        if not description_json_path.is_file():
+            shutil.copy(
+                BASE_MODULE_PATH / "ifc4x3dev_scrape_data_for_docs.py",
+                dev_code_path / "ifc4x3dev_scrape_data_for_docs.py",
+            )
+            raise Exception(
+                f'The entities description data expected to be located in \n"{description_json_path.resolve()}.\n'
+                f"To generate it `ifc4x3dev_scrape_data_for_docs.py` will be copied from current folder to \n{dev_code_path}\n"
+                "and you'll need to run in from `/code` folder.\nThis script will use development `server.py` "
+                "module to extract entities descriptions.\n\n"
+                "Before running it make sure you run `create_resources.sh` from `/code` folder first.\n"
+                "You'll need to complete at least 3 commands from `create_resources.sh`:\n"
+                "   py extract_concepts_from_xmi.py ../schemas/IFC.xml\n"
+                "   py to_pset.py ../schemas/IFC.xml psd\n"
+                "   py parse_xmi.py ../schemas/IFC.xml"
+            )
+
+        self.extract_ifc4x3_entities()
+        self.extract_ifc4x3_property_sets()
+
+    def extract_ifc4x3_entities(self):
+        with open(IFC4x3_DEV_LOCATION / "code/entities_description.json", "r") as fi:
+            entities_description = json.load(fi)
+
+        entities_dict = dict()
+        types_dict = dict()
+        schema = ifcopenshell.ifcopenshell_wrapper.schema_by_name("IFC4X3")
+
+        for entity in schema.declarations():
+            entity_name = entity.name()
+
+            entity_data = dict()
+            entity_data["spec_url"] = IFC4x3_SPEC_URL_TEMPLATE % entity_name
+
+            if entity_name not in entities_description:
+                print(
+                    f"WARNING. Entity {entity_name} is not present in data parsed from DEV DOCUMENTATION "
+                    "even though it's present in ifcopenshell schema. It's description will be left empty."
+                )
+                description = ""
+            else:
+                description = self.clean_highlighted_words(entities_description[entity_name]["description"])
+            entity_data["description"] = description
+
+            # types = type_declaration + enumeration_type + select_type
+            if not isinstance(entity, ifcopenshell.ifcopenshell_wrapper.entity):
+                types_dict[entity_name] = entity_data
+                continue
+
+            # entities processing
+            # assign attributes / predef types data
+            parsed_attributes_data = entities_description[entity_name]["attributes"]
+            parsed_predefined_types_data = entities_description[entity_name]["predefined_types"]
+            attributes_data = dict()
+            predefined_types = dict()
+
+            # iterate over forward and inverse entity attributes
+            # TODO: more eloquent way to get inverse attributes of the declaration?
+            for a in list(entity.attributes()) + get_inverse_attributes(entity):
+                attr_name = a.name()
+                # predefined types
+                if attr_name == "PredefinedType":
+                    for v in ifcopenshell.util.attribute.get_enum_items(a):
+                        if v not in parsed_predefined_types_data:
+                            print(
+                                f"WARNING. Predefined type {v} (of entity {entity_name}) is not present in data parsed from DEV DOCUMENTATION "
+                                "even though it's present in ifcopenshell schema. It's description will be left empty."
+                            )
+                            description = ""
+                        else:
+                            description = self.clean_description(parsed_predefined_types_data[v])
+                        predefined_types[v] = description
+                    continue
+                
+                # attributes
+                if attr_name not in parsed_attributes_data:
+                    print(
+                        f"WARNING. Attribute {attr_name} (of entity {entity_name}) is not present in data parsed from DEV DOCUMENTATION "
+                        "even though it's present in ifcopenshell schema. It's description will be left empty."
+                    )
+                    description = ""
+                else:
+                    description = self.clean_description(parsed_attributes_data[attr_name])
+                attributes_data[attr_name] = description
+
+            if attributes_data:
+                entity_data["attributes"] = attributes_data
+            if predefined_types:
+                entity_data["predefined_types"] = predefined_types
+
+            entities_dict[entity_name] = entity_data
+
+        # export entities data
+        with open(BASE_MODULE_PATH / "schema/ifc4x3_entities.json", "w", encoding="utf-8") as fo:
+            print(f"{len(entities_dict)} entities parsed")
+            json.dump(entities_dict, fo, sort_keys=True, indent=4)
+
+        # export entities data
+        with open(BASE_MODULE_PATH / "schema/ifc4x3_types.json", "w", encoding="utf-8") as fo:
+            print(f"{len(types_dict)} ifc types parsed")
+            json.dump(types_dict, fo, sort_keys=True, indent=4)
+
+    def extract_ifc4x3_property_sets(self):
+        pset_data_zip = IFC4x3_HTML_LOCATION / "IFC/RELEASE/IFC4x3/HTML/annex-a-psd.zip"
+        pset_data_location = BASE_MODULE_PATH / "temp/annex-a-psd"
+        with zipfile.ZipFile(pset_data_zip, "r") as fi_zip:
+            fi_zip.extractall(pset_data_location)
+
+        property_sets_dict = dict()
+
+        for pset_path in glob.iglob(f"{pset_data_location}/*.xml"):
+            pset_path = Path(pset_path)
+            pset_name = pset_path.stem
+
+            # pset / qset
+            pset_type = True if pset_name.split("_")[0] == "Pset" else False
+
+            pset_data = dict()
+            pset_data["spec_url"] = IFC4x3_SPEC_URL_TEMPLATE % pset_name
+
+            with open(pset_path, "r", encoding="utf-8") as fi:
+                root_xml = etree.fromstring(fi.read())
+
+            description = root_xml.find("Definition").text
+            pset_data["description"] = self.clean_description(description)
+
+            # parsing pset/qset properties data
+            prop_data = dict()
+            search_tag = "PropertyDef" if pset_type else "QtoDef"
+            props = root_xml.find(search_tag + "s").findall(search_tag)
+            for prop in props:
+                prop_name = prop.find("Name").text
+                prop_description = prop.find("Definition").text
+                if not prop_description:  # it could be just `<Definition/>`
+                    prop_description = ""
+                prop_description = self.clean_description(prop_description)
+                prop_data[prop_name] = {"description": prop_description}
+
+            pset_data["properties"] = prop_data
+            property_sets_dict[pset_name] = pset_data
+
+        # export property sets data
+        with open(BASE_MODULE_PATH / "schema/ifc4x3_properties.json", "w", encoding="utf-8") as fo:
+            print(f"{len(property_sets_dict)} property sets parsed")
+            json.dump(property_sets_dict, fo, sort_keys=True, indent=4)
+
+        shutil.rmtree(pset_data_location)
+
 
 def run_doc_api_examples():
     print("Entities (with parent entities attributes included):")
     print(get_entity_doc("IFC2X3", "IfcWindow"))
     print(get_entity_doc("IFC4", "IfcWindow"))
+    print(get_entity_doc("IFC4X3", "IfcWindow"))
 
     print("Entity attributes (with parent entities attributes included):")
     print(get_attribute_doc("IFC2X3", "IfcWindow", "OwnerHistory"))
     print(get_attribute_doc("IFC4", "IfcWindow", "OwnerHistory"))
+    print(get_attribute_doc("IFC4X3", "IfcWindow", "OwnerHistory"))
 
     print("Entity predefined types:")
     print(get_predefined_type_doc("IFC2X3", "IfcControllerType", "FLOATING"))
     print(get_predefined_type_doc("IFC4", "IfcControllerType", "FLOATING"))
+    print(get_predefined_type_doc("IFC4X3", "IfcControllerType", "FLOATING"))
 
     print("Propety sets:")
     print(get_property_set_doc("IFC2X3", "Pset_ZoneCommon"))
     print(get_property_set_doc("IFC4", "Pset_ZoneCommon"))
+    print(get_property_set_doc("IFC4X3", "Pset_ZoneCommon"))
 
     print("Propety sets attributes:")
     print(get_property_doc("IFC2X3", "Pset_ZoneCommon", "Category"))
     print(get_property_doc("IFC4", "Pset_ZoneCommon", "NetPlannedArea"))
+    print(get_property_doc("IFC4X3", "Pset_ZoneCommon", "NetPlannedArea"))
 
     print("Types:")
     print(get_type_doc("IFC2X3", "IfcIsothermalMoistureCapacityMeasure"))
     print(get_type_doc("IFC4", "IfcDuration"))
+    print(get_type_doc("IFC4X3", "IfcDuration"))
 
 
 if __name__ == "__main__":
     extractor = DocExtractor()
     extractor.extract_ifc2x3()
     extractor.extract_ifc4()
+    extractor.extract_ifc4x3()
 
     # run_doc_api_examples()

src/ifcopenshell-python/ifcopenshell/util/ifc4x3dev_scrape_data_for_docs.py

@@ -0,0 +1,203 @@
+# IfcOpenShell - IFC toolkit and geometry engine
+# Copyright (C) 2023 @Andrej730
+#
+# This file is part of IfcOpenShell.
+#
+# IfcOpenShell is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# IfcOpenShell is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with IfcOpenShell.  If not, see <http://www.gnu.org/licenses/>.
+
+try:
+    from server import get_resource_path, resource_documentation_builder, process_markdown, R
+except ModuleNotFoundError as e:
+    print(
+        "ERROR. Failed to import `server.py`.\n"
+        "Make sure you run this script from `/code` folder of https://github.com/buildingSMART/IFC4.3.x-development\n"
+    )
+    raise e
+
+from collections import Counter
+import itertools
+import operator
+from bs4 import BeautifulSoup
+import json
+import ifcopenshell
+
+
+# Hacky modified functions from server.py to make parser work
+def get_definition_from_md(resource, mdc):
+    # Only match up to the first h2
+    lines = []
+    for line in mdc.split("\n"):
+        if line.startswith("## "):
+            break
+        if line.startswith("### "):
+            words = line.split(" ")
+            line = " ".join((words[0], "", *words[1:]))
+        lines.append(line)
+    mdc = "\n".join(lines)
+    mdc_splitted = mdc.split("\n\n")
+    return mdc_splitted[1] if len(mdc_splitted) > 1 else ""
+
+
+def get_type_values(resource, mdc):
+    values = R.type_values.get(resource)
+    if not values:
+        return
+    has_description = values[0] == values[0].upper()
+    if has_description:
+        soup = BeautifulSoup(process_markdown(resource, mdc), features="lxml")
+        described_values = []
+        for value in values:
+            description = None
+            for h in soup.findAll("h3"):
+                if h.text != value:
+                    continue
+                description = BeautifulSoup(features="lxml")
+                for sibling in h.find_next_siblings():
+                    if sibling.name == "h3":
+                        break
+                    description.append(sibling)
+                description = str(description)
+            described_values.append({"name": value, "description": description})
+        values = described_values
+    return {"number": 0, "has_description": has_description, "schema_values": values}
+
+
+def get_attributes_keep_md(resource, builder):
+    attrs = builder.attributes
+    supertype_counts = Counter()
+    supertype_counts.update([a[0] for a in attrs])
+    attrs = [a[1:] for a in attrs]
+    supertype_counts = list(supertype_counts.items())[::-1]
+    insertion_points = [0] + list(itertools.accumulate(map(operator.itemgetter(1), supertype_counts[::-1])))[:-1]
+    group_data = supertype_counts[::-1]
+
+    groups = []
+    for i, attr in enumerate(attrs):
+        if i in insertion_points:
+            name, total_attributes = group_data[insertion_points.index(i)]
+            group = {
+                "name": name,
+                "attributes": [],
+                "is_inherited": insertion_points[-1] != i,
+                "total_attributes": total_attributes,
+            }
+            groups.append(group)
+
+        attribute = {
+            "number": attr[0],
+            "name": attr[1],
+            "type": attr[2][0] if isinstance(attr[2], list) else attr[2],
+            "formal": attr[2][1] if isinstance(attr[2], list) else None,
+            # @nb we're not really talking about markdown anymore
+            # since the new attribute parser operates on a converted
+            # dom, but it appears to work nonetheless.
+            "description": attr[3],
+            "is_inverse": not attr[0],
+        }
+        if attribute["name"] == "PredefinedType" and not attribute["description"]:
+            description = "A list of types to further identify the object. Some property sets may be specifically applicable to one of these types."
+            if "Type" not in group["name"]:
+                description += "\n> NOTE  If the object has an associated IfcTypeObject with a _PredefinedType_, then this attribute shall not be used."
+            attribute["description"] = description
+        group["attributes"].append(attribute)
+
+    total_inherited_attributes = sum([g["total_attributes"] for g in groups if g["is_inherited"]])
+
+    inherited_groups_with_attributes = [g for g in groups if g["is_inherited"] and g["total_attributes"]]
+    if inherited_groups_with_attributes:
+        inherited_groups_with_attributes[-1]["is_last_inherited_group"] = True
+
+    return {
+        "number": None,
+        "groups": groups,
+        "total_inherited_attributes": total_inherited_attributes,
+    }
+
+
+# -------------------------
+
+
+def get_description_json(resource):
+    md = get_resource_path(resource, abort_on_error=False)
+    mdc = open(md, "r", encoding="utf-8").read()
+    description = get_definition_from_md(resource, mdc)
+    return description
+
+
+def get_attributes_json(resource):
+    builder = resource_documentation_builder(resource)
+    attrs = get_attributes_keep_md(resource, builder)
+    if not attrs["groups"]:
+        return []
+    attrs = [a for a in attrs["groups"] if a["name"] == resource]
+    if not attrs:
+        return []
+
+    return attrs[0]["attributes"]
+
+
+def get_predefined_type_values_json(resource):
+    md = get_resource_path(resource, abort_on_error=False)
+    mdc = open(md, "r", encoding="utf-8").read()
+    return get_type_values(resource, mdc)["schema_values"]
+
+
+def save_entities_data(entities):
+    entities_description = dict()
+    for entity in entities:
+        entity_data = dict()
+
+        try:
+            entity_data["description"] = get_description_json(entity)
+        except Exception as e:
+            md = get_resource_path(entity, abort_on_error=False)
+            if md:
+                raise e
+            print(
+                f"WARNING. Cannot find resource path for `{entity}` in DEV DOCUMENTATION even though it's present in ifcopenshell schema. It will be skipped."
+            )
+            continue
+
+        attrs = get_attributes_json(entity)
+        attributes_data = dict()
+        predefined_types_data = dict()
+        for a in attrs:
+            if a["name"] == "PredefinedType":
+                predefined_type_name_enum = a["type"].split()[-1]
+                predef_values = get_predefined_type_values_json(predefined_type_name_enum)
+                for v in predef_values:
+                    description = v["description"]
+                    description = description.strip() if description else ""
+                    if description:
+                        description = BeautifulSoup(description, features="lxml").find("p").text
+                    predefined_types_data[v["name"]] = description
+                continue
+
+            description = a["description"]
+            description = description.strip() if description else ""
+            if description:
+                description = BeautifulSoup(description, features="lxml").find("p").text
+            attributes_data[a["name"]] = description
+        entity_data["attributes"] = attributes_data
+        entity_data["predefined_types"] = predefined_types_data
+        entities_description[entity] = entity_data
+
+    with open("entities_description.json", "w") as fo:
+        json.dump(entities_description, fo, sort_keys=True, indent=4)
+
+
+if __name__ == "__main__":
+    schema = ifcopenshell.ifcopenshell_wrapper.schema_by_name("IFC4X3")
+    entities = [e.name() for e in schema.declarations()]
+    save_entities_data(entities)