diff --git a/src/blenderbim/blenderbim/bim/helper.py b/src/blenderbim/blenderbim/bim/helper.py index f45d1d4c92..ec159200f1 100644 --- a/src/blenderbim/blenderbim/bim/helper.py +++ b/src/blenderbim/blenderbim/bim/helper.py @@ -139,11 +139,12 @@ def prop_with_search(layout, data, prop_name, **kwargs): except KeyError: # TODO : support attributes, pset, etc. pass - op_row = row.row(align=True) url = docs.get("spec_url", "") - url_op = op_row.operator("bim.open_webbrowser", icon="URL", text="") - url_op.url = url - op_row.enabled = bool(url) + if url: + op_row = row.row(align=True) + url_op = op_row.operator("bim.open_webbrowser", icon="INFO", text="") + url_op.url = url + op_row.enabled = bool(url) def get_enum_items(data, prop_name, context): diff --git a/src/ifcopenshell-python/ifcopenshell/util/doc.py b/src/ifcopenshell-python/ifcopenshell/util/doc.py index 2bfa142326..70d6d3a465 100644 --- a/src/ifcopenshell-python/ifcopenshell/util/doc.py +++ b/src/ifcopenshell-python/ifcopenshell/util/doc.py @@ -16,40 +16,42 @@ # You should have received a copy of the GNU Lesser General Public License # along with IfcOpenShell. If not, see . -import glob -from pathlib import Path import json -from pprint import pprint -import urllib.parse -import warnings +from pathlib import Path -from markdown import markdown -from bs4 import BeautifulSoup -from bs4 import MarkupResemblesLocatorWarning -import requests +try: + import glob + import warnings + import requests + import urllib.parse + from markdown import markdown + from bs4 import BeautifulSoup + from bs4 import MarkupResemblesLocatorWarning +except: + pass # Only necessary if you're using it to generate the docs database BASE_MODULE_PATH = Path(__file__).parent -IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / 'Ifc2.3.0.1' -IFC4_DOCS_LOCATION = BASE_MODULE_PATH / 'Ifc4.0.2.1' +IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc2.3.0.1" +IFC4_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc4.0.2.1" SCHEMA_FILES = { - 'IFC2X3': { - 'entities': BASE_MODULE_PATH / 'schema/ifc2x3_entities.json', - 'properties': BASE_MODULE_PATH / 'schema/ifc2x3_properties.json' + "IFC2X3": { + "entities": BASE_MODULE_PATH / "schema/ifc2x3_entities.json", + "properties": BASE_MODULE_PATH / "schema/ifc2x3_properties.json", + }, + "IFC4": { + "entities": BASE_MODULE_PATH / "schema/ifc4_entities.json", + "properties": BASE_MODULE_PATH / "schema/ifc4_properties.json", }, - 'IFC4': { - 'entities': BASE_MODULE_PATH / 'schema/ifc4_entities.json', - 'properties': BASE_MODULE_PATH / 'schema/ifc4_properties.json' - } } # singleton doc database # required so that database would be loaded only once -class DocDatabase(): +class DocDatabase: def __new__(cls): - if not hasattr(cls, 'db'): + if not hasattr(cls, "db"): db = {ifc_version: dict() for ifc_version in SCHEMA_FILES} files_missing = False for ifc_version in SCHEMA_FILES: @@ -59,15 +61,15 @@ class DocDatabase(): print(f"Schema file {schema_path} wasn't found.") files_missing = True continue - - with open(schema_path, 'r') as fi: + + with open(schema_path, "r") as fi: db[ifc_version][data_type] = json.load(fi) if files_missing: raise Exception( - 'Some schema files are missing - they contain neccessary data for DocAPI to work. \n' - 'Make sure those files are present. To generate them you can run DocExtractor extract functions \n' - 'but it will require corresponding Ifc docs to be in the same directory as the script.' + "Some schema files are missing - they contain neccessary data for DocAPI to work. \n" + "Make sure those files are present. To generate them you can run DocExtractor extract functions \n" + "but it will require corresponding Ifc docs to be in the same directory as the script." ) cls.db = db return cls.db @@ -76,37 +78,41 @@ class DocDatabase(): def check_version_name(version_name): if version_name not in SCHEMA_FILES: raise Exception( - f'Version: {version_name} is not supported. ' - f'Supported version: {", ".join(SCHEMA_FILES.keys())}') + f"Version: {version_name} is not supported. " f'Supported version: {", ".join(SCHEMA_FILES.keys())}' + ) return version_name + def get_entity_doc(version, entity): version = check_version_name(version) - return DocDatabase()[version]['entities'][entity] + return DocDatabase()[version]["entities"][entity] + def get_attribute_doc(version, entity, attribute): version = check_version_name(version) - return DocDatabase()[version]['entities'][entity]['attributes'][attribute] + return DocDatabase()[version]["entities"][entity]["attributes"][attribute] + def get_property_set_doc(version, pset): version = check_version_name(version) - return DocDatabase()[version]['properties'][pset] + return DocDatabase()[version]["properties"][pset] + def get_property_doc(version, pset, prop): version = check_version_name(version) - return DocDatabase()[version]['properties'][pset]['properties'][prop] + return DocDatabase()[version]["properties"][pset]["properties"][prop] class DocExtractor: def extract_ifc2x3(self): - print('Parsing data for Ifc2.3.0.1') + print("Parsing data for Ifc2.3.0.1") if not IFC2x3_DOCS_LOCATION.is_dir(): raise Exception( f'Docs for IFC 2.3.0.1 expected to be in folder "{IFC2x3_DOCS_LOCATION.resolve()}\\"\n' - 'For doc extraction please either setup docs as described above \n' - 'or change IFC2x3_DOCS_LOCATION in doc.py accordingly. \n' - 'You can download docs from the repository: \n' - 'https://github.com/buildingSMART/IFC/tree/Ifc2.3.0.1' + "For doc extraction please either setup docs as described above \n" + "or change IFC2x3_DOCS_LOCATION in doc.py accordingly. \n" + "You can download docs from the repository: \n" + "https://github.com/buildingSMART/IFC/tree/Ifc2.3.0.1" ) # need to parse actual domains from the website @@ -117,95 +123,90 @@ class DocExtractor: self.extract_ifc2x3_property_sets_site_domains() self.extract_ifc2x3_entities() self.extract_ifc2x3_property_sets() - + def extract_ifc2x3_property_sets_site_domains(self): property_sets_domains = dict() - r = requests.get('https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/psd/psd_index.htm') - html = BeautifulSoup(r.content, features='lxml') - for a in html.find_all('a'): - domain, pset = a['href'].removeprefix('./').removesuffix('.xml').split('/') + r = requests.get("https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/psd/psd_index.htm") + html = BeautifulSoup(r.content, features="lxml") + for a in html.find_all("a"): + domain, pset = a["href"].removeprefix("./").removesuffix(".xml").split("/") property_sets_domains[pset] = domain # export property sets data - with open( - BASE_MODULE_PATH / 'schema/ifc2x3_property_sets_site_domains.json', - 'w', encoding='utf-8') as fo: - print(f'{len(property_sets_domains)} property sets domains were parsed from the website') - json.dump( - property_sets_domains, fo, - sort_keys=True, indent=4 - ) + with open(BASE_MODULE_PATH / "schema/ifc2x3_property_sets_site_domains.json", "w", encoding="utf-8") as fo: + print(f"{len(property_sets_domains)} property sets domains were parsed from the website") + json.dump(property_sets_domains, fo, sort_keys=True, indent=4) def extract_ifc2x3_entities(self): entities_dict = dict() - # search - entities_paths = [filepath - for filepath in glob.iglob(f'{IFC2x3_DOCS_LOCATION}/Sections/**/Entities', recursive=True)] + # search + entities_paths = [ + filepath for filepath in glob.iglob(f"{IFC2x3_DOCS_LOCATION}/Sections/**/Entities", recursive=True) + ] for parse_folder_path in entities_paths: - for entity_path in glob.iglob(f'{parse_folder_path}/**/'): + for entity_path in glob.iglob(f"{parse_folder_path}/**/"): entity_path = Path(entity_path) entity_name = entity_path.stem entities_dict[entity_name] = dict() # utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded - md_path = entity_path / 'Documentation.md' - xml_path = entity_path / 'DocEntity.xml' + md_path = entity_path / "Documentation.md" + xml_path = entity_path / "DocEntity.xml" md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) - github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' + github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}" - with open(md_path, 'r', encoding='utf-8-sig') as fi: + with open(md_path, "r", encoding="utf-8-sig") as fi: # convert markdown to html for easier parsing html = markdown(fi.read()) - entity_description = BeautifulSoup(html, features="lxml").find('p').text - entity_description = entity_description.replace('\n', ' ') - entity_description = entity_description.replace('\u00a0', ' ') + entity_description = BeautifulSoup(html, features="lxml").find("p").text + entity_description = entity_description.replace("\n", " ") + entity_description = entity_description.replace("\u00a0", " ") - with open(xml_path, 'r', encoding='utf-8') as fi: - bs_tree = BeautifulSoup(fi.read(), features='lxml') + with open(xml_path, "r", encoding="utf-8") as fi: + bs_tree = BeautifulSoup(fi.read(), features="lxml") entity_attrs = dict() # temporarily disable MarkupResemblesLocatorWarning - # because BeautifulSoup wrongly assume we confused + # because BeautifulSoup wrongly assume we confused # html code for filepath and gives warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=MarkupResemblesLocatorWarning) + warnings.simplefilter("ignore", category=MarkupResemblesLocatorWarning) - for html_attr in bs_tree.find_all('docattribute'): - html_description = BeautifulSoup(html_attr.text, features='lxml') + for html_attr in bs_tree.find_all("docattribute"): + html_description = BeautifulSoup(html_attr.text, features="lxml") attr_description = html_description.get_text() - attr_description = attr_description.replace('\n', ' ') - attr_description = attr_description.replace('\u00a0', ' ') - attr_description = attr_description.replace('&npsp;', ' ') + attr_description = attr_description.replace("\n", " ") + attr_description = attr_description.replace("\u00a0", " ") + attr_description = attr_description.replace("&npsp;", " ") # discard part of the description with changelog # Example: # https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/ifcpresentationdefinitionresource/lexical/ifcannotationfillarea.htm - attr_description = attr_description.split('IFC2x Edition 3 CHANGE', 1)[0] - attr_description = attr_description.split('IFC2x Edition 2 Addendum 2 CHANGE', 1)[0] - attr_description = attr_description.split('IFC2x2 Addendum 1 change', 1)[0] - attr_description = attr_description.split('IFC2x PLATFORM CHANGE', 1)[0] - attr_description = attr_description.split('IFC2x3 CHANGE', 1)[0] - attr_description = attr_description.split('IFC2x Edition3 CHANGE', 1)[0] + attr_description = attr_description.split("IFC2x Edition 3 CHANGE", 1)[0] + attr_description = attr_description.split("IFC2x Edition 2 Addendum 2 CHANGE", 1)[0] + attr_description = attr_description.split("IFC2x2 Addendum 1 change", 1)[0] + attr_description = attr_description.split("IFC2x PLATFORM CHANGE", 1)[0] + attr_description = attr_description.split("IFC2x3 CHANGE", 1)[0] + attr_description = attr_description.split("IFC2x Edition3 CHANGE", 1)[0] - attr_description = attr_description.strip().rstrip('>').strip() - entity_attrs[html_attr['name']] = attr_description + attr_description = attr_description.strip().rstrip(">").strip() + entity_attrs[html_attr["name"]] = attr_description if entity_attrs: - entities_dict[entity_name]['attributes'] = entity_attrs + entities_dict[entity_name]["attributes"] = entity_attrs - entities_dict[entity_name]['description'] = entity_description - spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/' \ - f'{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm' - entities_dict[entity_name]['spec_url'] = spec_url + entities_dict[entity_name]["description"] = entity_description + spec_url = ( + "https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/" + f"{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm" + ) + entities_dict[entity_name]["spec_url"] = spec_url # export entities data - with open(BASE_MODULE_PATH / 'schema/ifc2x3_entities.json', 'w', encoding='utf-8') as fo: - print(f'{len(entities_dict)} entities parsed') - json.dump( - entities_dict, fo, - sort_keys=True, indent=4 - ) + with open(BASE_MODULE_PATH / "schema/ifc2x3_entities.json", "w", encoding="utf-8") as fo: + print(f"{len(entities_dict)} entities parsed") + json.dump(entities_dict, fo, sort_keys=True, indent=4) def extract_ifc2x3_property_sets(self): property_sets_dict = dict() @@ -213,39 +214,42 @@ class DocExtractor: property_sets_spec_urls = dict() # extract lists of properties and theirs references for each property set - parsed_paths = [filepath - for filepath in glob.iglob(f'{IFC2x3_DOCS_LOCATION}/Sections/**/PropertySets', recursive=True)] + parsed_paths = [ + filepath for filepath in glob.iglob(f"{IFC2x3_DOCS_LOCATION}/Sections/**/PropertySets", recursive=True) + ] # prepare property sets domains from the website we extracted earlier - with open(BASE_MODULE_PATH / 'schema/ifc2x3_property_sets_site_domains.json', 'r') as fi: + with open(BASE_MODULE_PATH / "schema/ifc2x3_property_sets_site_domains.json", "r") as fi: property_sets_site_domains = json.load(fi) for parse_folder_path in parsed_paths: - for property_set_path in glob.iglob(f'{parse_folder_path}/**/'): + for property_set_path in glob.iglob(f"{parse_folder_path}/**/"): property_set_path = Path(property_set_path) property_set_name = property_set_path.stem property_references = list() - xml_path = property_set_path / 'DocPropertySet.xml' - with open(xml_path, 'r', encoding='utf-8') as fi: - bs_tree = BeautifulSoup(fi.read(), features='lxml') - for html_attr in bs_tree.find_all('docproperty'): - property_references.append(html_attr['href']) + xml_path = property_set_path / "DocPropertySet.xml" + with open(xml_path, "r", encoding="utf-8") as fi: + bs_tree = BeautifulSoup(fi.read(), features="lxml") + for html_attr in bs_tree.find_all("docproperty"): + property_references.append(html_attr["href"]) property_sets_references[property_set_name] = property_references property_set_domain = property_sets_site_domains[property_set_name] - spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML' \ - f'/psd/{property_set_domain}/{property_set_name}.xml' + spec_url = ( + "https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML" + f"/psd/{property_set_domain}/{property_set_name}.xml" + ) property_sets_spec_urls[property_set_name] = spec_url # setup references look up tables to convert property hrefs to actual data paths references_paths_lookup = dict() - glob_query = f'{IFC2x3_DOCS_LOCATION}/Properties/*/*' + glob_query = f"{IFC2x3_DOCS_LOCATION}/Properties/*/*" for parsed_path in [filepath for filepath in glob.iglob(glob_query, recursive=False)]: parsed_path = Path(parsed_path) # all references omit "$" character, I've checked it on 2_3 # need to check it if moving to next IFC version - property_reference = parsed_path.name.replace('$', '') + property_reference = parsed_path.name.replace("$", "") references_paths_lookup[property_reference] = parsed_path # setup a function because we'll need to check child properties recusively @@ -253,51 +257,53 @@ class DocExtractor: property_dict = dict() property_path = references_paths_lookup[href] - md_path = property_path / 'Documentation.md' - xml_path = property_path / 'DocProperty.xml' + md_path = property_path / "Documentation.md" + xml_path = property_path / "DocProperty.xml" md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix())) - github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' - github_xml_url = f'https://github.com/buildingSMART/IFC/blob/{xml_url_part}' + github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}" + github_xml_url = f"https://github.com/buildingSMART/IFC/blob/{xml_url_part}" - with open(xml_path, 'r', encoding='utf-8') as fi: - bs_tree = BeautifulSoup(fi.read(), features='lxml') - tags = bs_tree.find_all('docproperty') + with open(xml_path, "r", encoding="utf-8") as fi: + bs_tree = BeautifulSoup(fi.read(), features="lxml") + tags = bs_tree.find_all("docproperty") # check for child properties - if they are present parse their data recursively - elements_tag = bs_tree.find('elements') + elements_tag = bs_tree.find("elements") if elements_tag is not None: - child_tags = elements_tag.find_all('docproperty') + child_tags = elements_tag.find_all("docproperty") child_tags_dict = dict() for child_tag in child_tags: - child_tag_href = child_tag['href'] + child_tag_href = child_tag["href"] child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href) child_tags_dict[child_tag_name] = child_tag_dict tags.remove(child_tag) - property_dict['children'] = child_tags_dict - print(f'Child nodes found inside property xml. Url: {github_xml_url}') + property_dict["children"] = child_tags_dict + print(f"Child nodes found inside property xml. Url: {github_xml_url}") if len(tags) != 1: - print(f'WARNING. Found more properties inside property xml, ' - f'only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}.') - property_name = tags[0]['name'] - + print( + f"WARNING. Found more properties inside property xml, " + f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}." + ) + property_name = tags[0]["name"] if not md_path.is_file(): - print(f'WARNING. Property {property_name} is missing documentation.md, ' - f'property will be left without description. Url: {github_xml_url}') + print( + f"WARNING. Property {property_name} is missing documentation.md, " + f"property will be left without description. Url: {github_xml_url}" + ) else: - with open(md_path, 'r', encoding='utf-8-sig') as fi: + with open(md_path, "r", encoding="utf-8-sig") as fi: # convert markdown to html for easier parsing html = markdown(fi.read()) - description = BeautifulSoup(html, features="lxml").find('p').text - description = description.replace('\n', ' ') - description = description.replace('\u00a0', ' ') - property_dict['description'] = description + description = BeautifulSoup(html, features="lxml").find("p").text + description = description.replace("\n", " ") + description = description.replace("\u00a0", " ") + property_dict["description"] = description return (property_name, property_dict) - # lookup each property reference and save it's name and description for property_set_name in property_sets_references: properties_dict = dict() @@ -305,28 +311,24 @@ class DocExtractor: property_name, property_dict = get_property_info_by_href(property_reference) properties_dict[property_name] = property_dict property_sets_dict[property_set_name] = { - 'properties': properties_dict, - 'spec_url': property_sets_spec_urls[property_set_name] + "properties": properties_dict, + "spec_url": property_sets_spec_urls[property_set_name], } - # export property sets data - with open(BASE_MODULE_PATH / 'schema/ifc2x3_properties.json', 'w', encoding='utf-8') as fo: - print(f'{len(property_sets_dict)} property sets parsed') - json.dump( - property_sets_dict, fo, - sort_keys=True, indent=4 - ) + with open(BASE_MODULE_PATH / "schema/ifc2x3_properties.json", "w", encoding="utf-8") as fo: + print(f"{len(property_sets_dict)} property sets parsed") + json.dump(property_sets_dict, fo, sort_keys=True, indent=4) def extract_ifc4(self): - print('Parsing data for Ifc4.0.2.1') + print("Parsing data for Ifc4.0.2.1") if not IFC4_DOCS_LOCATION.is_dir(): raise Exception( f'Docs for Ifc4.0.2.1 expected to be in folder "{IFC4_DOCS_LOCATION.resolve()}\\"\n' - 'For doc extraction please either setup docs as described above \n' - 'or change IFC4_DOCS_LOCATION in doc.py accordingly.' - 'You can download docs from the repository: \n' - 'https://github.com/buildingSMART/IFC/tree/Ifc4.0.2.1' + "For doc extraction please either setup docs as described above \n" + "or change IFC4_DOCS_LOCATION in doc.py accordingly." + "You can download docs from the repository: \n" + "https://github.com/buildingSMART/IFC/tree/Ifc4.0.2.1" ) # actually domains in Ifc 4.0 are consistent between website and docs @@ -337,107 +339,106 @@ class DocExtractor: self.extract_ifc4_property_sets_site_domains() self.extract_ifc4_entities() self.extract_ifc4_property_sets() - + def extract_ifc4_property_sets_site_domains(self): property_sets_domains = dict() with requests.get( - 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1' - '/HTML/annex/annex-b/alphabeticalorder_psets.htm') as r: - html = BeautifulSoup(r.content, features='lxml') - for a in html.find_all('a', {'class': 'listing-link'}): - href_split = a['href'].split('/') + "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1" + "/HTML/annex/annex-b/alphabeticalorder_psets.htm" + ) as r: + html = BeautifulSoup(r.content, features="lxml") + for a in html.find_all("a", {"class": "listing-link"}): + href_split = a["href"].split("/") domain = href_split[3] - pset = href_split[5].removesuffix('.htm') + pset = href_split[5].removesuffix(".htm") property_sets_domains[pset] = domain with requests.get( - 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/' - '/HTML/annex/annex-b/alphabeticalorder_qsets.htm') as r: - html = BeautifulSoup(r.content, features='lxml') - for a in html.find_all('a', {'class': 'listing-link'}): - href_split = a['href'].split('/') + "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/" + "/HTML/annex/annex-b/alphabeticalorder_qsets.htm" + ) as r: + html = BeautifulSoup(r.content, features="lxml") + for a in html.find_all("a", {"class": "listing-link"}): + href_split = a["href"].split("/") domain = href_split[3] - pset = href_split[5].removesuffix('.htm') + pset = href_split[5].removesuffix(".htm") property_sets_domains[pset] = domain # export property sets data - with open(BASE_MODULE_PATH / 'schema/ifc4_property_sets_site_domains.json', 'w', encoding='utf-8') as fo: - print(f'{len(property_sets_domains)} property sets domains were parsed from the website') - json.dump( - property_sets_domains, fo, - sort_keys=True, indent=4 - ) + with open(BASE_MODULE_PATH / "schema/ifc4_property_sets_site_domains.json", "w", encoding="utf-8") as fo: + print(f"{len(property_sets_domains)} property sets domains were parsed from the website") + json.dump(property_sets_domains, fo, sort_keys=True, indent=4) def extract_ifc4_entities(self): entities_dict = dict() - # search - entities_paths = [filepath - for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/Entities', recursive=True)] + # search + entities_paths = [ + filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/Entities", recursive=True) + ] for parse_folder_path in entities_paths: - for entity_path in glob.iglob(f'{parse_folder_path}/**/'): + for entity_path in glob.iglob(f"{parse_folder_path}/**/"): entity_path = Path(entity_path) entity_name = entity_path.stem entities_dict[entity_name] = dict() # utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded - md_path = entity_path / 'Documentation.md' - xml_path = entity_path / 'DocEntity.xml' + md_path = entity_path / "Documentation.md" + xml_path = entity_path / "DocEntity.xml" md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) - github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' + github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}" - with open(md_path, 'r', encoding='utf-8-sig') as fi: + with open(md_path, "r", encoding="utf-8-sig") as fi: # convert markdown to html for easier parsing html = markdown(fi.read()) - entity_description = BeautifulSoup(html, features="lxml").find('p').text - entity_description = entity_description.replace('\n', ' ') - entity_description = entity_description.replace('\u00a0', ' ') - entity_description = entity_description.replace('{ .extDef}', '') + entity_description = BeautifulSoup(html, features="lxml").find("p").text + entity_description = entity_description.replace("\n", " ") + entity_description = entity_description.replace("\u00a0", " ") + entity_description = entity_description.replace("{ .extDef}", "") entity_description = entity_description.strip() - with open(xml_path, 'r', encoding='utf-8') as fi: - bs_tree = BeautifulSoup(fi.read(), features='lxml') + with open(xml_path, "r", encoding="utf-8") as fi: + bs_tree = BeautifulSoup(fi.read(), features="lxml") entity_attrs = dict() # temporarily disable MarkupResemblesLocatorWarning - # because BeautifulSoup wrongly assume we confused + # because BeautifulSoup wrongly assume we confused # html code for filepath and gives warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=MarkupResemblesLocatorWarning) - for html_attr in bs_tree.find_all('docattribute'): - html_description = BeautifulSoup(html_attr.text, features='lxml') + warnings.simplefilter("ignore", category=MarkupResemblesLocatorWarning) + for html_attr in bs_tree.find_all("docattribute"): + html_description = BeautifulSoup(html_attr.text, features="lxml") attr_description = html_description.get_text() - attr_description = attr_description.replace('\n', ' ') - attr_description = attr_description.replace('\u00a0', ' ') + attr_description = attr_description.replace("\n", " ") + attr_description = attr_description.replace("\u00a0", " ") # discard part of the description with changelog, notes and examples etc. # Those notes actually can be useful but we'll need a way to reformat them # Example: # https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/ifcsharedbldgelements/lexical/ifcrelconnectspathelements.htm - attr_description = attr_description.split('{ .change-ifc', 1)[0] - attr_description = attr_description.split('{ .note', 1)[0] - attr_description = attr_description.split('{ .examples', 1)[0] - attr_description = attr_description.split('{ .deprecated', 1)[0] - attr_description = attr_description.split('{ .history', 1)[0] - + attr_description = attr_description.split("{ .change-ifc", 1)[0] + attr_description = attr_description.split("{ .note", 1)[0] + attr_description = attr_description.split("{ .examples", 1)[0] + attr_description = attr_description.split("{ .deprecated", 1)[0] + attr_description = attr_description.split("{ .history", 1)[0] + attr_description = attr_description.strip() - entity_attrs[html_attr['name']] = attr_description + entity_attrs[html_attr["name"]] = attr_description if entity_attrs: - entities_dict[entity_name]['attributes'] = entity_attrs + entities_dict[entity_name]["attributes"] = entity_attrs - entities_dict[entity_name]['description'] = entity_description - spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/' \ - f'{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm' - entities_dict[entity_name]['spec_url'] = spec_url + entities_dict[entity_name]["description"] = entity_description + spec_url = ( + "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/" + f"{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm" + ) + entities_dict[entity_name]["spec_url"] = spec_url # entities_dict[entity_name]['github_url'] = github_md_url # export entities data - with open(BASE_MODULE_PATH / 'schema/ifc4_entities.json', 'w', encoding='utf-8') as fo: - print(f'{len(entities_dict)} entities parsed') - json.dump( - entities_dict, fo, - sort_keys=True, indent=4 - ) + with open(BASE_MODULE_PATH / "schema/ifc4_entities.json", "w", encoding="utf-8") as fo: + print(f"{len(entities_dict)} entities parsed") + json.dump(entities_dict, fo, sort_keys=True, indent=4) def extract_ifc4_property_sets(self): # function parses both property and quantity sets @@ -446,105 +447,114 @@ class DocExtractor: property_sets_spec_urls = dict() # extract lists of properties and theirs references for each property set - parsed_paths = [filepath for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/PropertySets', recursive=True)] - parsed_paths += [filepath for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/QuantitySets', recursive=True)] + parsed_paths = [ + filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/PropertySets", recursive=True) + ] + parsed_paths += [ + filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/QuantitySets", recursive=True) + ] # prepare property sets domains from the website we extracted earlier - with open(BASE_MODULE_PATH / 'schema/ifc4_property_sets_site_domains.json', 'r') as fi: + with open(BASE_MODULE_PATH / "schema/ifc4_property_sets_site_domains.json", "r") as fi: property_sets_site_domains = json.load(fi) psets_test = set() for parse_folder_path in parsed_paths: - for property_set_path in glob.iglob(f'{parse_folder_path}/**/'): + for property_set_path in glob.iglob(f"{parse_folder_path}/**/"): property_set_path = Path(property_set_path) property_set_name = property_set_path.stem property_references = list() - property_quantity = property_set_path.parents[0].name == 'QuantitySets' - xml_path = property_set_path / ('DocQuantitySet.xml' if property_quantity else 'DocPropertySet.xml') + property_quantity = property_set_path.parents[0].name == "QuantitySets" + xml_path = property_set_path / ("DocQuantitySet.xml" if property_quantity else "DocPropertySet.xml") - with open(xml_path, 'r', encoding='utf-8') as fi: - bs_tree = BeautifulSoup(fi.read(), features='lxml') - for html_attr in bs_tree.find_all('docquantity' if property_quantity else 'docproperty'): - property_references.append(html_attr['href']) + with open(xml_path, "r", encoding="utf-8") as fi: + bs_tree = BeautifulSoup(fi.read(), features="lxml") + for html_attr in bs_tree.find_all("docquantity" if property_quantity else "docproperty"): + property_references.append(html_attr["href"]) property_sets_references[property_set_name] = property_references - + if property_set_name.lower() not in property_sets_site_domains: - print(f"WARNING. {property_set_name} was not found on the spec website, " - "this property set won't have any spec_url in schema.") + print( + f"WARNING. {property_set_name} was not found on the spec website, " + "this property set won't have any spec_url in schema." + ) else: - property_set_domain = property_sets_site_domains.get(property_set_name.lower(), '') - spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML' \ - f'/schema/{property_set_domain}' \ - f'/{"qset" if property_quantity else "pset"}' \ - f'/{property_set_name.lower()}.htm' + property_set_domain = property_sets_site_domains.get(property_set_name.lower(), "") + spec_url = ( + "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML" + f"/schema/{property_set_domain}" + f'/{"qset" if property_quantity else "pset"}' + f"/{property_set_name.lower()}.htm" + ) property_sets_spec_urls[property_set_name] = spec_url - # setup references look up tables to convert property hrefs to actual data paths references_paths_lookup = dict() - parsed_paths = [filepath - for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Properties/*/*', recursive=False)] - parsed_paths += [filepath - for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Quantities/*/*', recursive=False)] + parsed_paths = [filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Properties/*/*", recursive=False)] + parsed_paths += [filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Quantities/*/*", recursive=False)] for parsed_path in parsed_paths: parsed_path = Path(parsed_path) # all references omit "$" character, I've checked it on 4_0 # need to check it if moving to next IFC version # btw no reason to check if all references were used in properties # because there are also child properties - property_reference = parsed_path.name.replace('$', '') + property_reference = parsed_path.name.replace("$", "") references_paths_lookup[property_reference] = parsed_path - + # setup a function because we'll need to check child properties recusively def get_property_info_by_href(href): property_dict = dict() property_path = references_paths_lookup[href] - property_quantity = property_path.parents[1].name == 'Quantities' + property_quantity = property_path.parents[1].name == "Quantities" - md_path = property_path / 'Documentation.md' - xml_path = property_path / ('DocQuantity.xml' if property_quantity else 'DocProperty.xml') + md_path = property_path / "Documentation.md" + xml_path = property_path / ("DocQuantity.xml" if property_quantity else "DocProperty.xml") md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) - github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' + github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}" xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix())) - github_xml_url = f'https://github.com/buildingSMART/IFC/blob/{xml_url_part}' + github_xml_url = f"https://github.com/buildingSMART/IFC/blob/{xml_url_part}" - with open(xml_path, 'r', encoding='utf-8') as fi: - bs_tree = BeautifulSoup(fi.read(), features='lxml') - tags = bs_tree.find_all('docquantity' if property_quantity else 'docproperty') + with open(xml_path, "r", encoding="utf-8") as fi: + bs_tree = BeautifulSoup(fi.read(), features="lxml") + tags = bs_tree.find_all("docquantity" if property_quantity else "docproperty") # check for child properties - if they are present parse their data recursively - elements_tag = bs_tree.find('elements') + elements_tag = bs_tree.find("elements") if elements_tag is not None: - child_tags = elements_tag.find_all('docquantity' if property_quantity else 'docproperty') + child_tags = elements_tag.find_all("docquantity" if property_quantity else "docproperty") child_tags_dict = dict() for child_tag in child_tags: - child_tag_href = child_tag['href'] + child_tag_href = child_tag["href"] child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href) child_tags_dict[child_tag_name] = child_tag_dict tags.remove(child_tag) - property_dict['children'] = child_tags_dict - print(f'Child nodes found inside property xml. Url: {github_xml_url}') + property_dict["children"] = child_tags_dict + print(f"Child nodes found inside property xml. Url: {github_xml_url}") if len(tags) != 1: - print(f'WARNING. Found more properties inside property xml, ' - f'only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}.') - property_name = tags[0]['name'] + print( + f"WARNING. Found more properties inside property xml, " + f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}." + ) + property_name = tags[0]["name"] if not md_path.is_file(): - print(f'WARNING. Property {property_name} is missing documentation.md, property will be left without description. ' - f'Url: {github_xml_url}') + print( + f"WARNING. Property {property_name} is missing documentation.md, property will be left without description. " + f"Url: {github_xml_url}" + ) else: - with open(md_path, 'r', encoding='utf-8-sig') as fi: + with open(md_path, "r", encoding="utf-8-sig") as fi: # convert markdown to html for easier parsing html = markdown(fi.read()) - description = BeautifulSoup(html, features="lxml").find('p').text - description = description.replace('\n', ' ') - description = description.replace('\u00a0', ' ') - property_dict['description'] = description + description = BeautifulSoup(html, features="lxml").find("p").text + description = description.replace("\n", " ") + description = description.replace("\u00a0", " ") + property_dict["description"] = description return (property_name, property_dict) # lookup each property reference and save it's name and description @@ -553,47 +563,38 @@ class DocExtractor: for property_reference in property_sets_references[property_set_name]: property_name, property_dict = get_property_info_by_href(property_reference) properties_dict[property_name] = property_dict - property_sets_dict[property_set_name] = { - 'properties': properties_dict - } + property_sets_dict[property_set_name] = {"properties": properties_dict} if property_set_name in property_sets_spec_urls: spec_url = property_sets_spec_urls[property_set_name] - property_sets_dict[property_set_name]['spec_url'] = spec_url + property_sets_dict[property_set_name]["spec_url"] = spec_url # export property sets data - with open(BASE_MODULE_PATH / 'schema/ifc4_properties.json', 'w', encoding='utf-8') as fo: - print(f'{len(property_sets_dict)} property sets parsed') - json.dump( - property_sets_dict, fo, - sort_keys=True, indent=4 - ) + with open(BASE_MODULE_PATH / "schema/ifc4_properties.json", "w", encoding="utf-8") as fo: + print(f"{len(property_sets_dict)} property sets parsed") + json.dump(property_sets_dict, fo, sort_keys=True, indent=4) + def run_doc_api_examples(): - print('Entities:') - print(get_entity_doc('IFC2X3', 'IfcActionRequest')) - print(get_entity_doc('IFC4', 'IfcActionRequest')) + print("Entities:") + print(get_entity_doc("IFC2X3", "IfcActionRequest")) + print(get_entity_doc("IFC4", "IfcActionRequest")) - print('Entity attributes:') - print(get_attribute_doc('IFC2X3', 'IfcActionRequest', 'RequestID')) - print(get_attribute_doc('IFC4', 'IfcActionRequest', 'PredefinedType')) + print("Entity attributes:") + print(get_attribute_doc("IFC2X3", "IfcActionRequest", "RequestID")) + print(get_attribute_doc("IFC4", "IfcActionRequest", "PredefinedType")) - print('Propety sets:') - print(get_property_set_doc('IFC2X3', 'Pset_ZoneCommon')) - print(get_property_set_doc('IFC4', 'Pset_ZoneCommon')) + print("Propety sets:") + print(get_property_set_doc("IFC2X3", "Pset_ZoneCommon")) + print(get_property_set_doc("IFC4", "Pset_ZoneCommon")) - print('Propety sets attributes:') - print(get_property_doc('IFC2X3', 'Pset_ZoneCommon', 'Category')) - print(get_property_doc('IFC4', 'Pset_ZoneCommon', 'NetPlannedArea')) + print("Propety sets attributes:") + print(get_property_doc("IFC2X3", "Pset_ZoneCommon", "Category")) + print(get_property_doc("IFC4", "Pset_ZoneCommon", "NetPlannedArea")) -if __name__ == '__main__': +if __name__ == "__main__": extractor = DocExtractor() extractor.extract_ifc2x3() extractor.extract_ifc4() # run_doc_api_examples() - - - - -