Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-18 22:33:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run black and minor fix. See #1990.

Browse Source
This commit is contained in:
Dion Moult
2022-10-19 09:17:37 +11:00
parent ec1fbabd00
commit 159e2ca779
2 changed files with 278 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blenderbim/blenderbim/bim/helper.py
+5 -4
View File
@@ -139,11 +139,12 @@ def prop_with_search(layout, data, prop_name, **kwargs):
except KeyError: except KeyError:
# TODO : support attributes, pset, etc. # TODO : support attributes, pset, etc.
pass pass
op_row = row.row(align=True)
url = docs.get("spec_url", "") url = docs.get("spec_url", "")
url_op = op_row.operator("bim.open_webbrowser", icon="URL", text="") if url:
url_op.url = url op_row = row.row(align=True)
op_row.enabled = bool(url) url_op = op_row.operator("bim.open_webbrowser", icon="INFO", text="")
url_op.url = url
op_row.enabled = bool(url)
def get_enum_items(data, prop_name, context): def get_enum_items(data, prop_name, context):
src/ifcopenshell-python/ifcopenshell/util/doc.py
+273 -272
View File
@@ -16,40 +16,42 @@
# You should have received a copy of the GNU Lesser General Public License # You should have received a copy of the GNU Lesser General Public License
# along with IfcOpenShell. If not, see <http://www.gnu.org/licenses/>. # along with IfcOpenShell. If not, see <http://www.gnu.org/licenses/>.
import glob
from pathlib import Path
import json import json
from pprint import pprint from pathlib import Path
import urllib.parse
import warnings
from markdown import markdown try:
from bs4 import BeautifulSoup import glob
from bs4 import MarkupResemblesLocatorWarning import warnings
import requests import requests
import urllib.parse
from markdown import markdown
from bs4 import BeautifulSoup
from bs4 import MarkupResemblesLocatorWarning
except:
pass # Only necessary if you're using it to generate the docs database
BASE_MODULE_PATH = Path(__file__).parent BASE_MODULE_PATH = Path(__file__).parent
IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / 'Ifc2.3.0.1' IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc2.3.0.1"
IFC4_DOCS_LOCATION = BASE_MODULE_PATH / 'Ifc4.0.2.1' IFC4_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc4.0.2.1"
SCHEMA_FILES = { SCHEMA_FILES = {
'IFC2X3': { "IFC2X3": {
'entities': BASE_MODULE_PATH / 'schema/ifc2x3_entities.json', "entities": BASE_MODULE_PATH / "schema/ifc2x3_entities.json",
'properties': BASE_MODULE_PATH / 'schema/ifc2x3_properties.json' "properties": BASE_MODULE_PATH / "schema/ifc2x3_properties.json",
},
"IFC4": {
"entities": BASE_MODULE_PATH / "schema/ifc4_entities.json",
"properties": BASE_MODULE_PATH / "schema/ifc4_properties.json",
}, },
'IFC4': {
'entities': BASE_MODULE_PATH / 'schema/ifc4_entities.json',
'properties': BASE_MODULE_PATH / 'schema/ifc4_properties.json'
}
} }
# singleton doc database # singleton doc database
# required so that database would be loaded only once # required so that database would be loaded only once
class DocDatabase(): class DocDatabase:
def __new__(cls): def __new__(cls):
if not hasattr(cls, 'db'): if not hasattr(cls, "db"):
db = {ifc_version: dict() for ifc_version in SCHEMA_FILES} db = {ifc_version: dict() for ifc_version in SCHEMA_FILES}
files_missing = False files_missing = False
for ifc_version in SCHEMA_FILES: for ifc_version in SCHEMA_FILES:
@@ -59,15 +61,15 @@ class DocDatabase():
print(f"Schema file {schema_path} wasn't found.") print(f"Schema file {schema_path} wasn't found.")
files_missing = True files_missing = True
continue continue
with open(schema_path, 'r') as fi: with open(schema_path, "r") as fi:
db[ifc_version][data_type] = json.load(fi) db[ifc_version][data_type] = json.load(fi)
if files_missing: if files_missing:
raise Exception( raise Exception(
'Some schema files are missing - they contain neccessary data for DocAPI to work. \n' "Some schema files are missing - they contain neccessary data for DocAPI to work. \n"
'Make sure those files are present. To generate them you can run DocExtractor extract functions \n' "Make sure those files are present. To generate them you can run DocExtractor extract functions \n"
'but it will require corresponding Ifc docs to be in the same directory as the script.' "but it will require corresponding Ifc docs to be in the same directory as the script."
) )
cls.db = db cls.db = db
return cls.db return cls.db
@@ -76,37 +78,41 @@ class DocDatabase():
def check_version_name(version_name): def check_version_name(version_name):
if version_name not in SCHEMA_FILES: if version_name not in SCHEMA_FILES:
raise Exception( raise Exception(
f'Version: {version_name} is not supported. ' f"Version: {version_name} is not supported. " f'Supported version: {", ".join(SCHEMA_FILES.keys())}'
f'Supported version: {", ".join(SCHEMA_FILES.keys())}') )
return version_name return version_name
def get_entity_doc(version, entity): def get_entity_doc(version, entity):
version = check_version_name(version) version = check_version_name(version)
return DocDatabase()[version]['entities'][entity] return DocDatabase()[version]["entities"][entity]
def get_attribute_doc(version, entity, attribute): def get_attribute_doc(version, entity, attribute):
version = check_version_name(version) version = check_version_name(version)
return DocDatabase()[version]['entities'][entity]['attributes'][attribute] return DocDatabase()[version]["entities"][entity]["attributes"][attribute]
def get_property_set_doc(version, pset): def get_property_set_doc(version, pset):
version = check_version_name(version) version = check_version_name(version)
return DocDatabase()[version]['properties'][pset] return DocDatabase()[version]["properties"][pset]
def get_property_doc(version, pset, prop): def get_property_doc(version, pset, prop):
version = check_version_name(version) version = check_version_name(version)
return DocDatabase()[version]['properties'][pset]['properties'][prop] return DocDatabase()[version]["properties"][pset]["properties"][prop]
class DocExtractor: class DocExtractor:
def extract_ifc2x3(self): def extract_ifc2x3(self):
print('Parsing data for Ifc2.3.0.1') print("Parsing data for Ifc2.3.0.1")
if not IFC2x3_DOCS_LOCATION.is_dir(): if not IFC2x3_DOCS_LOCATION.is_dir():
raise Exception( raise Exception(
f'Docs for IFC 2.3.0.1 expected to be in folder "{IFC2x3_DOCS_LOCATION.resolve()}\\"\n' f'Docs for IFC 2.3.0.1 expected to be in folder "{IFC2x3_DOCS_LOCATION.resolve()}\\"\n'
'For doc extraction please either setup docs as described above \n' "For doc extraction please either setup docs as described above \n"
'or change IFC2x3_DOCS_LOCATION in doc.py accordingly. \n' "or change IFC2x3_DOCS_LOCATION in doc.py accordingly. \n"
'You can download docs from the repository: \n' "You can download docs from the repository: \n"
'https://github.com/buildingSMART/IFC/tree/Ifc2.3.0.1' "https://github.com/buildingSMART/IFC/tree/Ifc2.3.0.1"
) )
# need to parse actual domains from the website # need to parse actual domains from the website
@@ -117,95 +123,90 @@ class DocExtractor:
self.extract_ifc2x3_property_sets_site_domains() self.extract_ifc2x3_property_sets_site_domains()
self.extract_ifc2x3_entities() self.extract_ifc2x3_entities()
self.extract_ifc2x3_property_sets() self.extract_ifc2x3_property_sets()
def extract_ifc2x3_property_sets_site_domains(self): def extract_ifc2x3_property_sets_site_domains(self):
property_sets_domains = dict() property_sets_domains = dict()
r = requests.get('https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/psd/psd_index.htm') r = requests.get("https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/psd/psd_index.htm")
html = BeautifulSoup(r.content, features='lxml') html = BeautifulSoup(r.content, features="lxml")
for a in html.find_all('a'): for a in html.find_all("a"):
domain, pset = a['href'].removeprefix('./').removesuffix('.xml').split('/') domain, pset = a["href"].removeprefix("./").removesuffix(".xml").split("/")
property_sets_domains[pset] = domain property_sets_domains[pset] = domain
# export property sets data # export property sets data
with open( with open(BASE_MODULE_PATH / "schema/ifc2x3_property_sets_site_domains.json", "w", encoding="utf-8") as fo:
BASE_MODULE_PATH / 'schema/ifc2x3_property_sets_site_domains.json', print(f"{len(property_sets_domains)} property sets domains were parsed from the website")
'w', encoding='utf-8') as fo: json.dump(property_sets_domains, fo, sort_keys=True, indent=4)
print(f'{len(property_sets_domains)} property sets domains were parsed from the website')
json.dump(
property_sets_domains, fo,
sort_keys=True, indent=4
)
def extract_ifc2x3_entities(self): def extract_ifc2x3_entities(self):
entities_dict = dict() entities_dict = dict()
# search # search
entities_paths = [filepath entities_paths = [
for filepath in glob.iglob(f'{IFC2x3_DOCS_LOCATION}/Sections/**/Entities', recursive=True)] filepath for filepath in glob.iglob(f"{IFC2x3_DOCS_LOCATION}/Sections/**/Entities", recursive=True)
]
for parse_folder_path in entities_paths: for parse_folder_path in entities_paths:
for entity_path in glob.iglob(f'{parse_folder_path}/**/'): for entity_path in glob.iglob(f"{parse_folder_path}/**/"):
entity_path = Path(entity_path) entity_path = Path(entity_path)
entity_name = entity_path.stem entity_name = entity_path.stem
entities_dict[entity_name] = dict() entities_dict[entity_name] = dict()
# utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded # utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded
md_path = entity_path / 'Documentation.md' md_path = entity_path / "Documentation.md"
xml_path = entity_path / 'DocEntity.xml' xml_path = entity_path / "DocEntity.xml"
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
with open(md_path, 'r', encoding='utf-8-sig') as fi: with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing # convert markdown to html for easier parsing
html = markdown(fi.read()) html = markdown(fi.read())
entity_description = BeautifulSoup(html, features="lxml").find('p').text entity_description = BeautifulSoup(html, features="lxml").find("p").text
entity_description = entity_description.replace('\n', ' ') entity_description = entity_description.replace("\n", " ")
entity_description = entity_description.replace('\u00a0', ' ') entity_description = entity_description.replace("\u00a0", " ")
with open(xml_path, 'r', encoding='utf-8') as fi: with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml') bs_tree = BeautifulSoup(fi.read(), features="lxml")
entity_attrs = dict() entity_attrs = dict()
# temporarily disable MarkupResemblesLocatorWarning # temporarily disable MarkupResemblesLocatorWarning
# because BeautifulSoup wrongly assume we confused # because BeautifulSoup wrongly assume we confused
# html code for filepath and gives warnings # html code for filepath and gives warnings
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter('ignore', category=MarkupResemblesLocatorWarning) warnings.simplefilter("ignore", category=MarkupResemblesLocatorWarning)
for html_attr in bs_tree.find_all('docattribute'): for html_attr in bs_tree.find_all("docattribute"):
html_description = BeautifulSoup(html_attr.text, features='lxml') html_description = BeautifulSoup(html_attr.text, features="lxml")
attr_description = html_description.get_text() attr_description = html_description.get_text()
attr_description = attr_description.replace('\n', ' ') attr_description = attr_description.replace("\n", " ")
attr_description = attr_description.replace('\u00a0', ' ') attr_description = attr_description.replace("\u00a0", " ")
attr_description = attr_description.replace('&npsp;', ' ') attr_description = attr_description.replace("&npsp;", " ")
# discard part of the description with changelog # discard part of the description with changelog
# Example: # Example:
# https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/ifcpresentationdefinitionresource/lexical/ifcannotationfillarea.htm # https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/ifcpresentationdefinitionresource/lexical/ifcannotationfillarea.htm
attr_description = attr_description.split('IFC2x Edition 3 CHANGE', 1)[0] attr_description = attr_description.split("IFC2x Edition 3 CHANGE", 1)[0]
attr_description = attr_description.split('IFC2x Edition 2 Addendum 2 CHANGE', 1)[0] attr_description = attr_description.split("IFC2x Edition 2 Addendum 2 CHANGE", 1)[0]
attr_description = attr_description.split('IFC2x2 Addendum 1 change', 1)[0] attr_description = attr_description.split("IFC2x2 Addendum 1 change", 1)[0]
attr_description = attr_description.split('IFC2x PLATFORM CHANGE', 1)[0] attr_description = attr_description.split("IFC2x PLATFORM CHANGE", 1)[0]
attr_description = attr_description.split('IFC2x3 CHANGE', 1)[0] attr_description = attr_description.split("IFC2x3 CHANGE", 1)[0]
attr_description = attr_description.split('IFC2x Edition3 CHANGE', 1)[0] attr_description = attr_description.split("IFC2x Edition3 CHANGE", 1)[0]
attr_description = attr_description.strip().rstrip('>').strip() attr_description = attr_description.strip().rstrip(">").strip()
entity_attrs[html_attr['name']] = attr_description entity_attrs[html_attr["name"]] = attr_description
if entity_attrs: if entity_attrs:
entities_dict[entity_name]['attributes'] = entity_attrs entities_dict[entity_name]["attributes"] = entity_attrs
entities_dict[entity_name]['description'] = entity_description entities_dict[entity_name]["description"] = entity_description
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/' \ spec_url = (
f'{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm' "https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/"
entities_dict[entity_name]['spec_url'] = spec_url f"{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm"
)
entities_dict[entity_name]["spec_url"] = spec_url
# export entities data # export entities data
with open(BASE_MODULE_PATH / 'schema/ifc2x3_entities.json', 'w', encoding='utf-8') as fo: with open(BASE_MODULE_PATH / "schema/ifc2x3_entities.json", "w", encoding="utf-8") as fo:
print(f'{len(entities_dict)} entities parsed') print(f"{len(entities_dict)} entities parsed")
json.dump( json.dump(entities_dict, fo, sort_keys=True, indent=4)
entities_dict, fo,
sort_keys=True, indent=4
)
def extract_ifc2x3_property_sets(self): def extract_ifc2x3_property_sets(self):
property_sets_dict = dict() property_sets_dict = dict()
@@ -213,39 +214,42 @@ class DocExtractor:
property_sets_spec_urls = dict() property_sets_spec_urls = dict()
# extract lists of properties and theirs references for each property set # extract lists of properties and theirs references for each property set
parsed_paths = [filepath parsed_paths = [
for filepath in glob.iglob(f'{IFC2x3_DOCS_LOCATION}/Sections/**/PropertySets', recursive=True)] filepath for filepath in glob.iglob(f"{IFC2x3_DOCS_LOCATION}/Sections/**/PropertySets", recursive=True)
]
# prepare property sets domains from the website we extracted earlier # prepare property sets domains from the website we extracted earlier
with open(BASE_MODULE_PATH / 'schema/ifc2x3_property_sets_site_domains.json', 'r') as fi: with open(BASE_MODULE_PATH / "schema/ifc2x3_property_sets_site_domains.json", "r") as fi:
property_sets_site_domains = json.load(fi) property_sets_site_domains = json.load(fi)
for parse_folder_path in parsed_paths: for parse_folder_path in parsed_paths:
for property_set_path in glob.iglob(f'{parse_folder_path}/**/'): for property_set_path in glob.iglob(f"{parse_folder_path}/**/"):
property_set_path = Path(property_set_path) property_set_path = Path(property_set_path)
property_set_name = property_set_path.stem property_set_name = property_set_path.stem
property_references = list() property_references = list()
xml_path = property_set_path / 'DocPropertySet.xml' xml_path = property_set_path / "DocPropertySet.xml"
with open(xml_path, 'r', encoding='utf-8') as fi: with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml') bs_tree = BeautifulSoup(fi.read(), features="lxml")
for html_attr in bs_tree.find_all('docproperty'): for html_attr in bs_tree.find_all("docproperty"):
property_references.append(html_attr['href']) property_references.append(html_attr["href"])
property_sets_references[property_set_name] = property_references property_sets_references[property_set_name] = property_references
property_set_domain = property_sets_site_domains[property_set_name] property_set_domain = property_sets_site_domains[property_set_name]
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML' \ spec_url = (
f'/psd/{property_set_domain}/{property_set_name}.xml' "https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML"
f"/psd/{property_set_domain}/{property_set_name}.xml"
)
property_sets_spec_urls[property_set_name] = spec_url property_sets_spec_urls[property_set_name] = spec_url
# setup references look up tables to convert property hrefs to actual data paths # setup references look up tables to convert property hrefs to actual data paths
references_paths_lookup = dict() references_paths_lookup = dict()
glob_query = f'{IFC2x3_DOCS_LOCATION}/Properties/*/*' glob_query = f"{IFC2x3_DOCS_LOCATION}/Properties/*/*"
for parsed_path in [filepath for filepath in glob.iglob(glob_query, recursive=False)]: for parsed_path in [filepath for filepath in glob.iglob(glob_query, recursive=False)]:
parsed_path = Path(parsed_path) parsed_path = Path(parsed_path)
# all references omit "$" character, I've checked it on 2_3 # all references omit "$" character, I've checked it on 2_3
# need to check it if moving to next IFC version # need to check it if moving to next IFC version
property_reference = parsed_path.name.replace('$', '') property_reference = parsed_path.name.replace("$", "")
references_paths_lookup[property_reference] = parsed_path references_paths_lookup[property_reference] = parsed_path
# setup a function because we'll need to check child properties recusively # setup a function because we'll need to check child properties recusively
@@ -253,51 +257,53 @@ class DocExtractor:
property_dict = dict() property_dict = dict()
property_path = references_paths_lookup[href] property_path = references_paths_lookup[href]
md_path = property_path / 'Documentation.md' md_path = property_path / "Documentation.md"
xml_path = property_path / 'DocProperty.xml' xml_path = property_path / "DocProperty.xml"
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix())) xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
github_xml_url = f'https://github.com/buildingSMART/IFC/blob/{xml_url_part}' github_xml_url = f"https://github.com/buildingSMART/IFC/blob/{xml_url_part}"
with open(xml_path, 'r', encoding='utf-8') as fi: with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml') bs_tree = BeautifulSoup(fi.read(), features="lxml")
tags = bs_tree.find_all('docproperty') tags = bs_tree.find_all("docproperty")
# check for child properties - if they are present parse their data recursively # check for child properties - if they are present parse their data recursively
elements_tag = bs_tree.find('elements') elements_tag = bs_tree.find("elements")
if elements_tag is not None: if elements_tag is not None:
child_tags = elements_tag.find_all('docproperty') child_tags = elements_tag.find_all("docproperty")
child_tags_dict = dict() child_tags_dict = dict()
for child_tag in child_tags: for child_tag in child_tags:
child_tag_href = child_tag['href'] child_tag_href = child_tag["href"]
child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href) child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href)
child_tags_dict[child_tag_name] = child_tag_dict child_tags_dict[child_tag_name] = child_tag_dict
tags.remove(child_tag) tags.remove(child_tag)
property_dict['children'] = child_tags_dict property_dict["children"] = child_tags_dict
print(f'Child nodes found inside property xml. Url: {github_xml_url}') print(f"Child nodes found inside property xml. Url: {github_xml_url}")
if len(tags) != 1: if len(tags) != 1:
print(f'WARNING. Found more properties inside property xml, ' print(
f'only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}.') f"WARNING. Found more properties inside property xml, "
property_name = tags[0]['name'] f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
)
property_name = tags[0]["name"]
if not md_path.is_file(): if not md_path.is_file():
print(f'WARNING. Property {property_name} is missing documentation.md, ' print(
f'property will be left without description. Url: {github_xml_url}') f"WARNING. Property {property_name} is missing documentation.md, "
f"property will be left without description. Url: {github_xml_url}"
)
else: else:
with open(md_path, 'r', encoding='utf-8-sig') as fi: with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing # convert markdown to html for easier parsing
html = markdown(fi.read()) html = markdown(fi.read())
description = BeautifulSoup(html, features="lxml").find('p').text description = BeautifulSoup(html, features="lxml").find("p").text
description = description.replace('\n', ' ') description = description.replace("\n", " ")
description = description.replace('\u00a0', ' ') description = description.replace("\u00a0", " ")
property_dict['description'] = description property_dict["description"] = description
return (property_name, property_dict) return (property_name, property_dict)
# lookup each property reference and save it's name and description # lookup each property reference and save it's name and description
for property_set_name in property_sets_references: for property_set_name in property_sets_references:
properties_dict = dict() properties_dict = dict()
@@ -305,28 +311,24 @@ class DocExtractor:
property_name, property_dict = get_property_info_by_href(property_reference) property_name, property_dict = get_property_info_by_href(property_reference)
properties_dict[property_name] = property_dict properties_dict[property_name] = property_dict
property_sets_dict[property_set_name] = { property_sets_dict[property_set_name] = {
'properties': properties_dict, "properties": properties_dict,
'spec_url': property_sets_spec_urls[property_set_name] "spec_url": property_sets_spec_urls[property_set_name],
} }
# export property sets data # export property sets data
with open(BASE_MODULE_PATH / 'schema/ifc2x3_properties.json', 'w', encoding='utf-8') as fo: with open(BASE_MODULE_PATH / "schema/ifc2x3_properties.json", "w", encoding="utf-8") as fo:
print(f'{len(property_sets_dict)} property sets parsed') print(f"{len(property_sets_dict)} property sets parsed")
json.dump( json.dump(property_sets_dict, fo, sort_keys=True, indent=4)
property_sets_dict, fo,
sort_keys=True, indent=4
)
def extract_ifc4(self): def extract_ifc4(self):
print('Parsing data for Ifc4.0.2.1') print("Parsing data for Ifc4.0.2.1")
if not IFC4_DOCS_LOCATION.is_dir(): if not IFC4_DOCS_LOCATION.is_dir():
raise Exception( raise Exception(
f'Docs for Ifc4.0.2.1 expected to be in folder "{IFC4_DOCS_LOCATION.resolve()}\\"\n' f'Docs for Ifc4.0.2.1 expected to be in folder "{IFC4_DOCS_LOCATION.resolve()}\\"\n'
'For doc extraction please either setup docs as described above \n' "For doc extraction please either setup docs as described above \n"
'or change IFC4_DOCS_LOCATION in doc.py accordingly.' "or change IFC4_DOCS_LOCATION in doc.py accordingly."
'You can download docs from the repository: \n' "You can download docs from the repository: \n"
'https://github.com/buildingSMART/IFC/tree/Ifc4.0.2.1' "https://github.com/buildingSMART/IFC/tree/Ifc4.0.2.1"
) )
# actually domains in Ifc 4.0 are consistent between website and docs # actually domains in Ifc 4.0 are consistent between website and docs
@@ -337,107 +339,106 @@ class DocExtractor:
self.extract_ifc4_property_sets_site_domains() self.extract_ifc4_property_sets_site_domains()
self.extract_ifc4_entities() self.extract_ifc4_entities()
self.extract_ifc4_property_sets() self.extract_ifc4_property_sets()
def extract_ifc4_property_sets_site_domains(self): def extract_ifc4_property_sets_site_domains(self):
property_sets_domains = dict() property_sets_domains = dict()
with requests.get( with requests.get(
'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1' "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1"
'/HTML/annex/annex-b/alphabeticalorder_psets.htm') as r: "/HTML/annex/annex-b/alphabeticalorder_psets.htm"
html = BeautifulSoup(r.content, features='lxml') ) as r:
for a in html.find_all('a', {'class': 'listing-link'}): html = BeautifulSoup(r.content, features="lxml")
href_split = a['href'].split('/') for a in html.find_all("a", {"class": "listing-link"}):
href_split = a["href"].split("/")
domain = href_split[3] domain = href_split[3]
pset = href_split[5].removesuffix('.htm') pset = href_split[5].removesuffix(".htm")
property_sets_domains[pset] = domain property_sets_domains[pset] = domain
with requests.get( with requests.get(
'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/' "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/"
'/HTML/annex/annex-b/alphabeticalorder_qsets.htm') as r: "/HTML/annex/annex-b/alphabeticalorder_qsets.htm"
html = BeautifulSoup(r.content, features='lxml') ) as r:
for a in html.find_all('a', {'class': 'listing-link'}): html = BeautifulSoup(r.content, features="lxml")
href_split = a['href'].split('/') for a in html.find_all("a", {"class": "listing-link"}):
href_split = a["href"].split("/")
domain = href_split[3] domain = href_split[3]
pset = href_split[5].removesuffix('.htm') pset = href_split[5].removesuffix(".htm")
property_sets_domains[pset] = domain property_sets_domains[pset] = domain
# export property sets data # export property sets data
with open(BASE_MODULE_PATH / 'schema/ifc4_property_sets_site_domains.json', 'w', encoding='utf-8') as fo: with open(BASE_MODULE_PATH / "schema/ifc4_property_sets_site_domains.json", "w", encoding="utf-8") as fo:
print(f'{len(property_sets_domains)} property sets domains were parsed from the website') print(f"{len(property_sets_domains)} property sets domains were parsed from the website")
json.dump( json.dump(property_sets_domains, fo, sort_keys=True, indent=4)
property_sets_domains, fo,
sort_keys=True, indent=4
)
def extract_ifc4_entities(self): def extract_ifc4_entities(self):
entities_dict = dict() entities_dict = dict()
# search # search
entities_paths = [filepath entities_paths = [
for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/Entities', recursive=True)] filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/Entities", recursive=True)
]
for parse_folder_path in entities_paths: for parse_folder_path in entities_paths:
for entity_path in glob.iglob(f'{parse_folder_path}/**/'): for entity_path in glob.iglob(f"{parse_folder_path}/**/"):
entity_path = Path(entity_path) entity_path = Path(entity_path)
entity_name = entity_path.stem entity_name = entity_path.stem
entities_dict[entity_name] = dict() entities_dict[entity_name] = dict()
# utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded # utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded
md_path = entity_path / 'Documentation.md' md_path = entity_path / "Documentation.md"
xml_path = entity_path / 'DocEntity.xml' xml_path = entity_path / "DocEntity.xml"
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
with open(md_path, 'r', encoding='utf-8-sig') as fi: with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing # convert markdown to html for easier parsing
html = markdown(fi.read()) html = markdown(fi.read())
entity_description = BeautifulSoup(html, features="lxml").find('p').text entity_description = BeautifulSoup(html, features="lxml").find("p").text
entity_description = entity_description.replace('\n', ' ') entity_description = entity_description.replace("\n", " ")
entity_description = entity_description.replace('\u00a0', ' ') entity_description = entity_description.replace("\u00a0", " ")
entity_description = entity_description.replace('{ .extDef}', '') entity_description = entity_description.replace("{ .extDef}", "")
entity_description = entity_description.strip() entity_description = entity_description.strip()
with open(xml_path, 'r', encoding='utf-8') as fi: with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml') bs_tree = BeautifulSoup(fi.read(), features="lxml")
entity_attrs = dict() entity_attrs = dict()
# temporarily disable MarkupResemblesLocatorWarning # temporarily disable MarkupResemblesLocatorWarning
# because BeautifulSoup wrongly assume we confused # because BeautifulSoup wrongly assume we confused
# html code for filepath and gives warnings # html code for filepath and gives warnings
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter('ignore', category=MarkupResemblesLocatorWarning) warnings.simplefilter("ignore", category=MarkupResemblesLocatorWarning)
for html_attr in bs_tree.find_all('docattribute'): for html_attr in bs_tree.find_all("docattribute"):
html_description = BeautifulSoup(html_attr.text, features='lxml') html_description = BeautifulSoup(html_attr.text, features="lxml")
attr_description = html_description.get_text() attr_description = html_description.get_text()
attr_description = attr_description.replace('\n', ' ') attr_description = attr_description.replace("\n", " ")
attr_description = attr_description.replace('\u00a0', ' ') attr_description = attr_description.replace("\u00a0", " ")
# discard part of the description with changelog, notes and examples etc. # discard part of the description with changelog, notes and examples etc.
# Those notes actually can be useful but we'll need a way to reformat them # Those notes actually can be useful but we'll need a way to reformat them
# Example: # Example:
# https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/ifcsharedbldgelements/lexical/ifcrelconnectspathelements.htm # https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/ifcsharedbldgelements/lexical/ifcrelconnectspathelements.htm
attr_description = attr_description.split('{ .change-ifc', 1)[0] attr_description = attr_description.split("{ .change-ifc", 1)[0]
attr_description = attr_description.split('{ .note', 1)[0] attr_description = attr_description.split("{ .note", 1)[0]
attr_description = attr_description.split('{ .examples', 1)[0] attr_description = attr_description.split("{ .examples", 1)[0]
attr_description = attr_description.split('{ .deprecated', 1)[0] attr_description = attr_description.split("{ .deprecated", 1)[0]
attr_description = attr_description.split('{ .history', 1)[0] attr_description = attr_description.split("{ .history", 1)[0]
attr_description = attr_description.strip() attr_description = attr_description.strip()
entity_attrs[html_attr['name']] = attr_description entity_attrs[html_attr["name"]] = attr_description
if entity_attrs: if entity_attrs:
entities_dict[entity_name]['attributes'] = entity_attrs entities_dict[entity_name]["attributes"] = entity_attrs
entities_dict[entity_name]['description'] = entity_description entities_dict[entity_name]["description"] = entity_description
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/' \ spec_url = (
f'{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm' "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/"
entities_dict[entity_name]['spec_url'] = spec_url f"{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm"
)
entities_dict[entity_name]["spec_url"] = spec_url
# entities_dict[entity_name]['github_url'] = github_md_url # entities_dict[entity_name]['github_url'] = github_md_url
# export entities data # export entities data
with open(BASE_MODULE_PATH / 'schema/ifc4_entities.json', 'w', encoding='utf-8') as fo: with open(BASE_MODULE_PATH / "schema/ifc4_entities.json", "w", encoding="utf-8") as fo:
print(f'{len(entities_dict)} entities parsed') print(f"{len(entities_dict)} entities parsed")
json.dump( json.dump(entities_dict, fo, sort_keys=True, indent=4)
entities_dict, fo,
sort_keys=True, indent=4
)
def extract_ifc4_property_sets(self): def extract_ifc4_property_sets(self):
# function parses both property and quantity sets # function parses both property and quantity sets
@@ -446,105 +447,114 @@ class DocExtractor:
property_sets_spec_urls = dict() property_sets_spec_urls = dict()
# extract lists of properties and theirs references for each property set # extract lists of properties and theirs references for each property set
parsed_paths = [filepath for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/PropertySets', recursive=True)] parsed_paths = [
parsed_paths += [filepath for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/QuantitySets', recursive=True)] filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/PropertySets", recursive=True)
]
parsed_paths += [
filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/QuantitySets", recursive=True)
]
# prepare property sets domains from the website we extracted earlier # prepare property sets domains from the website we extracted earlier
with open(BASE_MODULE_PATH / 'schema/ifc4_property_sets_site_domains.json', 'r') as fi: with open(BASE_MODULE_PATH / "schema/ifc4_property_sets_site_domains.json", "r") as fi:
property_sets_site_domains = json.load(fi) property_sets_site_domains = json.load(fi)
psets_test = set() psets_test = set()
for parse_folder_path in parsed_paths: for parse_folder_path in parsed_paths:
for property_set_path in glob.iglob(f'{parse_folder_path}/**/'): for property_set_path in glob.iglob(f"{parse_folder_path}/**/"):
property_set_path = Path(property_set_path) property_set_path = Path(property_set_path)
property_set_name = property_set_path.stem property_set_name = property_set_path.stem
property_references = list() property_references = list()
property_quantity = property_set_path.parents[0].name == 'QuantitySets' property_quantity = property_set_path.parents[0].name == "QuantitySets"
xml_path = property_set_path / ('DocQuantitySet.xml' if property_quantity else 'DocPropertySet.xml') xml_path = property_set_path / ("DocQuantitySet.xml" if property_quantity else "DocPropertySet.xml")
with open(xml_path, 'r', encoding='utf-8') as fi: with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml') bs_tree = BeautifulSoup(fi.read(), features="lxml")
for html_attr in bs_tree.find_all('docquantity' if property_quantity else 'docproperty'): for html_attr in bs_tree.find_all("docquantity" if property_quantity else "docproperty"):
property_references.append(html_attr['href']) property_references.append(html_attr["href"])
property_sets_references[property_set_name] = property_references property_sets_references[property_set_name] = property_references
if property_set_name.lower() not in property_sets_site_domains: if property_set_name.lower() not in property_sets_site_domains:
print(f"WARNING. {property_set_name} was not found on the spec website, " print(
"this property set won't have any spec_url in schema.") f"WARNING. {property_set_name} was not found on the spec website, "
"this property set won't have any spec_url in schema."
)
else: else:
property_set_domain = property_sets_site_domains.get(property_set_name.lower(), '') property_set_domain = property_sets_site_domains.get(property_set_name.lower(), "")
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML' \ spec_url = (
f'/schema/{property_set_domain}' \ "https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML"
f'/{"qset" if property_quantity else "pset"}' \ f"/schema/{property_set_domain}"
f'/{property_set_name.lower()}.htm' f'/{"qset" if property_quantity else "pset"}'
f"/{property_set_name.lower()}.htm"
)
property_sets_spec_urls[property_set_name] = spec_url property_sets_spec_urls[property_set_name] = spec_url
# setup references look up tables to convert property hrefs to actual data paths # setup references look up tables to convert property hrefs to actual data paths
references_paths_lookup = dict() references_paths_lookup = dict()
parsed_paths = [filepath parsed_paths = [filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Properties/*/*", recursive=False)]
for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Properties/*/*', recursive=False)] parsed_paths += [filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Quantities/*/*", recursive=False)]
parsed_paths += [filepath
for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Quantities/*/*', recursive=False)]
for parsed_path in parsed_paths: for parsed_path in parsed_paths:
parsed_path = Path(parsed_path) parsed_path = Path(parsed_path)
# all references omit "$" character, I've checked it on 4_0 # all references omit "$" character, I've checked it on 4_0
# need to check it if moving to next IFC version # need to check it if moving to next IFC version
# btw no reason to check if all references were used in properties # btw no reason to check if all references were used in properties
# because there are also child properties # because there are also child properties
property_reference = parsed_path.name.replace('$', '') property_reference = parsed_path.name.replace("$", "")
references_paths_lookup[property_reference] = parsed_path references_paths_lookup[property_reference] = parsed_path
# setup a function because we'll need to check child properties recusively # setup a function because we'll need to check child properties recusively
def get_property_info_by_href(href): def get_property_info_by_href(href):
property_dict = dict() property_dict = dict()
property_path = references_paths_lookup[href] property_path = references_paths_lookup[href]
property_quantity = property_path.parents[1].name == 'Quantities' property_quantity = property_path.parents[1].name == "Quantities"
md_path = property_path / 'Documentation.md' md_path = property_path / "Documentation.md"
xml_path = property_path / ('DocQuantity.xml' if property_quantity else 'DocProperty.xml') xml_path = property_path / ("DocQuantity.xml" if property_quantity else "DocProperty.xml")
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix())) md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}' github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix())) xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix()))
github_xml_url = f'https://github.com/buildingSMART/IFC/blob/{xml_url_part}' github_xml_url = f"https://github.com/buildingSMART/IFC/blob/{xml_url_part}"
with open(xml_path, 'r', encoding='utf-8') as fi: with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml') bs_tree = BeautifulSoup(fi.read(), features="lxml")
tags = bs_tree.find_all('docquantity' if property_quantity else 'docproperty') tags = bs_tree.find_all("docquantity" if property_quantity else "docproperty")
# check for child properties - if they are present parse their data recursively # check for child properties - if they are present parse their data recursively
elements_tag = bs_tree.find('elements') elements_tag = bs_tree.find("elements")
if elements_tag is not None: if elements_tag is not None:
child_tags = elements_tag.find_all('docquantity' if property_quantity else 'docproperty') child_tags = elements_tag.find_all("docquantity" if property_quantity else "docproperty")
child_tags_dict = dict() child_tags_dict = dict()
for child_tag in child_tags: for child_tag in child_tags:
child_tag_href = child_tag['href'] child_tag_href = child_tag["href"]
child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href) child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href)
child_tags_dict[child_tag_name] = child_tag_dict child_tags_dict[child_tag_name] = child_tag_dict
tags.remove(child_tag) tags.remove(child_tag)
property_dict['children'] = child_tags_dict property_dict["children"] = child_tags_dict
print(f'Child nodes found inside property xml. Url: {github_xml_url}') print(f"Child nodes found inside property xml. Url: {github_xml_url}")
if len(tags) != 1: if len(tags) != 1:
print(f'WARNING. Found more properties inside property xml, ' print(
f'only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}.') f"WARNING. Found more properties inside property xml, "
property_name = tags[0]['name'] f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
)
property_name = tags[0]["name"]
if not md_path.is_file(): if not md_path.is_file():
print(f'WARNING. Property {property_name} is missing documentation.md, property will be left without description. ' print(
f'Url: {github_xml_url}') f"WARNING. Property {property_name} is missing documentation.md, property will be left without description. "
f"Url: {github_xml_url}"
)
else: else:
with open(md_path, 'r', encoding='utf-8-sig') as fi: with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing # convert markdown to html for easier parsing
html = markdown(fi.read()) html = markdown(fi.read())
description = BeautifulSoup(html, features="lxml").find('p').text description = BeautifulSoup(html, features="lxml").find("p").text
description = description.replace('\n', ' ') description = description.replace("\n", " ")
description = description.replace('\u00a0', ' ') description = description.replace("\u00a0", " ")
property_dict['description'] = description property_dict["description"] = description
return (property_name, property_dict) return (property_name, property_dict)
# lookup each property reference and save it's name and description # lookup each property reference and save it's name and description
@@ -553,47 +563,38 @@ class DocExtractor:
for property_reference in property_sets_references[property_set_name]: for property_reference in property_sets_references[property_set_name]:
property_name, property_dict = get_property_info_by_href(property_reference) property_name, property_dict = get_property_info_by_href(property_reference)
properties_dict[property_name] = property_dict properties_dict[property_name] = property_dict
property_sets_dict[property_set_name] = { property_sets_dict[property_set_name] = {"properties": properties_dict}
'properties': properties_dict
}
if property_set_name in property_sets_spec_urls: if property_set_name in property_sets_spec_urls:
spec_url = property_sets_spec_urls[property_set_name] spec_url = property_sets_spec_urls[property_set_name]
property_sets_dict[property_set_name]['spec_url'] = spec_url property_sets_dict[property_set_name]["spec_url"] = spec_url
# export property sets data # export property sets data
with open(BASE_MODULE_PATH / 'schema/ifc4_properties.json', 'w', encoding='utf-8') as fo: with open(BASE_MODULE_PATH / "schema/ifc4_properties.json", "w", encoding="utf-8") as fo:
print(f'{len(property_sets_dict)} property sets parsed') print(f"{len(property_sets_dict)} property sets parsed")
json.dump( json.dump(property_sets_dict, fo, sort_keys=True, indent=4)
property_sets_dict, fo,
sort_keys=True, indent=4
)
def run_doc_api_examples(): def run_doc_api_examples():
print('Entities:') print("Entities:")
print(get_entity_doc('IFC2X3', 'IfcActionRequest')) print(get_entity_doc("IFC2X3", "IfcActionRequest"))
print(get_entity_doc('IFC4', 'IfcActionRequest')) print(get_entity_doc("IFC4", "IfcActionRequest"))
print('Entity attributes:') print("Entity attributes:")
print(get_attribute_doc('IFC2X3', 'IfcActionRequest', 'RequestID')) print(get_attribute_doc("IFC2X3", "IfcActionRequest", "RequestID"))
print(get_attribute_doc('IFC4', 'IfcActionRequest', 'PredefinedType')) print(get_attribute_doc("IFC4", "IfcActionRequest", "PredefinedType"))
print('Propety sets:') print("Propety sets:")
print(get_property_set_doc('IFC2X3', 'Pset_ZoneCommon')) print(get_property_set_doc("IFC2X3", "Pset_ZoneCommon"))
print(get_property_set_doc('IFC4', 'Pset_ZoneCommon')) print(get_property_set_doc("IFC4", "Pset_ZoneCommon"))
print('Propety sets attributes:') print("Propety sets attributes:")
print(get_property_doc('IFC2X3', 'Pset_ZoneCommon', 'Category')) print(get_property_doc("IFC2X3", "Pset_ZoneCommon", "Category"))
print(get_property_doc('IFC4', 'Pset_ZoneCommon', 'NetPlannedArea')) print(get_property_doc("IFC4", "Pset_ZoneCommon", "NetPlannedArea"))
if __name__ == '__main__': if __name__ == "__main__":
extractor = DocExtractor() extractor = DocExtractor()
extractor.extract_ifc2x3() extractor.extract_ifc2x3()
extractor.extract_ifc4() extractor.extract_ifc4()
# run_doc_api_examples() # run_doc_api_examples()