Run black and minor fix. See #1990.

This commit is contained in:
Dion Moult
2022-10-19 09:17:37 +11:00
parent ec1fbabd00
commit 159e2ca779
2 changed files with 278 additions and 276 deletions
+273 -272
View File
@@ -16,40 +16,42 @@
# You should have received a copy of the GNU Lesser General Public License
# along with IfcOpenShell. If not, see <http://www.gnu.org/licenses/>.
import glob
from pathlib import Path
import json
from pprint import pprint
import urllib.parse
import warnings
from pathlib import Path
from markdown import markdown
from bs4 import BeautifulSoup
from bs4 import MarkupResemblesLocatorWarning
import requests
try:
import glob
import warnings
import requests
import urllib.parse
from markdown import markdown
from bs4 import BeautifulSoup
from bs4 import MarkupResemblesLocatorWarning
except:
pass # Only necessary if you're using it to generate the docs database
BASE_MODULE_PATH = Path(__file__).parent
IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / 'Ifc2.3.0.1'
IFC4_DOCS_LOCATION = BASE_MODULE_PATH / 'Ifc4.0.2.1'
IFC2x3_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc2.3.0.1"
IFC4_DOCS_LOCATION = BASE_MODULE_PATH / "Ifc4.0.2.1"
SCHEMA_FILES = {
'IFC2X3': {
'entities': BASE_MODULE_PATH / 'schema/ifc2x3_entities.json',
'properties': BASE_MODULE_PATH / 'schema/ifc2x3_properties.json'
"IFC2X3": {
"entities": BASE_MODULE_PATH / "schema/ifc2x3_entities.json",
"properties": BASE_MODULE_PATH / "schema/ifc2x3_properties.json",
},
"IFC4": {
"entities": BASE_MODULE_PATH / "schema/ifc4_entities.json",
"properties": BASE_MODULE_PATH / "schema/ifc4_properties.json",
},
'IFC4': {
'entities': BASE_MODULE_PATH / 'schema/ifc4_entities.json',
'properties': BASE_MODULE_PATH / 'schema/ifc4_properties.json'
}
}
# singleton doc database
# required so that database would be loaded only once
class DocDatabase():
class DocDatabase:
def __new__(cls):
if not hasattr(cls, 'db'):
if not hasattr(cls, "db"):
db = {ifc_version: dict() for ifc_version in SCHEMA_FILES}
files_missing = False
for ifc_version in SCHEMA_FILES:
@@ -59,15 +61,15 @@ class DocDatabase():
print(f"Schema file {schema_path} wasn't found.")
files_missing = True
continue
with open(schema_path, 'r') as fi:
with open(schema_path, "r") as fi:
db[ifc_version][data_type] = json.load(fi)
if files_missing:
raise Exception(
'Some schema files are missing - they contain neccessary data for DocAPI to work. \n'
'Make sure those files are present. To generate them you can run DocExtractor extract functions \n'
'but it will require corresponding Ifc docs to be in the same directory as the script.'
"Some schema files are missing - they contain neccessary data for DocAPI to work. \n"
"Make sure those files are present. To generate them you can run DocExtractor extract functions \n"
"but it will require corresponding Ifc docs to be in the same directory as the script."
)
cls.db = db
return cls.db
@@ -76,37 +78,41 @@ class DocDatabase():
def check_version_name(version_name):
if version_name not in SCHEMA_FILES:
raise Exception(
f'Version: {version_name} is not supported. '
f'Supported version: {", ".join(SCHEMA_FILES.keys())}')
f"Version: {version_name} is not supported. " f'Supported version: {", ".join(SCHEMA_FILES.keys())}'
)
return version_name
def get_entity_doc(version, entity):
version = check_version_name(version)
return DocDatabase()[version]['entities'][entity]
return DocDatabase()[version]["entities"][entity]
def get_attribute_doc(version, entity, attribute):
version = check_version_name(version)
return DocDatabase()[version]['entities'][entity]['attributes'][attribute]
return DocDatabase()[version]["entities"][entity]["attributes"][attribute]
def get_property_set_doc(version, pset):
version = check_version_name(version)
return DocDatabase()[version]['properties'][pset]
return DocDatabase()[version]["properties"][pset]
def get_property_doc(version, pset, prop):
version = check_version_name(version)
return DocDatabase()[version]['properties'][pset]['properties'][prop]
return DocDatabase()[version]["properties"][pset]["properties"][prop]
class DocExtractor:
def extract_ifc2x3(self):
print('Parsing data for Ifc2.3.0.1')
print("Parsing data for Ifc2.3.0.1")
if not IFC2x3_DOCS_LOCATION.is_dir():
raise Exception(
f'Docs for IFC 2.3.0.1 expected to be in folder "{IFC2x3_DOCS_LOCATION.resolve()}\\"\n'
'For doc extraction please either setup docs as described above \n'
'or change IFC2x3_DOCS_LOCATION in doc.py accordingly. \n'
'You can download docs from the repository: \n'
'https://github.com/buildingSMART/IFC/tree/Ifc2.3.0.1'
"For doc extraction please either setup docs as described above \n"
"or change IFC2x3_DOCS_LOCATION in doc.py accordingly. \n"
"You can download docs from the repository: \n"
"https://github.com/buildingSMART/IFC/tree/Ifc2.3.0.1"
)
# need to parse actual domains from the website
@@ -117,95 +123,90 @@ class DocExtractor:
self.extract_ifc2x3_property_sets_site_domains()
self.extract_ifc2x3_entities()
self.extract_ifc2x3_property_sets()
def extract_ifc2x3_property_sets_site_domains(self):
property_sets_domains = dict()
r = requests.get('https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/psd/psd_index.htm')
html = BeautifulSoup(r.content, features='lxml')
for a in html.find_all('a'):
domain, pset = a['href'].removeprefix('./').removesuffix('.xml').split('/')
r = requests.get("https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/psd/psd_index.htm")
html = BeautifulSoup(r.content, features="lxml")
for a in html.find_all("a"):
domain, pset = a["href"].removeprefix("./").removesuffix(".xml").split("/")
property_sets_domains[pset] = domain
# export property sets data
with open(
BASE_MODULE_PATH / 'schema/ifc2x3_property_sets_site_domains.json',
'w', encoding='utf-8') as fo:
print(f'{len(property_sets_domains)} property sets domains were parsed from the website')
json.dump(
property_sets_domains, fo,
sort_keys=True, indent=4
)
with open(BASE_MODULE_PATH / "schema/ifc2x3_property_sets_site_domains.json", "w", encoding="utf-8") as fo:
print(f"{len(property_sets_domains)} property sets domains were parsed from the website")
json.dump(property_sets_domains, fo, sort_keys=True, indent=4)
def extract_ifc2x3_entities(self):
entities_dict = dict()
# search
entities_paths = [filepath
for filepath in glob.iglob(f'{IFC2x3_DOCS_LOCATION}/Sections/**/Entities', recursive=True)]
# search
entities_paths = [
filepath for filepath in glob.iglob(f"{IFC2x3_DOCS_LOCATION}/Sections/**/Entities", recursive=True)
]
for parse_folder_path in entities_paths:
for entity_path in glob.iglob(f'{parse_folder_path}/**/'):
for entity_path in glob.iglob(f"{parse_folder_path}/**/"):
entity_path = Path(entity_path)
entity_name = entity_path.stem
entities_dict[entity_name] = dict()
# utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded
md_path = entity_path / 'Documentation.md'
xml_path = entity_path / 'DocEntity.xml'
md_path = entity_path / "Documentation.md"
xml_path = entity_path / "DocEntity.xml"
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}'
github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
with open(md_path, 'r', encoding='utf-8-sig') as fi:
with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing
html = markdown(fi.read())
entity_description = BeautifulSoup(html, features="lxml").find('p').text
entity_description = entity_description.replace('\n', ' ')
entity_description = entity_description.replace('\u00a0', ' ')
entity_description = BeautifulSoup(html, features="lxml").find("p").text
entity_description = entity_description.replace("\n", " ")
entity_description = entity_description.replace("\u00a0", " ")
with open(xml_path, 'r', encoding='utf-8') as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml')
with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features="lxml")
entity_attrs = dict()
# temporarily disable MarkupResemblesLocatorWarning
# because BeautifulSoup wrongly assume we confused
# because BeautifulSoup wrongly assume we confused
# html code for filepath and gives warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=MarkupResemblesLocatorWarning)
warnings.simplefilter("ignore", category=MarkupResemblesLocatorWarning)
for html_attr in bs_tree.find_all('docattribute'):
html_description = BeautifulSoup(html_attr.text, features='lxml')
for html_attr in bs_tree.find_all("docattribute"):
html_description = BeautifulSoup(html_attr.text, features="lxml")
attr_description = html_description.get_text()
attr_description = attr_description.replace('\n', ' ')
attr_description = attr_description.replace('\u00a0', ' ')
attr_description = attr_description.replace('&npsp;', ' ')
attr_description = attr_description.replace("\n", " ")
attr_description = attr_description.replace("\u00a0", " ")
attr_description = attr_description.replace("&npsp;", " ")
# discard part of the description with changelog
# Example:
# https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/ifcpresentationdefinitionresource/lexical/ifcannotationfillarea.htm
attr_description = attr_description.split('IFC2x Edition 3 CHANGE', 1)[0]
attr_description = attr_description.split('IFC2x Edition 2 Addendum 2 CHANGE', 1)[0]
attr_description = attr_description.split('IFC2x2 Addendum 1 change', 1)[0]
attr_description = attr_description.split('IFC2x PLATFORM CHANGE', 1)[0]
attr_description = attr_description.split('IFC2x3 CHANGE', 1)[0]
attr_description = attr_description.split('IFC2x Edition3 CHANGE', 1)[0]
attr_description = attr_description.split("IFC2x Edition 3 CHANGE", 1)[0]
attr_description = attr_description.split("IFC2x Edition 2 Addendum 2 CHANGE", 1)[0]
attr_description = attr_description.split("IFC2x2 Addendum 1 change", 1)[0]
attr_description = attr_description.split("IFC2x PLATFORM CHANGE", 1)[0]
attr_description = attr_description.split("IFC2x3 CHANGE", 1)[0]
attr_description = attr_description.split("IFC2x Edition3 CHANGE", 1)[0]
attr_description = attr_description.strip().rstrip('>').strip()
entity_attrs[html_attr['name']] = attr_description
attr_description = attr_description.strip().rstrip(">").strip()
entity_attrs[html_attr["name"]] = attr_description
if entity_attrs:
entities_dict[entity_name]['attributes'] = entity_attrs
entities_dict[entity_name]["attributes"] = entity_attrs
entities_dict[entity_name]['description'] = entity_description
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/' \
f'{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm'
entities_dict[entity_name]['spec_url'] = spec_url
entities_dict[entity_name]["description"] = entity_description
spec_url = (
"https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML/"
f"{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm"
)
entities_dict[entity_name]["spec_url"] = spec_url
# export entities data
with open(BASE_MODULE_PATH / 'schema/ifc2x3_entities.json', 'w', encoding='utf-8') as fo:
print(f'{len(entities_dict)} entities parsed')
json.dump(
entities_dict, fo,
sort_keys=True, indent=4
)
with open(BASE_MODULE_PATH / "schema/ifc2x3_entities.json", "w", encoding="utf-8") as fo:
print(f"{len(entities_dict)} entities parsed")
json.dump(entities_dict, fo, sort_keys=True, indent=4)
def extract_ifc2x3_property_sets(self):
property_sets_dict = dict()
@@ -213,39 +214,42 @@ class DocExtractor:
property_sets_spec_urls = dict()
# extract lists of properties and theirs references for each property set
parsed_paths = [filepath
for filepath in glob.iglob(f'{IFC2x3_DOCS_LOCATION}/Sections/**/PropertySets', recursive=True)]
parsed_paths = [
filepath for filepath in glob.iglob(f"{IFC2x3_DOCS_LOCATION}/Sections/**/PropertySets", recursive=True)
]
# prepare property sets domains from the website we extracted earlier
with open(BASE_MODULE_PATH / 'schema/ifc2x3_property_sets_site_domains.json', 'r') as fi:
with open(BASE_MODULE_PATH / "schema/ifc2x3_property_sets_site_domains.json", "r") as fi:
property_sets_site_domains = json.load(fi)
for parse_folder_path in parsed_paths:
for property_set_path in glob.iglob(f'{parse_folder_path}/**/'):
for property_set_path in glob.iglob(f"{parse_folder_path}/**/"):
property_set_path = Path(property_set_path)
property_set_name = property_set_path.stem
property_references = list()
xml_path = property_set_path / 'DocPropertySet.xml'
with open(xml_path, 'r', encoding='utf-8') as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml')
for html_attr in bs_tree.find_all('docproperty'):
property_references.append(html_attr['href'])
xml_path = property_set_path / "DocPropertySet.xml"
with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features="lxml")
for html_attr in bs_tree.find_all("docproperty"):
property_references.append(html_attr["href"])
property_sets_references[property_set_name] = property_references
property_set_domain = property_sets_site_domains[property_set_name]
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML' \
f'/psd/{property_set_domain}/{property_set_name}.xml'
spec_url = (
"https://standards.buildingsmart.org/IFC/RELEASE/IFC2x3/TC1/HTML"
f"/psd/{property_set_domain}/{property_set_name}.xml"
)
property_sets_spec_urls[property_set_name] = spec_url
# setup references look up tables to convert property hrefs to actual data paths
references_paths_lookup = dict()
glob_query = f'{IFC2x3_DOCS_LOCATION}/Properties/*/*'
glob_query = f"{IFC2x3_DOCS_LOCATION}/Properties/*/*"
for parsed_path in [filepath for filepath in glob.iglob(glob_query, recursive=False)]:
parsed_path = Path(parsed_path)
# all references omit "$" character, I've checked it on 2_3
# need to check it if moving to next IFC version
property_reference = parsed_path.name.replace('$', '')
property_reference = parsed_path.name.replace("$", "")
references_paths_lookup[property_reference] = parsed_path
# setup a function because we'll need to check child properties recusively
@@ -253,51 +257,53 @@ class DocExtractor:
property_dict = dict()
property_path = references_paths_lookup[href]
md_path = property_path / 'Documentation.md'
xml_path = property_path / 'DocProperty.xml'
md_path = property_path / "Documentation.md"
xml_path = property_path / "DocProperty.xml"
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}'
github_xml_url = f'https://github.com/buildingSMART/IFC/blob/{xml_url_part}'
github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
github_xml_url = f"https://github.com/buildingSMART/IFC/blob/{xml_url_part}"
with open(xml_path, 'r', encoding='utf-8') as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml')
tags = bs_tree.find_all('docproperty')
with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features="lxml")
tags = bs_tree.find_all("docproperty")
# check for child properties - if they are present parse their data recursively
elements_tag = bs_tree.find('elements')
elements_tag = bs_tree.find("elements")
if elements_tag is not None:
child_tags = elements_tag.find_all('docproperty')
child_tags = elements_tag.find_all("docproperty")
child_tags_dict = dict()
for child_tag in child_tags:
child_tag_href = child_tag['href']
child_tag_href = child_tag["href"]
child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href)
child_tags_dict[child_tag_name] = child_tag_dict
tags.remove(child_tag)
property_dict['children'] = child_tags_dict
print(f'Child nodes found inside property xml. Url: {github_xml_url}')
property_dict["children"] = child_tags_dict
print(f"Child nodes found inside property xml. Url: {github_xml_url}")
if len(tags) != 1:
print(f'WARNING. Found more properties inside property xml, '
f'only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}.')
property_name = tags[0]['name']
print(
f"WARNING. Found more properties inside property xml, "
f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
)
property_name = tags[0]["name"]
if not md_path.is_file():
print(f'WARNING. Property {property_name} is missing documentation.md, '
f'property will be left without description. Url: {github_xml_url}')
print(
f"WARNING. Property {property_name} is missing documentation.md, "
f"property will be left without description. Url: {github_xml_url}"
)
else:
with open(md_path, 'r', encoding='utf-8-sig') as fi:
with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing
html = markdown(fi.read())
description = BeautifulSoup(html, features="lxml").find('p').text
description = description.replace('\n', ' ')
description = description.replace('\u00a0', ' ')
property_dict['description'] = description
description = BeautifulSoup(html, features="lxml").find("p").text
description = description.replace("\n", " ")
description = description.replace("\u00a0", " ")
property_dict["description"] = description
return (property_name, property_dict)
# lookup each property reference and save it's name and description
for property_set_name in property_sets_references:
properties_dict = dict()
@@ -305,28 +311,24 @@ class DocExtractor:
property_name, property_dict = get_property_info_by_href(property_reference)
properties_dict[property_name] = property_dict
property_sets_dict[property_set_name] = {
'properties': properties_dict,
'spec_url': property_sets_spec_urls[property_set_name]
"properties": properties_dict,
"spec_url": property_sets_spec_urls[property_set_name],
}
# export property sets data
with open(BASE_MODULE_PATH / 'schema/ifc2x3_properties.json', 'w', encoding='utf-8') as fo:
print(f'{len(property_sets_dict)} property sets parsed')
json.dump(
property_sets_dict, fo,
sort_keys=True, indent=4
)
with open(BASE_MODULE_PATH / "schema/ifc2x3_properties.json", "w", encoding="utf-8") as fo:
print(f"{len(property_sets_dict)} property sets parsed")
json.dump(property_sets_dict, fo, sort_keys=True, indent=4)
def extract_ifc4(self):
print('Parsing data for Ifc4.0.2.1')
print("Parsing data for Ifc4.0.2.1")
if not IFC4_DOCS_LOCATION.is_dir():
raise Exception(
f'Docs for Ifc4.0.2.1 expected to be in folder "{IFC4_DOCS_LOCATION.resolve()}\\"\n'
'For doc extraction please either setup docs as described above \n'
'or change IFC4_DOCS_LOCATION in doc.py accordingly.'
'You can download docs from the repository: \n'
'https://github.com/buildingSMART/IFC/tree/Ifc4.0.2.1'
"For doc extraction please either setup docs as described above \n"
"or change IFC4_DOCS_LOCATION in doc.py accordingly."
"You can download docs from the repository: \n"
"https://github.com/buildingSMART/IFC/tree/Ifc4.0.2.1"
)
# actually domains in Ifc 4.0 are consistent between website and docs
@@ -337,107 +339,106 @@ class DocExtractor:
self.extract_ifc4_property_sets_site_domains()
self.extract_ifc4_entities()
self.extract_ifc4_property_sets()
def extract_ifc4_property_sets_site_domains(self):
property_sets_domains = dict()
with requests.get(
'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1'
'/HTML/annex/annex-b/alphabeticalorder_psets.htm') as r:
html = BeautifulSoup(r.content, features='lxml')
for a in html.find_all('a', {'class': 'listing-link'}):
href_split = a['href'].split('/')
"https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1"
"/HTML/annex/annex-b/alphabeticalorder_psets.htm"
) as r:
html = BeautifulSoup(r.content, features="lxml")
for a in html.find_all("a", {"class": "listing-link"}):
href_split = a["href"].split("/")
domain = href_split[3]
pset = href_split[5].removesuffix('.htm')
pset = href_split[5].removesuffix(".htm")
property_sets_domains[pset] = domain
with requests.get(
'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/'
'/HTML/annex/annex-b/alphabeticalorder_qsets.htm') as r:
html = BeautifulSoup(r.content, features='lxml')
for a in html.find_all('a', {'class': 'listing-link'}):
href_split = a['href'].split('/')
"https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/"
"/HTML/annex/annex-b/alphabeticalorder_qsets.htm"
) as r:
html = BeautifulSoup(r.content, features="lxml")
for a in html.find_all("a", {"class": "listing-link"}):
href_split = a["href"].split("/")
domain = href_split[3]
pset = href_split[5].removesuffix('.htm')
pset = href_split[5].removesuffix(".htm")
property_sets_domains[pset] = domain
# export property sets data
with open(BASE_MODULE_PATH / 'schema/ifc4_property_sets_site_domains.json', 'w', encoding='utf-8') as fo:
print(f'{len(property_sets_domains)} property sets domains were parsed from the website')
json.dump(
property_sets_domains, fo,
sort_keys=True, indent=4
)
with open(BASE_MODULE_PATH / "schema/ifc4_property_sets_site_domains.json", "w", encoding="utf-8") as fo:
print(f"{len(property_sets_domains)} property sets domains were parsed from the website")
json.dump(property_sets_domains, fo, sort_keys=True, indent=4)
def extract_ifc4_entities(self):
entities_dict = dict()
# search
entities_paths = [filepath
for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/Entities', recursive=True)]
# search
entities_paths = [
filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/Entities", recursive=True)
]
for parse_folder_path in entities_paths:
for entity_path in glob.iglob(f'{parse_folder_path}/**/'):
for entity_path in glob.iglob(f"{parse_folder_path}/**/"):
entity_path = Path(entity_path)
entity_name = entity_path.stem
entities_dict[entity_name] = dict()
# utf-8-sig because of \ufeff occcurs - meaning it's utf bom encoded
md_path = entity_path / 'Documentation.md'
xml_path = entity_path / 'DocEntity.xml'
md_path = entity_path / "Documentation.md"
xml_path = entity_path / "DocEntity.xml"
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}'
github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
with open(md_path, 'r', encoding='utf-8-sig') as fi:
with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing
html = markdown(fi.read())
entity_description = BeautifulSoup(html, features="lxml").find('p').text
entity_description = entity_description.replace('\n', ' ')
entity_description = entity_description.replace('\u00a0', ' ')
entity_description = entity_description.replace('{ .extDef}', '')
entity_description = BeautifulSoup(html, features="lxml").find("p").text
entity_description = entity_description.replace("\n", " ")
entity_description = entity_description.replace("\u00a0", " ")
entity_description = entity_description.replace("{ .extDef}", "")
entity_description = entity_description.strip()
with open(xml_path, 'r', encoding='utf-8') as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml')
with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features="lxml")
entity_attrs = dict()
# temporarily disable MarkupResemblesLocatorWarning
# because BeautifulSoup wrongly assume we confused
# because BeautifulSoup wrongly assume we confused
# html code for filepath and gives warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=MarkupResemblesLocatorWarning)
for html_attr in bs_tree.find_all('docattribute'):
html_description = BeautifulSoup(html_attr.text, features='lxml')
warnings.simplefilter("ignore", category=MarkupResemblesLocatorWarning)
for html_attr in bs_tree.find_all("docattribute"):
html_description = BeautifulSoup(html_attr.text, features="lxml")
attr_description = html_description.get_text()
attr_description = attr_description.replace('\n', ' ')
attr_description = attr_description.replace('\u00a0', ' ')
attr_description = attr_description.replace("\n", " ")
attr_description = attr_description.replace("\u00a0", " ")
# discard part of the description with changelog, notes and examples etc.
# Those notes actually can be useful but we'll need a way to reformat them
# Example:
# https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/ifcsharedbldgelements/lexical/ifcrelconnectspathelements.htm
attr_description = attr_description.split('{ .change-ifc', 1)[0]
attr_description = attr_description.split('{ .note', 1)[0]
attr_description = attr_description.split('{ .examples', 1)[0]
attr_description = attr_description.split('{ .deprecated', 1)[0]
attr_description = attr_description.split('{ .history', 1)[0]
attr_description = attr_description.split("{ .change-ifc", 1)[0]
attr_description = attr_description.split("{ .note", 1)[0]
attr_description = attr_description.split("{ .examples", 1)[0]
attr_description = attr_description.split("{ .deprecated", 1)[0]
attr_description = attr_description.split("{ .history", 1)[0]
attr_description = attr_description.strip()
entity_attrs[html_attr['name']] = attr_description
entity_attrs[html_attr["name"]] = attr_description
if entity_attrs:
entities_dict[entity_name]['attributes'] = entity_attrs
entities_dict[entity_name]["attributes"] = entity_attrs
entities_dict[entity_name]['description'] = entity_description
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/' \
f'{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm'
entities_dict[entity_name]['spec_url'] = spec_url
entities_dict[entity_name]["description"] = entity_description
spec_url = (
"https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML/schema/"
f"{md_path.parents[2].name.lower()}/lexical/{entity_name.lower()}.htm"
)
entities_dict[entity_name]["spec_url"] = spec_url
# entities_dict[entity_name]['github_url'] = github_md_url
# export entities data
with open(BASE_MODULE_PATH / 'schema/ifc4_entities.json', 'w', encoding='utf-8') as fo:
print(f'{len(entities_dict)} entities parsed')
json.dump(
entities_dict, fo,
sort_keys=True, indent=4
)
with open(BASE_MODULE_PATH / "schema/ifc4_entities.json", "w", encoding="utf-8") as fo:
print(f"{len(entities_dict)} entities parsed")
json.dump(entities_dict, fo, sort_keys=True, indent=4)
def extract_ifc4_property_sets(self):
# function parses both property and quantity sets
@@ -446,105 +447,114 @@ class DocExtractor:
property_sets_spec_urls = dict()
# extract lists of properties and theirs references for each property set
parsed_paths = [filepath for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/PropertySets', recursive=True)]
parsed_paths += [filepath for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Sections/**/QuantitySets', recursive=True)]
parsed_paths = [
filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/PropertySets", recursive=True)
]
parsed_paths += [
filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Sections/**/QuantitySets", recursive=True)
]
# prepare property sets domains from the website we extracted earlier
with open(BASE_MODULE_PATH / 'schema/ifc4_property_sets_site_domains.json', 'r') as fi:
with open(BASE_MODULE_PATH / "schema/ifc4_property_sets_site_domains.json", "r") as fi:
property_sets_site_domains = json.load(fi)
psets_test = set()
for parse_folder_path in parsed_paths:
for property_set_path in glob.iglob(f'{parse_folder_path}/**/'):
for property_set_path in glob.iglob(f"{parse_folder_path}/**/"):
property_set_path = Path(property_set_path)
property_set_name = property_set_path.stem
property_references = list()
property_quantity = property_set_path.parents[0].name == 'QuantitySets'
xml_path = property_set_path / ('DocQuantitySet.xml' if property_quantity else 'DocPropertySet.xml')
property_quantity = property_set_path.parents[0].name == "QuantitySets"
xml_path = property_set_path / ("DocQuantitySet.xml" if property_quantity else "DocPropertySet.xml")
with open(xml_path, 'r', encoding='utf-8') as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml')
for html_attr in bs_tree.find_all('docquantity' if property_quantity else 'docproperty'):
property_references.append(html_attr['href'])
with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features="lxml")
for html_attr in bs_tree.find_all("docquantity" if property_quantity else "docproperty"):
property_references.append(html_attr["href"])
property_sets_references[property_set_name] = property_references
if property_set_name.lower() not in property_sets_site_domains:
print(f"WARNING. {property_set_name} was not found on the spec website, "
"this property set won't have any spec_url in schema.")
print(
f"WARNING. {property_set_name} was not found on the spec website, "
"this property set won't have any spec_url in schema."
)
else:
property_set_domain = property_sets_site_domains.get(property_set_name.lower(), '')
spec_url = 'https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML' \
f'/schema/{property_set_domain}' \
f'/{"qset" if property_quantity else "pset"}' \
f'/{property_set_name.lower()}.htm'
property_set_domain = property_sets_site_domains.get(property_set_name.lower(), "")
spec_url = (
"https://standards.buildingsmart.org/IFC/RELEASE/IFC4/ADD2_TC1/HTML"
f"/schema/{property_set_domain}"
f'/{"qset" if property_quantity else "pset"}'
f"/{property_set_name.lower()}.htm"
)
property_sets_spec_urls[property_set_name] = spec_url
# setup references look up tables to convert property hrefs to actual data paths
references_paths_lookup = dict()
parsed_paths = [filepath
for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Properties/*/*', recursive=False)]
parsed_paths += [filepath
for filepath in glob.iglob(f'{IFC4_DOCS_LOCATION}/Quantities/*/*', recursive=False)]
parsed_paths = [filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Properties/*/*", recursive=False)]
parsed_paths += [filepath for filepath in glob.iglob(f"{IFC4_DOCS_LOCATION}/Quantities/*/*", recursive=False)]
for parsed_path in parsed_paths:
parsed_path = Path(parsed_path)
# all references omit "$" character, I've checked it on 4_0
# need to check it if moving to next IFC version
# btw no reason to check if all references were used in properties
# because there are also child properties
property_reference = parsed_path.name.replace('$', '')
property_reference = parsed_path.name.replace("$", "")
references_paths_lookup[property_reference] = parsed_path
# setup a function because we'll need to check child properties recusively
def get_property_info_by_href(href):
property_dict = dict()
property_path = references_paths_lookup[href]
property_quantity = property_path.parents[1].name == 'Quantities'
property_quantity = property_path.parents[1].name == "Quantities"
md_path = property_path / 'Documentation.md'
xml_path = property_path / ('DocQuantity.xml' if property_quantity else 'DocProperty.xml')
md_path = property_path / "Documentation.md"
xml_path = property_path / ("DocQuantity.xml" if property_quantity else "DocProperty.xml")
md_url_part = urllib.parse.quote(str(md_path.relative_to(Path(__file__).parent).as_posix()))
github_md_url = f'https://github.com/buildingSMART/IFC/blob/{md_url_part}'
github_md_url = f"https://github.com/buildingSMART/IFC/blob/{md_url_part}"
xml_url_part = urllib.parse.quote(str(xml_path.relative_to(Path(__file__).parent).as_posix()))
github_xml_url = f'https://github.com/buildingSMART/IFC/blob/{xml_url_part}'
github_xml_url = f"https://github.com/buildingSMART/IFC/blob/{xml_url_part}"
with open(xml_path, 'r', encoding='utf-8') as fi:
bs_tree = BeautifulSoup(fi.read(), features='lxml')
tags = bs_tree.find_all('docquantity' if property_quantity else 'docproperty')
with open(xml_path, "r", encoding="utf-8") as fi:
bs_tree = BeautifulSoup(fi.read(), features="lxml")
tags = bs_tree.find_all("docquantity" if property_quantity else "docproperty")
# check for child properties - if they are present parse their data recursively
elements_tag = bs_tree.find('elements')
elements_tag = bs_tree.find("elements")
if elements_tag is not None:
child_tags = elements_tag.find_all('docquantity' if property_quantity else 'docproperty')
child_tags = elements_tag.find_all("docquantity" if property_quantity else "docproperty")
child_tags_dict = dict()
for child_tag in child_tags:
child_tag_href = child_tag['href']
child_tag_href = child_tag["href"]
child_tag_name, child_tag_dict = get_property_info_by_href(child_tag_href)
child_tags_dict[child_tag_name] = child_tag_dict
tags.remove(child_tag)
property_dict['children'] = child_tags_dict
print(f'Child nodes found inside property xml. Url: {github_xml_url}')
property_dict["children"] = child_tags_dict
print(f"Child nodes found inside property xml. Url: {github_xml_url}")
if len(tags) != 1:
print(f'WARNING. Found more properties inside property xml, '
f'only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}.')
property_name = tags[0]['name']
print(
f"WARNING. Found more properties inside property xml, "
f"only first one were parsed (number of properties: {len(tags)}). Url: {github_xml_url}."
)
property_name = tags[0]["name"]
if not md_path.is_file():
print(f'WARNING. Property {property_name} is missing documentation.md, property will be left without description. '
f'Url: {github_xml_url}')
print(
f"WARNING. Property {property_name} is missing documentation.md, property will be left without description. "
f"Url: {github_xml_url}"
)
else:
with open(md_path, 'r', encoding='utf-8-sig') as fi:
with open(md_path, "r", encoding="utf-8-sig") as fi:
# convert markdown to html for easier parsing
html = markdown(fi.read())
description = BeautifulSoup(html, features="lxml").find('p').text
description = description.replace('\n', ' ')
description = description.replace('\u00a0', ' ')
property_dict['description'] = description
description = BeautifulSoup(html, features="lxml").find("p").text
description = description.replace("\n", " ")
description = description.replace("\u00a0", " ")
property_dict["description"] = description
return (property_name, property_dict)
# lookup each property reference and save it's name and description
@@ -553,47 +563,38 @@ class DocExtractor:
for property_reference in property_sets_references[property_set_name]:
property_name, property_dict = get_property_info_by_href(property_reference)
properties_dict[property_name] = property_dict
property_sets_dict[property_set_name] = {
'properties': properties_dict
}
property_sets_dict[property_set_name] = {"properties": properties_dict}
if property_set_name in property_sets_spec_urls:
spec_url = property_sets_spec_urls[property_set_name]
property_sets_dict[property_set_name]['spec_url'] = spec_url
property_sets_dict[property_set_name]["spec_url"] = spec_url
# export property sets data
with open(BASE_MODULE_PATH / 'schema/ifc4_properties.json', 'w', encoding='utf-8') as fo:
print(f'{len(property_sets_dict)} property sets parsed')
json.dump(
property_sets_dict, fo,
sort_keys=True, indent=4
)
with open(BASE_MODULE_PATH / "schema/ifc4_properties.json", "w", encoding="utf-8") as fo:
print(f"{len(property_sets_dict)} property sets parsed")
json.dump(property_sets_dict, fo, sort_keys=True, indent=4)
def run_doc_api_examples():
print('Entities:')
print(get_entity_doc('IFC2X3', 'IfcActionRequest'))
print(get_entity_doc('IFC4', 'IfcActionRequest'))
print("Entities:")
print(get_entity_doc("IFC2X3", "IfcActionRequest"))
print(get_entity_doc("IFC4", "IfcActionRequest"))
print('Entity attributes:')
print(get_attribute_doc('IFC2X3', 'IfcActionRequest', 'RequestID'))
print(get_attribute_doc('IFC4', 'IfcActionRequest', 'PredefinedType'))
print("Entity attributes:")
print(get_attribute_doc("IFC2X3", "IfcActionRequest", "RequestID"))
print(get_attribute_doc("IFC4", "IfcActionRequest", "PredefinedType"))
print('Propety sets:')
print(get_property_set_doc('IFC2X3', 'Pset_ZoneCommon'))
print(get_property_set_doc('IFC4', 'Pset_ZoneCommon'))
print("Propety sets:")
print(get_property_set_doc("IFC2X3", "Pset_ZoneCommon"))
print(get_property_set_doc("IFC4", "Pset_ZoneCommon"))
print('Propety sets attributes:')
print(get_property_doc('IFC2X3', 'Pset_ZoneCommon', 'Category'))
print(get_property_doc('IFC4', 'Pset_ZoneCommon', 'NetPlannedArea'))
print("Propety sets attributes:")
print(get_property_doc("IFC2X3", "Pset_ZoneCommon", "Category"))
print(get_property_doc("IFC4", "Pset_ZoneCommon", "NetPlannedArea"))
if __name__ == '__main__':
if __name__ == "__main__":
extractor = DocExtractor()
extractor.extract_ifc2x3()
extractor.extract_ifc4()
# run_doc_api_examples()