From e893552f24732f12bfe1c302472f04b8dda630df Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Thu, 7 May 2026 14:43:26 +0200 Subject: [PATCH] Fixes to plug-in loading in and outside of pyodide --- .github/workflows/build_pyodide.yml | 9 + cmake/utilities.cmake | 6 + pyodide/split_pyodide_ifcopenshell_wheel.py | 293 ++++++++++++++++++ ...opencascade_geometry_ifc_writer_plugin.cpp | 10 +- src/ifcgeom/kernel_plugin.cpp | 10 +- src/ifcgeom/kernel_registry.cpp | 97 ++++-- src/ifcgeom/kernels/cgal/plugin.cpp | 6 +- src/ifcgeom/mapping_plugin.cpp | 10 +- src/ifcgeom/tree_plugin.cpp | 10 +- .../ifcopenshell/__init__.py | 15 + .../ifcopenshell/ifcopenshell_wrapper.pyi | 3 + src/ifcparse/CMakeLists.txt | 1 + src/ifcparse/schema.cpp | 12 +- src/ifcwrap/IfcPython.i | 26 ++ src/plugin/plugin.cpp | 139 ++++++++- src/pyodide/demo-app/context.py | 6 + src/pyodide/demo-app/index.html | 9 +- .../document_serializer_plugin.cpp | 13 +- .../geometry_serializer_plugin.cpp | 13 +- 19 files changed, 642 insertions(+), 46 deletions(-) create mode 100644 pyodide/split_pyodide_ifcopenshell_wheel.py diff --git a/.github/workflows/build_pyodide.yml b/.github/workflows/build_pyodide.yml index a5e80a32df..2d0751908e 100644 --- a/.github/workflows/build_pyodide.yml +++ b/.github/workflows/build_pyodide.yml @@ -40,6 +40,14 @@ jobs: NEW_FILE=`echo $FILE | sed "s/-/+${GITHUB_SHA:0:7}-/2"` mv $FILE $NEW_FILE + - name: Split packages + run: | + VERSION=v`cat ./IfcOpenShell/VERSION` + mkdir -p dist-modular + python ./IfcOpenShell/pyodide/split_pyodide_ifcopenshell_wheel.py dist/ifcopenshell-*.whl ./dist-modular + cd dist-modular + zip -r -qq ifcopenshell-modular-${VERSION}-${GITHUB_SHA:0:7}-pyodide.zip *.whl + - name: Upload Build Logs if: always() uses: actions/upload-artifact@v7 @@ -86,3 +94,4 @@ jobs: - name: Upload .zip archives to S3 run: | aws s3 cp dist s3://ifcopenshell-builds/ --recursive --exclude "*" --include "*.whl" + aws s3 cp dist-modular s3://ifcopenshell-builds/ --recursive --exclude "*" --include "*.zip" diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index 78be5271f6..71183daa84 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -41,7 +41,13 @@ macro(SET_INSTALL_RPATHS _target _paths) set_target_properties(${_target} PROPERTIES INSTALL_RPATH "${${_target}_rpaths}") endmacro() +function(ifcopenshell_plugin_target TARGET) + set_target_properties(${TARGET} PROPERTIES PREFIX "") +endfunction() + function(ifcopenshell_wasm_plugin_link_options TARGET REGISTRATION_SYMBOL) + ifcopenshell_plugin_target(${TARGET}) + if(NOT WASM_BUILD) return() endif() diff --git a/pyodide/split_pyodide_ifcopenshell_wheel.py b/pyodide/split_pyodide_ifcopenshell_wheel.py new file mode 100644 index 0000000000..d6c9e8194f --- /dev/null +++ b/pyodide/split_pyodide_ifcopenshell_wheel.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Split optional IfcOpenShell Pyodide payloads into separate wheels.""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import hashlib +import io +import os +import re +import sys +import time +import zipfile +from email.parser import Parser +from pathlib import Path + + +MAIN_SHARED_OBJECT_RE = re.compile(r"(^|/)_ifcopenshell_wrapper(?:\.|$)") +PURE_PYTHON_PACKAGE_NAME = "ifcopenshell-pure-python" +PURE_PYTHON_PREFIXES = ( + "ifcopenshell/api/", + "ifcopenshell/express/", + "ifcopenshell/mvd/", + "ifcopenshell/simple_spf/", +) + + +def wheel_parts(path: Path) -> tuple[str, str, str, str, str]: + if path.suffix != ".whl": + raise ValueError(f"not a wheel: {path}") + stem = path.name[:-4] + left, py_tag, abi_tag, platform_tag = stem.rsplit("-", 3) + dist, version = left.rsplit("-", 1) + return dist, version, py_tag, abi_tag, platform_tag + + +def safe_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower().strip("-") + + +def wheel_escape(value: str) -> str: + return re.sub(r"[^\w\d.]+", "_", value, flags=re.UNICODE) + + +def dist_info_dir(name: str, version: str) -> str: + return f"{wheel_escape(name)}-{wheel_escape(version)}.dist-info" + + +def sha256_record_value(data: bytes) -> str: + digest = hashlib.sha256(data).digest() + return "sha256=" + base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + + +def make_info(name: str, *, source: zipfile.ZipInfo | None = None, mode: int | None = None) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name) + if source is not None: + info.date_time = source.date_time + info.external_attr = source.external_attr + info.comment = source.comment + info.create_system = source.create_system + else: + info.date_time = time.localtime(time.time())[:6] + info.external_attr = ((mode if mode is not None else 0o644) & 0xFFFF) << 16 + info.create_system = 3 + info.compress_type = zipfile.ZIP_DEFLATED + return info + + +def write_record(zf: zipfile.ZipFile, entries: dict[str, bytes | None], record_name: str) -> None: + rows: list[list[str]] = [] + for name in sorted(entries): + data = entries[name] + if name == record_name: + rows.append([name, "", ""]) + elif data is None: + raise ValueError(f"missing bytes for RECORD entry {name}") + else: + rows.append([name, sha256_record_value(data), str(len(data))]) + + buf = io.StringIO(newline="") + writer = csv.writer(buf, lineterminator="\n") + writer.writerows(rows) + zf.writestr(make_info(record_name), buf.getvalue().encode("utf-8")) + + +def read_original_metadata(zf: zipfile.ZipFile) -> tuple[str, str, str]: + metadata_names = [n for n in zf.namelist() if n.endswith(".dist-info/METADATA")] + wheel_names = [n for n in zf.namelist() if n.endswith(".dist-info/WHEEL")] + record_names = [n for n in zf.namelist() if n.endswith(".dist-info/RECORD")] + if len(metadata_names) != 1 or len(wheel_names) != 1 or len(record_names) != 1: + raise ValueError("expected exactly one METADATA, WHEEL, and RECORD in the source wheel") + return metadata_names[0], wheel_names[0], record_names[0] + + +def shared_package_name(so_path: str) -> str: + stem = Path(so_path).name.removesuffix(".so") + stem = re.sub(r"[^A-Za-z0-9]+", "-", stem).strip("-") + return safe_name(stem) + + +def is_pure_python_split_path(path: str) -> bool: + return any(path.startswith(prefix) for prefix in PURE_PYTHON_PREFIXES) + + +def build_wheel( + output_dir: Path, + package_name: str, + version: str, + tag: str, + root_is_purelib: bool, + summary: str, + payloads: list[tuple[zipfile.ZipInfo, bytes]], + license_files: dict[str, bytes], +) -> Path: + di = dist_info_dir(package_name, version) + wheel_name = f"{wheel_escape(package_name)}-{wheel_escape(version)}-{tag}.whl" + out = output_dir / wheel_name + record_name = f"{di}/RECORD" + entries: dict[str, bytes | None] = {} + + metadata = ( + "Metadata-Version: 2.4\n" + f"Name: {package_name}\n" + f"Version: {version}\n" + f"Summary: {summary}\n" + "License-File: COPYING\n" + "License-File: COPYING.LESSER\n" + "\n" + ).encode("utf-8") + wheel = ( + "Wheel-Version: 1.0\n" + "Generator: split_pyodide_ifcopenshell_wheel.py\n" + f"Root-Is-Purelib: {str(root_is_purelib).lower()}\n" + f"Tag: {tag}\n" + "\n" + ).encode("utf-8") + + with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as zf: + for info, data in payloads: + zf.writestr(make_info(info.filename, source=info), data) + entries[info.filename] = data + + metadata_name = f"{di}/METADATA" + wheel_meta_name = f"{di}/WHEEL" + zf.writestr(make_info(metadata_name), metadata) + zf.writestr(make_info(wheel_meta_name), wheel) + entries[metadata_name] = metadata + entries[wheel_meta_name] = wheel + + for basename, data in license_files.items(): + name = f"{di}/licenses/{basename}" + zf.writestr(make_info(name), data) + entries[name] = data + + entries[record_name] = None + write_record(zf, entries, record_name) + return out + + +def rewrite_main_wheel(source: Path, target: Path, split_paths: set[str]) -> None: + with zipfile.ZipFile(source) as zin, zipfile.ZipFile( + target, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as zout: + _, _, record_name = read_original_metadata(zin) + entries: dict[str, bytes | None] = {} + for info in zin.infolist(): + if info.filename in split_paths or info.filename == record_name: + continue + data = zin.read(info.filename) + zout.writestr(make_info(info.filename, source=info), data) + entries[info.filename] = data + entries[record_name] = None + write_record(zout, entries, record_name) + + +def verify_wheel(path: Path) -> None: + with zipfile.ZipFile(path) as zf: + zf.testzip() + metadata_name, wheel_name, record_name = read_original_metadata(zf) + Parser().parsestr(zf.read(metadata_name).decode("utf-8")) + wheel_text = zf.read(wheel_name).decode("utf-8") + if "Wheel-Version:" not in wheel_text or "Tag:" not in wheel_text: + raise ValueError(f"invalid WHEEL metadata in {path}") + + record_rows = list(csv.reader(io.StringIO(zf.read(record_name).decode("utf-8")))) + names = {row[0] for row in record_rows} + missing = set(zf.namelist()) - names + if missing: + raise ValueError(f"{path} RECORD is missing entries: {sorted(missing)[:5]}") + for name, digest, size in record_rows: + if name == record_name: + continue + data = zf.read(name) + if digest != sha256_record_value(data) or size != str(len(data)): + raise ValueError(f"{path} RECORD mismatch for {name}") + + +def split_wheel(wheel_path: Path, output_dir: Path) -> None: + wheel_path = wheel_path.expanduser().resolve() + if not wheel_path.exists(): + raise FileNotFoundError(wheel_path) + + output_dir = output_dir.expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + main_wheel_path = output_dir / wheel_path.name + if main_wheel_path.resolve(strict=False) == wheel_path: + raise ValueError("output directory must not point to the input wheel location") + + _, version, py_tag, abi_tag, platform_tag = wheel_parts(wheel_path) + binary_tag = f"{py_tag}-{abi_tag}-{platform_tag}" + pure_tag = "py3-none-any" + + with zipfile.ZipFile(wheel_path) as zf: + file_infos = [info for info in zf.infolist() if not info.is_dir()] + so_infos = [info for info in file_infos if info.filename.endswith(".so")] + split_so_infos = [info for info in so_infos if not MAIN_SHARED_OBJECT_RE.search(Path(info.filename).name)] + pure_python_infos = [info for info in file_infos if is_pure_python_split_path(info.filename)] + if not split_so_infos and not pure_python_infos: + raise RuntimeError("no secondary .so files or pure Python subpackages found to split") + + license_files = { + Path(info.filename).name: zf.read(info.filename) + for info in file_infos + if ".dist-info/licenses/" in info.filename + } + split_so_payloads = [(info, zf.read(info.filename)) for info in split_so_infos] + pure_python_payloads = [(info, zf.read(info.filename)) for info in pure_python_infos] + + created_wheels: list[Path] = [] + for info, data in split_so_payloads: + package_name = shared_package_name(info.filename) + created_wheels.append( + build_wheel( + output_dir, + package_name, + version, + binary_tag, + False, + f"Pyodide shared library split from IfcOpenShell ({Path(info.filename).name}).", + [(info, data)], + license_files, + ) + ) + + if pure_python_payloads: + created_wheels.append( + build_wheel( + output_dir, + PURE_PYTHON_PACKAGE_NAME, + version, + pure_tag, + True, + "Pure Python subpackages split from IfcOpenShell.", + pure_python_payloads, + license_files, + ) + ) + + temp_main_wheel = output_dir / f".{wheel_path.name}.tmp" + try: + rewrite_main_wheel( + wheel_path, + temp_main_wheel, + {info.filename for info, _ in split_so_payloads + pure_python_payloads}, + ) + verify_wheel(temp_main_wheel) + for created in created_wheels: + verify_wheel(created) + os.replace(temp_main_wheel, main_wheel_path) + finally: + if temp_main_wheel.exists(): + temp_main_wheel.unlink() + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Extract optional IfcOpenShell Pyodide payloads into separate wheel artifacts." + ) + parser.add_argument("wheel", help="IfcOpenShell Pyodide wheel to split") + parser.add_argument("output_dir", help="Directory for generated wheels") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + split_wheel(Path(args.wheel), Path(args.output_dir)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ifcgeom/Serialization/opencascade_geometry_ifc_writer_plugin.cpp b/src/ifcgeom/Serialization/opencascade_geometry_ifc_writer_plugin.cpp index 25846af5f3..d4964d5758 100644 --- a/src/ifcgeom/Serialization/opencascade_geometry_ifc_writer_plugin.cpp +++ b/src/ifcgeom/Serialization/opencascade_geometry_ifc_writer_plugin.cpp @@ -21,6 +21,12 @@ #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { constexpr const char* opencascade_geometry_ifc_writer_plugin_prefix = "geometry.serialization."; } @@ -43,7 +49,7 @@ std::filesystem::path IfcGeom::opencascade_geometry_ifc_writer_plugin_directory( void IfcGeom::load_opencascade_geometry_ifc_writer_plugins(opencascade_geometry_ifc_writer_registry& registry) { ifcopenshell::plugin::manager manager; - manager.add_search_path(opencascade_geometry_ifc_writer_plugin_directory()); + ifcopenshell::plugin::add_search_paths_or_default(manager, &opencascade_geometry_ifc_writer_plugin_directory); for (const auto& path : manager.discover(opencascade_geometry_ifc_writer_plugin_prefix)) { auto module = manager.load(path); @@ -58,7 +64,7 @@ void IfcGeom::load_opencascade_geometry_ifc_writer_plugins(opencascade_geometry_ bool IfcGeom::load_opencascade_geometry_ifc_writer_plugin(opencascade_geometry_ifc_writer_registry& registry, const std::string& schema_name) { ifcopenshell::plugin::manager manager; - manager.add_search_path(opencascade_geometry_ifc_writer_plugin_directory()); + ifcopenshell::plugin::add_search_paths_or_default(manager, &opencascade_geometry_ifc_writer_plugin_directory); const auto expected_schema = boost::to_upper_copy(schema_name); const auto basename = std::string(opencascade_geometry_ifc_writer_plugin_prefix) + boost::to_lower_copy(schema_name); diff --git a/src/ifcgeom/kernel_plugin.cpp b/src/ifcgeom/kernel_plugin.cpp index d216e4e467..53128a9bc1 100644 --- a/src/ifcgeom/kernel_plugin.cpp +++ b/src/ifcgeom/kernel_plugin.cpp @@ -24,6 +24,12 @@ #include #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { constexpr const char* kernel_plugin_prefix = "geometry.kernel."; @@ -51,7 +57,7 @@ std::filesystem::path ifcopenshell::geometry::kernels::kernel_plugin_directory() void ifcopenshell::geometry::kernels::load_kernel_plugins(kernel_registry& registry) { plugin::manager manager; - manager.add_search_path(kernel_plugin_directory()); + plugin::add_search_paths_or_default(manager, &kernel_plugin_directory); for (const auto& path : manager.discover(kernel_plugin_prefix)) { auto module = manager.load(path); @@ -66,7 +72,7 @@ void ifcopenshell::geometry::kernels::load_kernel_plugins(kernel_registry& regis bool ifcopenshell::geometry::kernels::load_kernel_plugin(kernel_registry& registry, const std::string& backend_id) { plugin::manager manager; - manager.add_search_path(kernel_plugin_directory()); + plugin::add_search_paths_or_default(manager, &kernel_plugin_directory); const auto plugin_name = kernel_plugin_name(backend_id); const auto basename = std::string(kernel_plugin_prefix) + plugin_name; diff --git a/src/ifcgeom/kernel_registry.cpp b/src/ifcgeom/kernel_registry.cpp index 695061aa25..962710da43 100644 --- a/src/ifcgeom/kernel_registry.cpp +++ b/src/ifcgeom/kernel_registry.cpp @@ -27,13 +27,85 @@ #include #include +#include #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { + constexpr const char* kernel_plugin_prefix = "geometry.kernel."; + std::string kernel_key(const std::string& backend_id) { return boost::to_lower_copy(backend_id); } + bool is_prefix(const std::string& text, const std::string& prefix) { + return !prefix.empty() && text.rfind(prefix, 0) == 0; + } + + void register_kernel_module(ifcopenshell::geometry::kernels::kernel_registry& registry, const ifcopenshell::plugin::module& module) { + auto register_plugin = module.get_alias( + ifcopenshell::geometry::kernels::kernel_plugin_registration_symbol()); + register_plugin(registry, module); + } + + struct kernel_match { + std::string backend_id; + ifcopenshell::plugin::module module; + bool has_module = false; + }; + + void consider_kernel_info(kernel_match& match, const std::string& geometry_library_lower, const ifcopenshell::geometry::kernels::kernel_info& info) { + const auto backend_id = kernel_key(info.backend_id); + if (is_prefix(geometry_library_lower, backend_id) && backend_id.size() > match.backend_id.size()) { + match.backend_id = backend_id; + match.has_module = false; + } + } + + kernel_match find_kernel_match(ifcopenshell::geometry::kernels::kernel_registry& registry, const std::string& geometry_library_lower) { + kernel_match match; + + for (const auto& info : registry.kernels()) { + consider_kernel_info(match, geometry_library_lower, info); + } + + ifcopenshell::plugin::manager manager; + ifcopenshell::plugin::add_search_paths_or_default(manager, &ifcopenshell::geometry::kernels::kernel_plugin_directory); + for (const auto& path : manager.discover(kernel_plugin_prefix)) { + ifcopenshell::plugin::module module; + try { + module = manager.load(path); + } catch (const std::exception& e) { + std::cerr << "[ifcopenshell.plugin] skip kernel plugin " << path << ": " << e.what() << std::endl; + continue; + } + if (module.meta().kind_ != ifcopenshell::plugin::kind::kernel) { + continue; + } + + ifcopenshell::geometry::kernels::kernel_registry plugin_registry; + register_kernel_module(plugin_registry, module); + for (const auto& info : plugin_registry.kernels()) { + const auto backend_id = kernel_key(info.backend_id); + if (is_prefix(geometry_library_lower, backend_id) && backend_id.size() > match.backend_id.size()) { + match.backend_id = backend_id; + match.module = module; + match.has_module = true; + } + } + } + + if (!match.backend_id.empty() && !registry.has(match.backend_id) && match.has_module) { + register_kernel_module(registry, match.module); + } + + return match; + } } void ifcopenshell::geometry::kernels::kernel_registry::bind(const kernel_info& info, create_fn create, const plugin::module& module) { @@ -90,30 +162,11 @@ std::unique_ptr ifcopenshell::g throw ifcopenshell::exception("Invalid hybrid kernel " + geometry_library); } - std::vector candidates; - candidates.push_back(geometry_library_lower); - for (auto pos = geometry_library_lower.find('-'); pos != std::string::npos; pos = geometry_library_lower.find('-', pos + 1)) { - candidates.push_back(geometry_library_lower.substr(0, pos)); - } - std::sort(candidates.begin(), candidates.end(), [](const auto& a, const auto& b) { - return a.size() > b.size(); - }); - for (const auto& candidate : candidates) { - if (!registry.has(candidate) && load_kernel_plugin(registry, candidate)) { - break; - } - } - - std::string matched_backend_id; - for (const auto& info : registry.kernels()) { - const auto backend_id = kernel_key(info.backend_id); - if (geometry_library_lower.rfind(backend_id, 0) == 0 && backend_id.size() > matched_backend_id.size()) { - matched_backend_id = backend_id; - } - } + const auto match = find_kernel_match(registry, geometry_library_lower); + const auto& matched_backend_id = match.backend_id; if (matched_backend_id.empty()) { - throw ifcopenshell::exception("Invalid hybrid kernel " + geometry_library); + throw ifcopenshell::exception("Invalid hybrid kernel; no match for prefix of " + geometry_library_lower); } kernels.push_back(registry.create(matched_backend_id, file, settings)); diff --git a/src/ifcgeom/kernels/cgal/plugin.cpp b/src/ifcgeom/kernels/cgal/plugin.cpp index 53d197c269..1e440100b1 100644 --- a/src/ifcgeom/kernels/cgal/plugin.cpp +++ b/src/ifcgeom/kernels/cgal/plugin.cpp @@ -22,10 +22,14 @@ #include +#ifdef IFOPSH_SIMPLE_KERNEL +#define cgal_plugin cgalsimple_plugin +#endif + namespace ifcopenshell { namespace geometry { namespace kernels { - namespace cgal_plugin { + namespace cgal_plugin { #ifdef IFOPSH_SIMPLE_KERNEL constexpr const char* plugin_name = "cgalsimple"; diff --git a/src/ifcgeom/mapping_plugin.cpp b/src/ifcgeom/mapping_plugin.cpp index ffa994352d..49736ce087 100644 --- a/src/ifcgeom/mapping_plugin.cpp +++ b/src/ifcgeom/mapping_plugin.cpp @@ -21,6 +21,12 @@ #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { constexpr const char* mapping_plugin_prefix = "geometry.mapping."; } @@ -43,7 +49,7 @@ std::filesystem::path ifcopenshell::geometry::impl::mapping_plugin_directory() { void ifcopenshell::geometry::impl::load_mapping_plugins(mapping_registry& registry) { plugin::manager manager; - manager.add_search_path(mapping_plugin_directory()); + plugin::add_search_paths_or_default(manager, &mapping_plugin_directory); for (const auto& path : manager.discover(mapping_plugin_prefix)) { auto module = manager.load(path); @@ -58,7 +64,7 @@ void ifcopenshell::geometry::impl::load_mapping_plugins(mapping_registry& regist bool ifcopenshell::geometry::impl::load_mapping_plugin(mapping_registry& registry, const std::string& schema_name) { plugin::manager manager; - manager.add_search_path(mapping_plugin_directory()); + plugin::add_search_paths_or_default(manager, &mapping_plugin_directory); const auto expected_schema = boost::to_upper_copy(schema_name); const auto basename = std::string(mapping_plugin_prefix) + boost::to_lower_copy(schema_name); diff --git a/src/ifcgeom/tree_plugin.cpp b/src/ifcgeom/tree_plugin.cpp index 3e715d289e..c5ee31a58a 100644 --- a/src/ifcgeom/tree_plugin.cpp +++ b/src/ifcgeom/tree_plugin.cpp @@ -21,6 +21,12 @@ #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { constexpr const char* tree_plugin_prefix = "geometry.tree."; } @@ -42,7 +48,7 @@ std::filesystem::path ifcopenshell::geometry::trees::tree_plugin_directory() { void ifcopenshell::geometry::trees::load_tree_plugins(tree_registry& registry) { plugin::manager manager; - manager.add_search_path(tree_plugin_directory()); + plugin::add_search_paths_or_default(manager, &tree_plugin_directory); for (const auto& path : manager.discover(tree_plugin_prefix)) { auto module = manager.load(path); @@ -57,7 +63,7 @@ void ifcopenshell::geometry::trees::load_tree_plugins(tree_registry& registry) { bool ifcopenshell::geometry::trees::load_tree_plugin(tree_registry& registry, const std::string& backend_id) { plugin::manager manager; - manager.add_search_path(tree_plugin_directory()); + plugin::add_search_paths_or_default(manager, &tree_plugin_directory); const auto plugin_name = boost::to_lower_copy(backend_id); const auto basename = std::string(tree_plugin_prefix) + plugin_name; diff --git a/src/ifcopenshell-python/ifcopenshell/__init__.py b/src/ifcopenshell-python/ifcopenshell/__init__.py index f3e03b633f..5c46e2d0d0 100644 --- a/src/ifcopenshell-python/ifcopenshell/__init__.py +++ b/src/ifcopenshell-python/ifcopenshell/__init__.py @@ -101,8 +101,11 @@ __all__ = [ "entity_instance", "file", "guid", + "get_plugin_search_paths", "ifcopenshell_wrapper", "rocksdb_lazy_instance", + "clear_plugin_search_paths", + "set_plugin_search_paths", "sqlite", "sqlite_entity", "stream", @@ -116,6 +119,18 @@ except: pass +def set_plugin_search_paths(paths: Sequence[Union[os.PathLike, str]]) -> None: + ifcopenshell_wrapper.set_plugin_search_paths([os.fspath(path) for path in paths]) + + +def get_plugin_search_paths() -> tuple[str, ...]: + return tuple(ifcopenshell_wrapper.get_plugin_search_paths()) + + +def clear_plugin_search_paths() -> None: + ifcopenshell_wrapper.clear_plugin_search_paths() + + class Error(Exception): """Error used when a generic problem occurs""" diff --git a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi index 28caafc262..0d0b7a202a 100644 --- a/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi +++ b/src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi @@ -1696,6 +1696,7 @@ class type_declaration(declaration): class uninitialized_tag: ... def arrange_polygons(polygons): ... +def clear_plugin_search_paths() -> None: ... def clear_schemas(): ... def construct_iterator(geometry_library, settings, file, num_threads): ... def construct_iterator_with_include_exclude(geometry_library, settings, file, elems, include, num_threads): ... @@ -1707,6 +1708,7 @@ def create_epeck(*args): ... def create_shape(*args): ... def flatten(deep): ... def get_feature(x): ... +def get_plugin_search_paths() -> tuple[str, ...]: ... def get_info_cpp(v, include_identifier=True): ... def get_log(): ... def guess_file_type(fn): ... @@ -1726,6 +1728,7 @@ def schema_by_name(arg1: str) -> schema_definition: ... def schema_names() -> tuple[str, ...]: ... def serialise(schema_name, shape_str, advanced=True): ... def set_feature(x, v): ... +def set_plugin_search_paths(paths) -> None: ... def set_log_format_json(): ... def set_log_format_text(): ... def stream_from_string(data): ... diff --git a/src/ifcparse/CMakeLists.txt b/src/ifcparse/CMakeLists.txt index 1b6254d2da..7d3c4db3d1 100644 --- a/src/ifcparse/CMakeLists.txt +++ b/src/ifcparse/CMakeLists.txt @@ -89,6 +89,7 @@ foreach(schema ${SCHEMA_VERSIONS}) set_target_properties(${SCHEMA_PLUGIN_TARGET} PROPERTIES OUTPUT_NAME "ifcopenshell.parse.schema.ifc${schema}" ) + ifcopenshell_plugin_target(${SCHEMA_PLUGIN_TARGET}) if(WASM_BUILD) target_link_options(${SCHEMA_PLUGIN_TARGET} PRIVATE "SHELL:-s SIDE_MODULE=1" "-O1") diff --git a/src/ifcparse/schema.cpp b/src/ifcparse/schema.cpp index 0e7d21bd69..9b56e4e7d2 100644 --- a/src/ifcparse/schema.cpp +++ b/src/ifcparse/schema.cpp @@ -27,6 +27,12 @@ #include "schemas/Header_section_schema.h" +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + bool ifcopenshell::declaration::is(const std::string& name) const { const std::string* name_ptr = &name; if (std::any_of(name.begin(), name.end(), [](char character) { return std::islower(character); })) { @@ -115,7 +121,7 @@ namespace { bool load_schema_plugin(ifcopenshell::schema_registry& registry, const std::string& schema_name) { ifcopenshell::plugin::manager manager; - manager.add_search_path(ifcopenshell::schema_plugin_directory()); + ifcopenshell::plugin::add_search_paths_or_default(manager, &ifcopenshell::schema_plugin_directory); const auto expected_key = schema_key(schema_name); const auto basename = std::string(schema_plugin_prefix) + boost::to_lower_copy(schema_name); @@ -196,7 +202,7 @@ std::filesystem::path ifcopenshell::schema_plugin_directory() { void ifcopenshell::load_schema_plugins(schema_registry& registry) { plugin::manager manager; - manager.add_search_path(schema_plugin_directory()); + plugin::add_search_paths_or_default(manager, &schema_plugin_directory); for (const auto& path : manager.discover(schema_plugin_prefix)) { auto module = manager.load(path); @@ -247,7 +253,7 @@ std::vector ifcopenshell::schema_registry::names() { } plugin::manager manager; - manager.add_search_path(schema_plugin_directory()); + plugin::add_search_paths_or_default(manager, &schema_plugin_directory); for (const auto& path : manager.discover(schema_plugin_prefix)) { auto module = manager.load(path); if (module.meta().kind_ == plugin::kind::parse_schema && !module.meta().schema.empty()) { diff --git a/src/ifcwrap/IfcPython.i b/src/ifcwrap/IfcPython.i index 52524315af..2e2c911f1c 100644 --- a/src/ifcwrap/IfcPython.i +++ b/src/ifcwrap/IfcPython.i @@ -245,5 +245,31 @@ #include "../serializers/RocksDbSerializer.h" %} +%{ +#include +#include +namespace ifcopenshell { +namespace plugin { +void set_search_paths(const std::vector& paths); +std::vector search_paths(); +void clear_search_paths(); +} +} +%} + +%inline %{ +void set_plugin_search_paths(const std::vector& paths) { + ifcopenshell::plugin::set_search_paths(paths); +} + +std::vector get_plugin_search_paths() { + return ifcopenshell::plugin::search_paths(); +} + +void clear_plugin_search_paths() { + ifcopenshell::plugin::clear_search_paths(); +} +%} + %include "IfcGeomWrapper.i" %include "IfcParseWrapper.i" diff --git a/src/plugin/plugin.cpp b/src/plugin/plugin.cpp index 2e7231f751..d8b13be99c 100644 --- a/src/plugin/plugin.cpp +++ b/src/plugin/plugin.cpp @@ -30,6 +30,8 @@ #include #include +#include +#include #include #include @@ -37,6 +39,50 @@ namespace { using plugin_abi_fn = ifcopenshell::plugin::abi_info(); using plugin_metadata_fn = ifcopenshell::plugin::metadata(); + std::mutex& configured_search_paths_mutex() { + static std::mutex mutex; + return mutex; + } + + std::vector& configured_search_paths() { + static std::vector paths; + return paths; + } + + std::string path_string(const std::filesystem::path& path) { + return path.string(); + } + + void plugin_debug(const std::string& message) { + std::cerr << "[ifcopenshell.plugin] " << message << std::endl; + } + + const char* plugin_kind_name(ifcopenshell::plugin::kind kind) { + switch (kind) { + case ifcopenshell::plugin::kind::parse_schema: + return "parse_schema"; + case ifcopenshell::plugin::kind::mapping: + return "mapping"; + case ifcopenshell::plugin::kind::kernel: + return "kernel"; + case ifcopenshell::plugin::kind::tree: + return "tree"; + case ifcopenshell::plugin::kind::document_serializer: + return "document_serializer"; + case ifcopenshell::plugin::kind::geometry_serializer: + return "geometry_serializer"; + case ifcopenshell::plugin::kind::opencascade_geometry_ifc_writer: + return "opencascade_geometry_ifc_writer"; + default: + return "unknown"; + } + } + + std::vector configured_search_paths_copy() { + std::lock_guard lock(configured_search_paths_mutex()); + return configured_search_paths(); + } + std::string compiler_id() { #if defined(_MSC_VER) return "msvc"; @@ -74,11 +120,7 @@ namespace { } std::vector platform_basenames(const std::string& basename) { - const auto decorated = decorated_basename(basename); - return { - decorated, - "lib" + decorated - }; + return {decorated_basename(basename)}; } #ifdef _WIN32 @@ -99,6 +141,15 @@ namespace { #endif } +namespace ifcopenshell { +namespace plugin { +PLUGIN_API void set_search_paths(const std::vector& paths); +PLUGIN_API std::vector search_paths(); +PLUGIN_API void clear_search_paths(); +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + struct ifcopenshell::plugin::module::data { metadata metadata_; std::filesystem::path path_; @@ -149,6 +200,7 @@ boost::dll::shared_library& ifcopenshell::plugin::module::library() const { ifcopenshell::plugin::manager::manager() = default; void ifcopenshell::plugin::manager::add_search_path(const std::filesystem::path& path) { + plugin_debug("add_search_path " + path_string(path)); search_paths_.push_back(path); } @@ -161,11 +213,14 @@ std::vector ifcopenshell::plugin::manager::discover(const const auto suffix = boost::dll::shared_library::suffix().string(); const auto basename_prefixes = platform_basenames(basename_prefix); + plugin_debug("discover prefix='" + basename_prefix + "' suffix='" + suffix + "' search_paths=" + std::to_string(search_paths_.size())); for (const auto& search_path : search_paths_) { if (!std::filesystem::exists(search_path) || !std::filesystem::is_directory(search_path)) { + plugin_debug("discover skip missing/non-directory search path " + path_string(search_path)); continue; } + plugin_debug("discover scan " + path_string(search_path)); for (const auto& entry : std::filesystem::directory_iterator(search_path)) { if (!entry.is_regular_file()) { continue; @@ -183,12 +238,14 @@ std::vector ifcopenshell::plugin::manager::discover(const continue; } + plugin_debug("discover candidate " + path_string(entry.path())); result.push_back(entry.path()); } } std::sort(result.begin(), result.end()); result.erase(std::unique(result.begin(), result.end()), result.end()); + plugin_debug("discover result count=" + std::to_string(result.size())); return result; } @@ -197,14 +254,18 @@ std::vector ifcopenshell::plugin::manager::discover_exact const auto suffix = boost::dll::shared_library::suffix().string(); const auto basename_candidates = platform_basenames(basename); + plugin_debug("discover_exact basename='" + basename + "' suffix='" + suffix + "' search_paths=" + std::to_string(search_paths_.size())); for (const auto& search_path : search_paths_) { if (!std::filesystem::exists(search_path) || !std::filesystem::is_directory(search_path)) { + plugin_debug("discover_exact skip missing/non-directory search path " + path_string(search_path)); continue; } for (const auto& candidate : basename_candidates) { const auto path = search_path / (candidate + suffix); + plugin_debug("discover_exact probe " + path_string(path)); if (std::filesystem::is_regular_file(path)) { + plugin_debug("discover_exact candidate " + path_string(path)); result.push_back(path); } } @@ -212,13 +273,18 @@ std::vector ifcopenshell::plugin::manager::discover_exact std::sort(result.begin(), result.end()); result.erase(std::unique(result.begin(), result.end()), result.end()); + plugin_debug("discover_exact result count=" + std::to_string(result.size())); return result; } ifcopenshell::plugin::module ifcopenshell::plugin::manager::load(const std::filesystem::path& path) const { + plugin_debug("load " + path_string(path)); #ifdef _WIN32 dll_error_mode_guard error_mode_guard; const auto load_mode = boost::dll::load_mode::load_with_altered_search_path; +#elif defined(__EMSCRIPTEN__) + // Pyodide side modules resolve shared IfcOpenShell symbols from modules loaded earlier. + const auto load_mode = boost::dll::load_mode::rtld_global; #else const auto load_mode = boost::dll::load_mode::default_mode; #endif @@ -226,6 +292,8 @@ ifcopenshell::plugin::module ifcopenshell::plugin::manager::load(const std::file auto abi = library->get_alias("ifcopenshell_plugin_abi_v1")(); validate_abi(abi); auto metadata = library->get_alias("ifcopenshell_plugin_metadata_v1")(); + plugin_debug(std::string("load metadata kind=") + plugin_kind_name(metadata.kind_) + + " id='" + metadata.id + "' schema='" + metadata.schema + "' format='" + metadata.format + "'"); auto data = std::make_shared(); data->metadata_ = metadata; @@ -234,6 +302,45 @@ ifcopenshell::plugin::module ifcopenshell::plugin::manager::load(const std::file return module(data); } +PLUGIN_API void ifcopenshell::plugin::set_search_paths(const std::vector& paths) { + std::lock_guard lock(configured_search_paths_mutex()); + configured_search_paths() = paths; + plugin_debug("set configured search paths count=" + std::to_string(configured_search_paths().size())); + for (const auto& path : configured_search_paths()) { + plugin_debug("configured search path " + path); + } +} + +PLUGIN_API std::vector ifcopenshell::plugin::search_paths() { + const auto paths = configured_search_paths_copy(); + plugin_debug("get configured search paths count=" + std::to_string(paths.size())); + return paths; +} + +PLUGIN_API void ifcopenshell::plugin::clear_search_paths() { + std::lock_guard lock(configured_search_paths_mutex()); + configured_search_paths().clear(); + plugin_debug("cleared configured search paths"); +} + +PLUGIN_API std::filesystem::path ifcopenshell::plugin::add_search_paths_or_default( + manager& manager, std::filesystem::path (*default_search_path)()) +{ + const auto paths = configured_search_paths_copy(); + if (!paths.empty()) { + plugin_debug("using configured plugin search paths; default module directory will not be resolved"); + for (const auto& path : paths) { + manager.add_search_path(path); + } + return {}; + } + + plugin_debug("using default plugin search path from module directory"); + const auto path = default_search_path(); + manager.add_search_path(path); + return path; +} + ifcopenshell::plugin::abi_info ifcopenshell::plugin::host_abi() { abi_info abi; abi.debug_build = is_debug_build(); @@ -245,11 +352,20 @@ ifcopenshell::plugin::abi_info ifcopenshell::plugin::host_abi() { void ifcopenshell::plugin::validate_abi(const abi_info& abi) { const auto host = host_abi(); + plugin_debug("validate_abi plugin_api=" + std::to_string(abi.plugin_api_version) + + " host_api=" + std::to_string(host.plugin_api_version) + + " plugin_compiler='" + abi.compiler_id + " " + abi.compiler_version + "'" + + " host_compiler='" + host.compiler_id + " " + host.compiler_version + "'" + + " plugin_pointer_size=" + std::to_string(abi.pointer_size) + + " host_pointer_size=" + std::to_string(host.pointer_size) + + " plugin_debug=" + std::to_string(abi.debug_build) + + " host_debug=" + std::to_string(host.debug_build)); if (abi.plugin_api_version == host.plugin_api_version && abi.pointer_size == host.pointer_size && abi.debug_build == host.debug_build && abi.compiler_id == host.compiler_id && abi.compiler_version == host.compiler_version) { + plugin_debug("validate_abi compatible"); return; } @@ -260,32 +376,41 @@ void ifcopenshell::plugin::validate_abi(const abi_info& abi) { stream << ", host compiler " << host.compiler_id << " " << host.compiler_version << ")"; stream << " (plugin pointer size " << abi.pointer_size << ", host pointer size " << host.pointer_size << ")"; stream << " (plugin debug " << abi.debug_build << ", host debug " << host.debug_build << ")"; + plugin_debug(stream.str()); throw std::runtime_error(stream.str()); } std::filesystem::path ifcopenshell::plugin::module_directory(const void* symbol) { + plugin_debug("resolve module_directory for symbol " + std::to_string(reinterpret_cast(symbol))); #ifdef _WIN32 HMODULE module_handle = nullptr; if (!GetModuleHandleExW( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(symbol), &module_handle)) { + plugin_debug("module_directory failed in GetModuleHandleExW"); throw std::runtime_error("Unable to resolve module path"); } wchar_t buffer[MAX_PATH]; const DWORD length = GetModuleFileNameW(module_handle, buffer, MAX_PATH); if (length == 0) { + plugin_debug("module_directory failed in GetModuleFileNameW"); throw std::runtime_error("Unable to read module filename"); } - return std::filesystem::path(std::wstring(buffer, length)).parent_path(); + const auto directory = std::filesystem::path(std::wstring(buffer, length)).parent_path(); + plugin_debug("module_directory resolved " + path_string(directory)); + return directory; #else Dl_info info; if (dladdr(symbol, &info) == 0 || !info.dli_fname) { + plugin_debug("module_directory failed in dladdr"); throw std::runtime_error("Unable to resolve module path"); } - return std::filesystem::path(info.dli_fname).parent_path(); + const auto directory = std::filesystem::path(info.dli_fname).parent_path(); + plugin_debug("module_directory resolved " + path_string(directory)); + return directory; #endif } diff --git a/src/pyodide/demo-app/context.py b/src/pyodide/demo-app/context.py index f330c21940..ae0bfd10ef 100644 --- a/src/pyodide/demo-app/context.py +++ b/src/pyodide/demo-app/context.py @@ -1,4 +1,10 @@ +from pathlib import Path + import ifcopenshell + +ifcopenshell.set_plugin_search_paths([str(Path(ifcopenshell.__file__).parent)]) +model = ifcopenshell.file() + import ifcopenshell.api import ifcopenshell.api.aggregate import ifcopenshell.api.context diff --git a/src/pyodide/demo-app/index.html b/src/pyodide/demo-app/index.html index d13e8c6697..adddadf620 100644 --- a/src/pyodide/demo-app/index.html +++ b/src/pyodide/demo-app/index.html @@ -77,7 +77,14 @@ const micropip = pyodide.pyimport("micropip"); await micropip.install("typing-extensions"); document.querySelector("#status2").innerHTML = "Loading IfcOpenShell"; - await micropip.install("wheels/ifcopenshell-0.8.3+34a1bc6-cp313-cp313-emscripten_4_0_9_wasm32.whl"); + + // await micropip.install("wheels/ifcopenshell-0.8.6-cp313-cp313-emscripten_4_0_9_wasm32.whl"); + + await micropip.install("wheels/split2/ifcopenshell-0.8.6-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/split2/ifcopenshell_parse_schema_ifc4-0.8.6-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/split2/ifcopenshell_pure_python-0.8.6-py3-none-any.whl"); + await micropip.install("wheels/split2/ifcopenshell_geometry_kernel_cgalsimple-0.8.6-cp313-cp313-pyodide_2025_0_wasm32.whl"); + await micropip.install("wheels/split2/ifcopenshell_geometry_kernel_opencascade-0.8.6-cp313-cp313-pyodide_2025_0_wasm32.whl"); document.body.className = ''; diff --git a/src/serializers/document_serializer_plugin.cpp b/src/serializers/document_serializer_plugin.cpp index 385ebb587a..4a68a5750b 100644 --- a/src/serializers/document_serializer_plugin.cpp +++ b/src/serializers/document_serializer_plugin.cpp @@ -26,6 +26,12 @@ #include #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { std::string document_serializer_key(const std::string& format) { @@ -50,8 +56,11 @@ std::string document_serializer_plugin_prefix(const std::string& format = std::s } void add_document_serializer_search_paths(ifcopenshell::plugin::manager& manager) { - const auto directory = ifcopenshell::serializers::document_serializer_plugin_directory(); - manager.add_search_path(directory); + const auto directory = ifcopenshell::plugin::add_search_paths_or_default( + manager, &ifcopenshell::serializers::document_serializer_plugin_directory); + if (directory.empty()) { + return; + } const auto sibling_directory = directory.parent_path().parent_path() / "serializers" / directory.filename(); if (sibling_directory != directory && std::filesystem::exists(sibling_directory)) { diff --git a/src/serializers/geometry_serializer_plugin.cpp b/src/serializers/geometry_serializer_plugin.cpp index 625f39daa2..54d2e56105 100644 --- a/src/serializers/geometry_serializer_plugin.cpp +++ b/src/serializers/geometry_serializer_plugin.cpp @@ -24,6 +24,12 @@ #include #include +namespace ifcopenshell { +namespace plugin { +PLUGIN_API std::filesystem::path add_search_paths_or_default(manager& manager, std::filesystem::path (*default_search_path)()); +} +} + namespace { std::string geometry_serializer_plugin_prefix(const std::string& format = std::string()) { @@ -52,8 +58,11 @@ std::string geometry_serializer_format_from_extension(const std::string& extensi } void add_geometry_serializer_search_paths(ifcopenshell::plugin::manager& manager) { - const auto directory = ifcopenshell::serializers::geometry_serializer_plugin_directory(); - manager.add_search_path(directory); + const auto directory = ifcopenshell::plugin::add_search_paths_or_default( + manager, &ifcopenshell::serializers::geometry_serializer_plugin_directory); + if (directory.empty()) { + return; + } const auto sibling_directory = directory.parent_path().parent_path() / "serializers" / directory.filename(); if (sibling_directory != directory && std::filesystem::exists(sibling_directory)) {