diff --git a/.github/workflows/build_win.yml b/.github/workflows/build_win.yml index 8768196e7e..7880296a2a 100644 --- a/.github/workflows/build_win.yml +++ b/.github/workflows/build_win.yml @@ -68,64 +68,6 @@ jobs: cd win python build-all-win.py - - name: Repack .zip Archives With Runtime Plugins - shell: powershell - run: | - $version = (Get-Content VERSION | Select-Object -First 1).Trim() - $sha = $env:GITHUB_SHA.Substring(0, 7) - $outputDir = Join-Path $env:USERPROFILE "output" - $installDir = Get-ChildItem -Path . -Directory | Where-Object { $_.Name -like "_installed*" } | Select-Object -First 1 - if (-not $installDir) { - throw "Install directory not found." - } - - $runtimeFiles = @() - foreach ($runtimeDir in @((Join-Path $installDir.FullName "bin"), (Join-Path $installDir.FullName "lib"))) { - if (Test-Path $runtimeDir) { - $runtimeFiles += Get-ChildItem -Path $runtimeDir -Recurse -File | Where-Object { $_.Extension -ieq ".dll" } - } - } - $runtimeFiles = $runtimeFiles | Sort-Object FullName -Unique - - Get-ChildItem -Path $outputDir -Filter *.zip -ErrorAction SilentlyContinue | Remove-Item -Force - - $binDir = Join-Path $installDir.FullName "bin" - foreach ($exe in Get-ChildItem -Path $binDir -File -Filter *.exe) { - $stageDir = Join-Path $env:RUNNER_TEMP ("package-" + $exe.BaseName) - Remove-Item -Path $stageDir -Recurse -Force -ErrorAction SilentlyContinue - New-Item -ItemType Directory -Path $stageDir | Out-Null - Copy-Item -Path $exe.FullName -Destination $stageDir - foreach ($runtimeFile in $runtimeFiles) { - Copy-Item -Path $runtimeFile.FullName -Destination $stageDir -Force - } - $zipPath = Join-Path $outputDir ("{0}-v{1}-{2}-win64.zip" -f $exe.BaseName, $version, $sha) - Compress-Archive -Path (Join-Path $stageDir '*') -DestinationPath $zipPath -Force - } - - $depsRoot = Join-Path (Get-Location) "_deps" - if (Test-Path $depsRoot) { - foreach ($pythonRoot in Get-ChildItem -Path $depsRoot -Directory | Where-Object { $_.Name -like "python.*" }) { - $pythonVersion = $pythonRoot.Name.Substring("python.".Length) - $pythonVersionMajorMinor = ($pythonVersion.Split(".")[0..1] -join "") - $sitePackages = Join-Path $pythonRoot.FullName "tools\\Lib\\site-packages" - $packagePath = Join-Path $sitePackages "ifcopenshell" - if (!(Test-Path $packagePath)) { - continue - } - Get-ChildItem -Path $packagePath -Recurse -Filter *.pyc -ErrorAction SilentlyContinue | Remove-Item -Force - foreach ($runtimeFile in $runtimeFiles) { - Copy-Item -Path $runtimeFile.FullName -Destination $packagePath -Force - } - $zipPath = Join-Path $outputDir ("ifcopenshell-python-{0}-v{1}-{2}-win64.zip" -f $pythonVersionMajorMinor, $version, $sha) - Push-Location $sitePackages - try { - Compress-Archive -Path "ifcopenshell" -DestinationPath $zipPath -Force - } finally { - Pop-Location - } - } - } - - name: Pack Dependencies run: | cd ${{ matrix.deps_dir }} diff --git a/win/build-all-win.py b/win/build-all-win.py index ded9a36bb6..15538f5560 100644 --- a/win/build-all-win.py +++ b/win/build-all-win.py @@ -6,6 +6,8 @@ but also archives them to '~/outputs'. import os import platform +import re +import shutil import subprocess import zipfile from pathlib import Path @@ -20,6 +22,11 @@ def is_arm64() -> bool: return False return platform.machine().lower() in ("arm64", "aarch64") + +def build_generator() -> str: + return "vs2022-ARM64" if is_arm64() else "vs2022-x64" + + assert Path.cwd() == Path(__file__).parent, "Run this script from the 'win' directory." PYTHON_VERSIONS = ["3.10.3", "3.11.8", "3.12.1", "3.13.0", "3.14.0"] @@ -29,11 +36,19 @@ VERSION = (REPO_PATH / "VERSION").read_text().strip() if "GITHUB_SHA" in os.environ: SHA = os.environ["GITHUB_SHA"][:7] else: - SHA = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + SHA = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()[:7] OUTPUT_DIR = Path.home() / "output" -OUTPUT_DIR.mkdir(exist_ok=True) -print("Output directory:", OUTPUT_DIR) ZIP_TEMPLATE = f"{{package_name}}-v{VERSION}-{SHA}-{'win-arm64' if is_arm64() else 'win64'}.zip" +DEPENDENT_DLL_RE = re.compile(r"^\s*([A-Za-z0-9_.+-]+\.dll)\s*$", re.IGNORECASE) +QT_DEPLOYMENT_DLLS = { + "dxcompiler.dll", + "dxil.dll", + "libegl.dll", + "libglesv2.dll", + "opengl32sw.dll", + "vulkan-1.dll", +} +QT_CONF = "[Paths]\nPrefix = .\n" def run(command: list[str]) -> None: @@ -57,12 +72,112 @@ def restore_env(var_name: str, old_value: str | None) -> None: os.environ[var_name] = old_value +def find_install_dir() -> Path: + arch_install_dir = REPO_PATH / f"_installed-{build_generator()}" + if arch_install_dir.exists(): + return arch_install_dir + + install_dirs = [d for d in REPO_PATH.iterdir() if d.is_dir() and d.name.startswith("_installed")] + if not install_dirs: + raise RuntimeError("Install directory not found.") + return max(install_dirs, key=lambda d: d.stat().st_mtime) + + +def find_dumpbin() -> str: + dumpbin = shutil.which("dumpbin") + if dumpbin: + return dumpbin + + arch_dir = "arm64" if is_arm64() else "x64" + roots = [Path(p) for p in (os.environ.get("VSINSTALLDIR"),) if p] + roots.extend( + Path(p) + for p in ( + r"C:\Program Files\Microsoft Visual Studio\2022\Enterprise", + r"C:\Program Files\Microsoft Visual Studio\2022\Community", + r"C:\Program Files\Microsoft Visual Studio\2022\BuildTools", + ) + ) + for root in roots: + candidates = sorted((root / "VC" / "Tools" / "MSVC").glob(f"*/bin/Host*/{arch_dir}/dumpbin.exe")) + if candidates: + return str(candidates[-1]) + + raise RuntimeError("dumpbin.exe not found. Run build-all-win.py from a Visual Studio developer prompt.") + + +def dumpbin_dependents(file: Path, dumpbin: str) -> set[str]: + output = subprocess.check_output( + [dumpbin, "/nologo", "/dependents", str(file)], + text=True, + errors="replace", + ) + return {match.group(1).lower() for line in output.splitlines() if (match := DEPENDENT_DLL_RE.match(line))} + + +def runtime_candidate_files(install_dir: Path, extra_files: list[Path] | None = None) -> list[Path]: + candidates = list((install_dir / "bin").glob("*.dll")) + plugins_dir = install_dir / "plugins" + if plugins_dir.exists(): + candidates.extend(plugins_dir.rglob("*.dll")) + if extra_files: + candidates.extend(extra_files) + return sorted({file.resolve(): file for file in candidates}.values()) + + +def trace_runtime_dependencies(roots: set[Path], candidates: set[Path]) -> set[Path]: + dumpbin = find_dumpbin() + lookup = {file.name.lower(): file for file in candidates} + resolved: set[Path] = set() + seen: set[Path] = set() + queue = list(roots) + + while queue: + file = queue.pop(0) + file_key = file.resolve() + if file_key in seen: + continue + seen.add(file_key) + + for dependent_name in dumpbin_dependents(file, dumpbin): + dependent = lookup.get(dependent_name) + if dependent is None or dependent in resolved: + continue + resolved.add(dependent) + queue.append(dependent) + + return resolved + + +def is_qt_deployment_dll(file: Path) -> bool: + name = file.name.lower() + return name.startswith("qt") or name.startswith("d3dcompiler_") or name in QT_DEPLOYMENT_DLLS + + +def collect_qt_deployment_files(install_dir: Path) -> dict[str, Path]: + files = {file.name: file for file in (install_dir / "bin").glob("*.dll") if is_qt_deployment_dll(file)} + plugins_dir = install_dir / "plugins" + if plugins_dir.exists(): + for file in plugins_dir.rglob("*.dll"): + files[file.relative_to(install_dir).as_posix()] = file + return files + + +def write_zip(zip_path: Path, files: dict[str, Path], generated_files: dict[str, str] | None = None) -> None: + zip_path.parent.mkdir(exist_ok=True) + with ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: + for arcname, file in sorted(files.items()): + zipf.write(file, arcname=arcname) + for arcname, contents in sorted((generated_files or {}).items()): + zipf.writestr(arcname, contents) + + def build() -> None: for python_version in PYTHON_VERSIONS: os.environ["PYTHON_VERSION"] = python_version print(f"Building for Python {python_version}...") subprocess.run( - [str(REPO_WIN / "build-deps.cmd"), "vs2022-ARM64" if is_arm64() else "vs2022-x64", "Release"], + [str(REPO_WIN / "build-deps.cmd"), build_generator(), "Release"], check=True, text=True, input="y\n", @@ -71,29 +186,69 @@ def build() -> None: run( [ str(REPO_WIN / "run-cmake.bat"), - "vs2022-ARM64" if is_arm64() else "vs2022-x64", + build_generator(), "-DENABLE_BUILD_OPTIMIZATIONS=ON", "-DGLTF_SUPPORT=ON", ] ) restore_env(*OLD_ADD_COMMIT_SHA) - run([str(REPO_WIN / "install-ifcopenshell.bat"), "vs2022-ARM64" if is_arm64() else "vs2022-x64", "Release"]) + run([str(REPO_WIN / "install-ifcopenshell.bat"), build_generator(), "Release"]) def archive_executables() -> None: - # Typically '_installed-vs2022-x64'. - install_dir = next(d for d in REPO_PATH.iterdir() if d.is_dir() and d.name.startswith("_installed")) + install_dir = find_install_dir() + + bin_files = set((install_dir / "bin").iterdir()) + exes = {file for file in bin_files if file.suffix.lower() == ".exe"} + dlls = {file for file in bin_files if file.suffix.lower() == ".dll"} + dependencies = trace_runtime_dependencies(exes, dlls) + ifc_runtime_plugins = { + d + for d in (set(dlls) - dependencies) + if d.name.startswith("ifcopenshell.") and not d.name.startswith("ifcopenshell.geometry.writer.") + } + qt_deployment_files = collect_qt_deployment_files(install_dir) + + for file in sorted(exes): + files: dict[str, Path] = {file.name: file} + roots = {file} + + # IfcOpenShell plugins are loaded by name at runtime, so dumpbin cannot discover them. + # svgfill does not require plugins. + if not file.name.lower().startswith("svgfill"): + roots.update(ifc_runtime_plugins) + for plugin in ifc_runtime_plugins: + files[plugin.name] = plugin + + runtime_dependencies = trace_runtime_dependencies(roots, dlls) + for dependency in runtime_dependencies: + files[dependency.name] = dependency + + generated_files = {} + if any(dependency.name.lower().startswith("qt") for dependency in runtime_dependencies): + for arcname, dependency in qt_deployment_files.items(): + files[arcname] = dependency + generated_files["qt.conf"] = QT_CONF - for file in (install_dir / "bin").iterdir(): - if file.suffix.lower() != ".exe": - continue zip_name = ZIP_TEMPLATE.format(package_name=file.stem) - with ZipFile(OUTPUT_DIR / zip_name, "w", compression=zipfile.ZIP_DEFLATED) as zipf: - zipf.write(file, arcname=file.name) + write_zip(OUTPUT_DIR / zip_name, files, generated_files) print(f"{file} -> {zip_name}") def archive_python_package(python_version: str, python_path: Path) -> None: + install_dir = find_install_dir() + + bin_files = set((install_dir / "bin").iterdir()) + exes = {file for file in bin_files if file.suffix.lower() == ".exe"} + dlls = {file for file in bin_files if file.suffix.lower() == ".dll"} + dependencies = trace_runtime_dependencies(exes, dlls) + ifc_runtime_plugins = { + d + for d in (set(dlls) - dependencies) + if d.name.startswith("ifcopenshell.") and not d.name.startswith("ifcopenshell.geometry.writer.") + } + geometry_writing = {f for f in bin_files if f.name.startswith("ifcopenshell.geometry.writer.")} + python_version_major_minor = "".join(python_version.split(".")[:2]) site_packages = python_path / "Lib" / "site-packages" package_path = site_packages / "ifcopenshell" @@ -102,11 +257,22 @@ def archive_python_package(python_version: str, python_path: Path) -> None: for file in package_path.rglob("*.pyc"): file.unlink() + files: dict[str, Path] = {} + package_binaries = set() + for file in package_path.rglob("*"): + arcname = file.relative_to(site_packages) + files[str(arcname)] = file + if file.suffix.lower() in (".dll", ".exe", ".pyd"): + package_binaries.add(file) + + runtime_files = ifc_runtime_plugins | geometry_writing + runtime_dependencies = trace_runtime_dependencies(package_binaries | runtime_files, dlls | package_binaries) + + for file in runtime_files | runtime_dependencies: + files[f"ifcopenshell/{file.name}"] = file + zip_name = ZIP_TEMPLATE.format(package_name=f"ifcopenshell-python-{python_version_major_minor}") - with ZipFile(OUTPUT_DIR / zip_name, "w", compression=zipfile.ZIP_DEFLATED) as zipf: - for file in package_path.rglob("*"): - arcname = file.relative_to(site_packages) - zipf.write(file, arcname=arcname) + write_zip(OUTPUT_DIR / zip_name, files) print(f"{package_path} -> {zip_name}") @@ -122,6 +288,7 @@ def archive_python_packages() -> None: def main() -> None: + print("Output directory:", OUTPUT_DIR) build() archive_executables() archive_python_packages()