Win packaging changes

.github/workflows/build_win.yml

@@ -68,64 +68,6 @@ jobs:
           cd win
           python build-all-win.py
 
-      - name: Repack .zip Archives With Runtime Plugins
-        shell: powershell
-        run: |
-          $version = (Get-Content VERSION | Select-Object -First 1).Trim()
-          $sha = $env:GITHUB_SHA.Substring(0, 7)
-          $outputDir = Join-Path $env:USERPROFILE "output"
-          $installDir = Get-ChildItem -Path . -Directory | Where-Object { $_.Name -like "_installed*" } | Select-Object -First 1
-          if (-not $installDir) {
-            throw "Install directory not found."
-          }
-
-          $runtimeFiles = @()
-          foreach ($runtimeDir in @((Join-Path $installDir.FullName "bin"), (Join-Path $installDir.FullName "lib"))) {
-            if (Test-Path $runtimeDir) {
-              $runtimeFiles += Get-ChildItem -Path $runtimeDir -Recurse -File | Where-Object { $_.Extension -ieq ".dll" }
-            }
-          }
-          $runtimeFiles = $runtimeFiles | Sort-Object FullName -Unique
-
-          Get-ChildItem -Path $outputDir -Filter *.zip -ErrorAction SilentlyContinue | Remove-Item -Force
-
-          $binDir = Join-Path $installDir.FullName "bin"
-          foreach ($exe in Get-ChildItem -Path $binDir -File -Filter *.exe) {
-            $stageDir = Join-Path $env:RUNNER_TEMP ("package-" + $exe.BaseName)
-            Remove-Item -Path $stageDir -Recurse -Force -ErrorAction SilentlyContinue
-            New-Item -ItemType Directory -Path $stageDir | Out-Null
-            Copy-Item -Path $exe.FullName -Destination $stageDir
-            foreach ($runtimeFile in $runtimeFiles) {
-              Copy-Item -Path $runtimeFile.FullName -Destination $stageDir -Force
-            }
-            $zipPath = Join-Path $outputDir ("{0}-v{1}-{2}-win64.zip" -f $exe.BaseName, $version, $sha)
-            Compress-Archive -Path (Join-Path $stageDir '*') -DestinationPath $zipPath -Force
-          }
-
-          $depsRoot = Join-Path (Get-Location) "_deps"
-          if (Test-Path $depsRoot) {
-            foreach ($pythonRoot in Get-ChildItem -Path $depsRoot -Directory | Where-Object { $_.Name -like "python.*" }) {
-              $pythonVersion = $pythonRoot.Name.Substring("python.".Length)
-              $pythonVersionMajorMinor = ($pythonVersion.Split(".")[0..1] -join "")
-              $sitePackages = Join-Path $pythonRoot.FullName "tools\\Lib\\site-packages"
-              $packagePath = Join-Path $sitePackages "ifcopenshell"
-              if (!(Test-Path $packagePath)) {
-                continue
-              }
-              Get-ChildItem -Path $packagePath -Recurse -Filter *.pyc -ErrorAction SilentlyContinue | Remove-Item -Force
-              foreach ($runtimeFile in $runtimeFiles) {
-                Copy-Item -Path $runtimeFile.FullName -Destination $packagePath -Force
-              }
-              $zipPath = Join-Path $outputDir ("ifcopenshell-python-{0}-v{1}-{2}-win64.zip" -f $pythonVersionMajorMinor, $version, $sha)
-              Push-Location $sitePackages
-              try {
-                Compress-Archive -Path "ifcopenshell" -DestinationPath $zipPath -Force
-              } finally {
-                Pop-Location
-              }
-            }
-          }
-
       - name: Pack Dependencies
         run: |
           cd ${{ matrix.deps_dir }}

win/build-all-win.py

@@ -6,6 +6,8 @@ but also archives them to '~/outputs'.
 
 import os
 import platform
+import re
+import shutil
 import subprocess
 import zipfile
 from pathlib import Path
@@ -20,6 +22,11 @@ def is_arm64() -> bool:
         return False
     return platform.machine().lower() in ("arm64", "aarch64")
 
+
+def build_generator() -> str:
+    return "vs2022-ARM64" if is_arm64() else "vs2022-x64"
+
+
 assert Path.cwd() == Path(__file__).parent, "Run this script from the 'win' directory."
 
 PYTHON_VERSIONS = ["3.10.3", "3.11.8", "3.12.1", "3.13.0", "3.14.0"]
@@ -29,11 +36,19 @@ VERSION = (REPO_PATH / "VERSION").read_text().strip()
 if "GITHUB_SHA" in os.environ:
     SHA = os.environ["GITHUB_SHA"][:7]
 else:
-    SHA = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
+    SHA = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()[:7]
 OUTPUT_DIR = Path.home() / "output"
-OUTPUT_DIR.mkdir(exist_ok=True)
-print("Output directory:", OUTPUT_DIR)
 ZIP_TEMPLATE = f"{{package_name}}-v{VERSION}-{SHA}-{'win-arm64' if is_arm64() else 'win64'}.zip"
+DEPENDENT_DLL_RE = re.compile(r"^\s*([A-Za-z0-9_.+-]+\.dll)\s*$", re.IGNORECASE)
+QT_DEPLOYMENT_DLLS = {
+    "dxcompiler.dll",
+    "dxil.dll",
+    "libegl.dll",
+    "libglesv2.dll",
+    "opengl32sw.dll",
+    "vulkan-1.dll",
+}
+QT_CONF = "[Paths]\nPrefix = .\n"
 
 
 def run(command: list[str]) -> None:
@@ -57,12 +72,112 @@ def restore_env(var_name: str, old_value: str | None) -> None:
         os.environ[var_name] = old_value
 
 
+def find_install_dir() -> Path:
+    arch_install_dir = REPO_PATH / f"_installed-{build_generator()}"
+    if arch_install_dir.exists():
+        return arch_install_dir
+
+    install_dirs = [d for d in REPO_PATH.iterdir() if d.is_dir() and d.name.startswith("_installed")]
+    if not install_dirs:
+        raise RuntimeError("Install directory not found.")
+    return max(install_dirs, key=lambda d: d.stat().st_mtime)
+
+
+def find_dumpbin() -> str:
+    dumpbin = shutil.which("dumpbin")
+    if dumpbin:
+        return dumpbin
+
+    arch_dir = "arm64" if is_arm64() else "x64"
+    roots = [Path(p) for p in (os.environ.get("VSINSTALLDIR"),) if p]
+    roots.extend(
+        Path(p)
+        for p in (
+            r"C:\Program Files\Microsoft Visual Studio\2022\Enterprise",
+            r"C:\Program Files\Microsoft Visual Studio\2022\Community",
+            r"C:\Program Files\Microsoft Visual Studio\2022\BuildTools",
+        )
+    )
+    for root in roots:
+        candidates = sorted((root / "VC" / "Tools" / "MSVC").glob(f"*/bin/Host*/{arch_dir}/dumpbin.exe"))
+        if candidates:
+            return str(candidates[-1])
+
+    raise RuntimeError("dumpbin.exe not found. Run build-all-win.py from a Visual Studio developer prompt.")
+
+
+def dumpbin_dependents(file: Path, dumpbin: str) -> set[str]:
+    output = subprocess.check_output(
+        [dumpbin, "/nologo", "/dependents", str(file)],
+        text=True,
+        errors="replace",
+    )
+    return {match.group(1).lower() for line in output.splitlines() if (match := DEPENDENT_DLL_RE.match(line))}
+
+
+def runtime_candidate_files(install_dir: Path, extra_files: list[Path] | None = None) -> list[Path]:
+    candidates = list((install_dir / "bin").glob("*.dll"))
+    plugins_dir = install_dir / "plugins"
+    if plugins_dir.exists():
+        candidates.extend(plugins_dir.rglob("*.dll"))
+    if extra_files:
+        candidates.extend(extra_files)
+    return sorted({file.resolve(): file for file in candidates}.values())
+
+
+def trace_runtime_dependencies(roots: set[Path], candidates: set[Path]) -> set[Path]:
+    dumpbin = find_dumpbin()
+    lookup = {file.name.lower(): file for file in candidates}
+    resolved: set[Path] = set()
+    seen: set[Path] = set()
+    queue = list(roots)
+
+    while queue:
+        file = queue.pop(0)
+        file_key = file.resolve()
+        if file_key in seen:
+            continue
+        seen.add(file_key)
+
+        for dependent_name in dumpbin_dependents(file, dumpbin):
+            dependent = lookup.get(dependent_name)
+            if dependent is None or dependent in resolved:
+                continue
+            resolved.add(dependent)
+            queue.append(dependent)
+
+    return resolved
+
+
+def is_qt_deployment_dll(file: Path) -> bool:
+    name = file.name.lower()
+    return name.startswith("qt") or name.startswith("d3dcompiler_") or name in QT_DEPLOYMENT_DLLS
+
+
+def collect_qt_deployment_files(install_dir: Path) -> dict[str, Path]:
+    files = {file.name: file for file in (install_dir / "bin").glob("*.dll") if is_qt_deployment_dll(file)}
+    plugins_dir = install_dir / "plugins"
+    if plugins_dir.exists():
+        for file in plugins_dir.rglob("*.dll"):
+            files[file.relative_to(install_dir).as_posix()] = file
+    return files
+
+
+def write_zip(zip_path: Path, files: dict[str, Path], generated_files: dict[str, str] | None = None) -> None:
+    zip_path.parent.mkdir(exist_ok=True)
+    with ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
+        for arcname, file in sorted(files.items()):
+            zipf.write(file, arcname=arcname)
+        for arcname, contents in sorted((generated_files or {}).items()):
+            zipf.writestr(arcname, contents)
+
+
 def build() -> None:
     for python_version in PYTHON_VERSIONS:
         os.environ["PYTHON_VERSION"] = python_version
         print(f"Building for Python {python_version}...")
         subprocess.run(
-            [str(REPO_WIN / "build-deps.cmd"), "vs2022-ARM64" if is_arm64() else "vs2022-x64", "Release"],
+            [str(REPO_WIN / "build-deps.cmd"), build_generator(), "Release"],
             check=True,
             text=True,
             input="y\n",
@@ -71,29 +186,69 @@ def build() -> None:
         run(
             [
                 str(REPO_WIN / "run-cmake.bat"),
-                "vs2022-ARM64" if is_arm64() else "vs2022-x64",
+                build_generator(),
                 "-DENABLE_BUILD_OPTIMIZATIONS=ON",
                 "-DGLTF_SUPPORT=ON",
             ]
         )
         restore_env(*OLD_ADD_COMMIT_SHA)
-        run([str(REPO_WIN / "install-ifcopenshell.bat"), "vs2022-ARM64" if is_arm64() else "vs2022-x64", "Release"])
+        run([str(REPO_WIN / "install-ifcopenshell.bat"), build_generator(), "Release"])
 
 
 def archive_executables() -> None:
-    # Typically '_installed-vs2022-x64'.
+    install_dir = find_install_dir()
-    install_dir = next(d for d in REPO_PATH.iterdir() if d.is_dir() and d.name.startswith("_installed"))
+
+    bin_files = set((install_dir / "bin").iterdir())
+    exes = {file for file in bin_files if file.suffix.lower() == ".exe"}
+    dlls = {file for file in bin_files if file.suffix.lower() == ".dll"}
+    dependencies = trace_runtime_dependencies(exes, dlls)
+    ifc_runtime_plugins = {
+        d
+        for d in (set(dlls) - dependencies)
+        if d.name.startswith("ifcopenshell.") and not d.name.startswith("ifcopenshell.geometry.writer.")
+    }
+    qt_deployment_files = collect_qt_deployment_files(install_dir)
+
+    for file in sorted(exes):
+        files: dict[str, Path] = {file.name: file}
+        roots = {file}
+
+        # IfcOpenShell plugins are loaded by name at runtime, so dumpbin cannot discover them.
+        # svgfill does not require plugins.
+        if not file.name.lower().startswith("svgfill"):
+            roots.update(ifc_runtime_plugins)
+            for plugin in ifc_runtime_plugins:
+                files[plugin.name] = plugin
+
+        runtime_dependencies = trace_runtime_dependencies(roots, dlls)
+        for dependency in runtime_dependencies:
+            files[dependency.name] = dependency
+
+        generated_files = {}
+        if any(dependency.name.lower().startswith("qt") for dependency in runtime_dependencies):
+            for arcname, dependency in qt_deployment_files.items():
+                files[arcname] = dependency
+            generated_files["qt.conf"] = QT_CONF
 
-    for file in (install_dir / "bin").iterdir():
-        if file.suffix.lower() != ".exe":
-            continue
         zip_name = ZIP_TEMPLATE.format(package_name=file.stem)
-        with ZipFile(OUTPUT_DIR / zip_name, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
+        write_zip(OUTPUT_DIR / zip_name, files, generated_files)
-            zipf.write(file, arcname=file.name)
         print(f"{file} -> {zip_name}")
 
 
 def archive_python_package(python_version: str, python_path: Path) -> None:
+    install_dir = find_install_dir()
+
+    bin_files = set((install_dir / "bin").iterdir())
+    exes = {file for file in bin_files if file.suffix.lower() == ".exe"}
+    dlls = {file for file in bin_files if file.suffix.lower() == ".dll"}
+    dependencies = trace_runtime_dependencies(exes, dlls)
+    ifc_runtime_plugins = {
+        d
+        for d in (set(dlls) - dependencies)
+        if d.name.startswith("ifcopenshell.") and not d.name.startswith("ifcopenshell.geometry.writer.")
+    }
+    geometry_writing = {f for f in bin_files if f.name.startswith("ifcopenshell.geometry.writer.")}
+
     python_version_major_minor = "".join(python_version.split(".")[:2])
     site_packages = python_path / "Lib" / "site-packages"
     package_path = site_packages / "ifcopenshell"
@@ -102,11 +257,22 @@ def archive_python_package(python_version: str, python_path: Path) -> None:
     for file in package_path.rglob("*.pyc"):
         file.unlink()
 
+    files: dict[str, Path] = {}
+    package_binaries = set()
+    for file in package_path.rglob("*"):
+        arcname = file.relative_to(site_packages)
+        files[str(arcname)] = file
+        if file.suffix.lower() in (".dll", ".exe", ".pyd"):
+            package_binaries.add(file)
+
+    runtime_files = ifc_runtime_plugins | geometry_writing
+    runtime_dependencies = trace_runtime_dependencies(package_binaries | runtime_files, dlls | package_binaries)
+
+    for file in runtime_files | runtime_dependencies:
+        files[f"ifcopenshell/{file.name}"] = file
+
     zip_name = ZIP_TEMPLATE.format(package_name=f"ifcopenshell-python-{python_version_major_minor}")
-    with ZipFile(OUTPUT_DIR / zip_name, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
+    write_zip(OUTPUT_DIR / zip_name, files)
-        for file in package_path.rglob("*"):
-            arcname = file.relative_to(site_packages)
-            zipf.write(file, arcname=arcname)
     print(f"{package_path} -> {zip_name}")
 
 
@@ -122,6 +288,7 @@ def archive_python_packages() -> None:
 
 
 def main() -> None:
+    print("Output directory:", OUTPUT_DIR)
     build()
     archive_executables()
     archive_python_packages()