build_pyodide - pack cached dependencies

.github/workflows/build_pyodide.yml

@@ -23,6 +23,20 @@ jobs:
           token: ${{ secrets.BUILD_REPO_TOKEN }}
           path: pyodide
 
+      - name: Checkout Build Repository
+        uses: actions/checkout@v3
+        with:
+          repository: IfcOpenShell/build-outputs
+          path: ifcopenshell_build
+          ref: wasm
+          lfs: true
+          token: ${{ secrets.BUILD_REPO_TOKEN }}
+
+      - name: Unpack Dependencies
+        run: |
+          cd ifcopenshell_build
+          python ../IfcOpenShell/pyodide/cache_dependencies.py unpack
+
       - name: Build
         run: |
           VERSION=`cat IfcOpenShell/VERSION`
@@ -33,6 +47,20 @@ jobs:
           NEW_FILE=`echo $FILE | sed "s/$VERSION/$VERSION+${GITHUB_SHA:0:7}/"`
           mv $FILE $NEW_FILE
 
+      - name: Pack Dependencies
+        run: |
+          cd ifcopenshell_build
+          python ../IfcOpenShell/pyodide/cache_dependencies.py pack
+
+      - name: Commit and Push Changes to Build Repository
+        run: |
+          cd ifcopenshell_build
+          git config user.name "IfcOpenBot"
+          git config user.email "ifcopenbot@ifcopenshell.org"
+          git add */*/install/cache-*.tar.gz
+          git commit -m "Update build artifacts [skip ci]" || echo "No changes to commit"
+          git push || echo "Push failed"
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:

pyodide/build_pyodide.sh

@@ -9,4 +9,7 @@ cp IfcOpenShell/pyodide/meta.yaml packages/ifcopenshell
 export PYODIDE_ROOT=/src/pyodide
 # Ensure emsdk tools are in PATH.
 export PATH=$PYODIDE_ROOT/emsdk/emsdk:$PYODIDE_ROOT/emsdk/emsdk/node/22.16.0_64bit/bin:$PYODIDE_ROOT/emsdk/emsdk/upstream/emscripten:$PATH
+# Use custom build ifcopenshell directory in build-all to make caching simpler
+# Otherwise pyodide build path typically includes package version, so cached cmake configs might break.
+export BUILD_DIR=/src/ifcopenshell_build
 pyodide build-recipes ifcopenshell --install

pyodide/cache_dependencies.py

@@ -0,0 +1,58 @@
+"""
+Cache built dependencies for builds.
+
+This script is finding common install directory and either
+packs each folder into a tar.gz archive, if it wasn't packed before,
+or unpacks existing archives.
+
+Usage: python cache_dependencies.py [pack|unpack]
+"""
+
+import tarfile
+import sys
+from pathlib import Path
+
+
+CACHE_PREFIX = "cache-"
+
+
+def get_install_dir() -> Path:
+    for data in Path.cwd().glob("*/*/install"):
+        return data
+    raise Exception("No install dir found")
+
+
+def pack_dependencies(install_dir: Path) -> None:
+    # Process each install_dir
+    for dependency_path in install_dir.iterdir():
+        if not dependency_path.is_dir():
+            continue
+        dependency_name = dependency_path.name
+        tar_path = install_dir / f"{CACHE_PREFIX}{dependency_name}.tar.gz"
+        if tar_path.exists():
+            print(f"Skipping existing cache: '{tar_path}'")
+        else:
+            with tarfile.open(tar_path, "w:gz") as tar:
+                tar.add(dependency_path, arcname=dependency_path.name)
+            print(f"Created cache: '{tar_path}'")
+
+
+def unpack_dependencies(install_dir: Path) -> None:
+    for tar_path in install_dir.glob(f"{CACHE_PREFIX}*.tar.gz"):
+        with tarfile.open(tar_path, "r:gz") as tar:
+            tar.extractall(path=install_dir)
+        print(f"Extracted cache: '{tar_path.name}'.")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2 or (action := sys.argv[1].lower()) not in ("pack", "unpack"):
+        print(__doc__)
+        sys.exit(1)
+
+    install_dir = get_install_dir()
+    print(f"Found install dir: '{install_dir}'")
+
+    if action == "pack":
+        pack_dependencies(install_dir)
+    else:
+        unpack_dependencies(install_dir)