diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 0000000000..2979bddbab --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,4 @@ +.env +*.pyc +__pycache__ +redis-data diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000000..2979bddbab --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,4 @@ +.env +*.pyc +__pycache__ +redis-data diff --git a/docker/.ifcos_env b/docker/.ifcos_env new file mode 100644 index 0000000000..2603df959e --- /dev/null +++ b/docker/.ifcos_env @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# .ifcos_env +# register autocompletes. just source the file in your shell, i.e. +# source .ifcos_env + +.ifcos_env() { + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + opts="create update up down restart build attach logs ps config remove help" + + # Basic static completion + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + + return 0 +} + +# Register the completion for the command "ifcos_env" +complete -F .ifcos_env ./ifcos_env diff --git a/docker/Dockerfile_init b/docker/Dockerfile_init new file mode 100644 index 0000000000..a48ffe3262 --- /dev/null +++ b/docker/Dockerfile_init @@ -0,0 +1,40 @@ +FROM rockylinux:9 + +# Update system +RUN dnf update -y + +# Enable CRB (needed by some EPEL packages) and install EPEL +RUN dnf install -y epel-release && \ + dnf config-manager --set-enabled crb && \ + dnf install -y --setopt=install_weak_deps=False \ + ccache + +# Install required packages + some common tools for a bit of command line comfort +RUN dnf install -y --allowerasing bash-completion vim git curl wget which tree htop \ + gcc gcc-c++ git autoconf automake bison make zip cmake python3 python3-pip \ + bzip2 patch mesa-libGL-devel libffi-devel fontconfig-devel \ + sqlite-devel bzip2-devel zlib-devel openssl-devel xz-devel \ + readline-devel ncurses-devel libffi-devel libuuid-devel git-lfs \ + findutils xz byacc ccache + +# Clean the caches +RUN dnf clean all && \ + rm -rf /var/cache/dnf + +# Configure ccache +ENV CCACHE_DIR=/ccache +ENV PATH="/usr/lib/ccache:$PATH" + +# Optional: Set a reasonable cache size limit (adjust as needed) +RUN ccache -M 5G # e.g. 5 GB max + +# Setup uv +COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /uvx /bin/ + +# Install Python +RUN uv python install + +# Prevent dubious ownership error +RUN git config --global --add safe.directory '*' + +CMD ["sleep", "infinity"] diff --git a/docker/Dockerfile_update b/docker/Dockerfile_update new file mode 100644 index 0000000000..a6ee2b62fa --- /dev/null +++ b/docker/Dockerfile_update @@ -0,0 +1,16 @@ +FROM ifcopenshell-build-env:updated + +# Update system +RUN dnf update -y + +# Clean the caches +RUN dnf clean all && \ + rm -rf /var/cache/dnf + +# Setup uv +COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /uvx /bin/ + +# Install Python +RUN uv python install + +CMD ["sleep", "infinity"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..8674c89685 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,78 @@ +Docker build environment +======================== + +This is a small utility to make it easy to compile a perfect `_ifcopenshell_wrapper.cpython-*-x86_64-linux-gnu.so` +files. + +The reason for this tool is that I was trying to follow the web page directions, and my build was behaving differently +to the release builds. Eventually I concluded that the differences between toolchains on the RHEL based rocky9 image +and Ubuntu were just too great. Getting the build setup was already a lot of trial and error, so I thought I'd spend +more time trying to reuse the github actions that perform the build, using a utility called `act`. I learnt a lot, in +particular how much time, energy, and bandwidth Github waste. I also realised I was most of the way to a regular docker +setup anyway, so I might as well just do that. So I've deconstructed all the github action steps, and turned it into +a local docker build environment that uses the exact same base, tools, libraries, and build command/flags etc. + +Right now a Github action will: +- launch the rocky9 base +- upgrade all the packages +- install a bunch of extra tools +- do a recursive checkout of your repo +- checkout the build repository +- unpack dependencies +- run the build script, making all python versions (5? right now I think) +- create the .zip release files + +And it does _all_ of that _every_ time. This is not a fault of the action writers - it's just how Github seems to work. + +These dockers tools do the following differently, and it's actually a bit more powerful too: +- build the base image once. +- update the packages once. +- install the extra tools once. +- the repository is the one on your host, that gets bind mounted in the container as the working directory. +- by adding an environment variable to .env, restricts to compiling for just a single python version. +- when the build is finished the created files are right there under your local repositry (but not added to git) for + ease of access +- each repository can have it's own build environment container. +- the image is shared between those environments. +- the containers share the ccache, so additional envs should get a helping hand. +- it has a simple set of user friendly commands to drive it all. + +For example: +``` bash +# To see the commands (a superset of docker compose commands) +./ifcos_env + +# Enable autocomplete of commands +source .ifcos_env + +# First time commands +./ifcos_env create +./ifcos_env up +./ifcos_env build + +# install and test library +# find an issue +# edit code +./ifcos_env build + +# and so on. When done stop and optionally delete the container +./ifcos_env stop +./ifcos_env remove +``` + +To limit the build to one python version just add +``` bash +PY_TGT=py-311 +``` +or whichever version your Blender requires. + +You might see UNIQUE_ID in the .env file too. This keeps containers for separate folders, separate. + +System requirements +1. Linux-x64 only at this time. +2. Docker and docker-compose need to be installed. +3. Have a good amount of disk space. (image is in /var (typically the root partition) and will be about 1.7 GB) +4. The build action will create about 10GB in your repository folder. Make sure this partition is spacious + particularly if you intent on having multiple clones building. +5. ... I think that covers most of it. + diff --git a/docker/SKILL.md b/docker/SKILL.md new file mode 100644 index 0000000000..4ddc3ce18c --- /dev/null +++ b/docker/SKILL.md @@ -0,0 +1,137 @@ +--- +name: ifcopenshell-docker-build +description: >- + Build a real ifcopenshell_wrapper (.so + .py) and IfcConvert locally via + the docker/ifcos_env toolchain, then wire them into a checkout for + running C++-dependent parts of the test suite (geometry, the SWIG + wrapper stub, the C++ parser). Use whenever a task needs to compile + IfcOpenShell's C++ core rather than just read/patch source - e.g. + reproducing or fixing a bug in src/ifcgeom, src/ifcparse, src/ifcwrap, + or validating util/scripts/validate_stub.py against the actual + generated wrapper. +--- + +# Building IfcOpenShell locally with docker/ifcos_env + +`docker/` is a small toolchain (see `docker/README.md` for the original +author's own description and design rationale - read that first for the +*why*; this file is the practical *how*, distilled from actually driving it +end-to-end) that mirrors the project's GitHub Actions build environment +locally, with a persistent container and ccache so repeat builds are fast. +Pure-Python changes don't need any of this - only reach for it when you need +a real compiled `_ifcopenshell_wrapper*.so` or `IfcConvert` binary. + +## Placement + +This `docker/` folder must live as a direct child of the repo root you want +to build (sibling of `src/`, `cmake/`, etc.) - `compose.yaml` and +`ifcos_env` resolve the repo via `../` relative to wherever `docker/` +itself sits, and bind-mount it into the container. If you're setting this +up in a fresh clone, copy the whole `docker/` directory there first. + +## First-time setup + +```bash +cd docker +./ifcos_env create # build the base image (shared across all your clones/checkouts by name, so usually instant after the first time anywhere) +./ifcos_env up # start the container, clone+unpack the third-party dependency cache (~10GB, one-time per container) +./ifcos_env build # full build: all deps + IfcParse + IfcGeom + IfcConvert + the Python wrapper, for one Python version +``` + +`PY_TGT` and `UNIQUE_ID` live in `docker/.env` - `PY_TGT` (e.g. `py-311`) +restricts the build to one Python version instead of building five; +`UNIQUE_ID` is a hash of the folder path, recalculated on every `up`, so +each checkout gets its own container/volumes automatically. + +A full first build takes ~1.5 hours (mostly compiling IfcOpenShell's own +C++, not the cached third-party deps). After that, ccache makes incremental +rebuilds of a couple of touched `.cpp` files **under a minute**. + +## Fast iteration + +Pass a target to `build` to skip the parts you don't need: + +```bash +./ifcos_env build IfcConvert # only the executables (IfcConvert, IfcGeomServer) - skips the Python wrapper entirely +./ifcos_env build IfcOpenShell-Python # only the SWIG Python wrapper - skips executables entirely +./ifcos_env build # no target = everything (needed the first time, or after touching shared headers) +``` + +Use this to keep the edit -> rebuild -> test loop fast when debugging: if +you're only touching `src/ifcgeom/`, build `IfcConvert`; if you're only +exercising the Python API, build `IfcOpenShell-Python`. + +## Where the artifacts land + +Build output goes to `/build/Linux/x86_64/install/` on the host +(bind-mounted, not just inside the container): + +- `ifcopenshell/bin/IfcConvert` - the CLI binary +- `python-/lib/python/site-packages/ifcopenshell/_ifcopenshell_wrapper*.so` + and `ifcopenshell_wrapper.py` - the compiled wrapper + its generated + Python glue + +## Wiring the build into a checkout for testing + +`_ifcopenshell_wrapper*.so` and `ifcopenshell_wrapper.py` are already +gitignored under `src/ifcopenshell-python/ifcopenshell/`, which is exactly +where a normal in-tree build would put them - copy the two files there: + +```bash +SRC=build/Linux/x86_64/install/python-3.11.8/lib/python3.11/site-packages/ifcopenshell +cp "$SRC/_ifcopenshell_wrapper.cpython-311-x86_64-linux-gnu.so" src/ifcopenshell-python/ifcopenshell/ +cp "$SRC/ifcopenshell_wrapper.py" src/ifcopenshell-python/ifcopenshell/ +``` + +Then, to run the test suite against it: + +```bash +export PATH="$PWD/build/Linux/x86_64/install/ifcopenshell/bin:$PATH" # for IfcConvert-dependent tests +cd src/ifcopenshell-python/test +PYTHONPATH="$PWD/.." python3.11 -m pytest -p no:pytest-blender . +``` + +(`-p no:pytest-blender` avoids the pytest-blender plugin trying to find a +`blender` executable and failing collection entirely, even for non-Blender +tests.) You'll need the matching Python version's `pip install`s too +(numpy, shapely, isodate, lark, tabulate, pytest, ... - whatever the +modules under test import) since this is a bare interpreter, not the +project's pixi env. + +## Known gotchas (some fixed in this copy, watch for them if you're on an +## older/different copy of this script) + +- **`try` is an unimplemented stub** - it prints a message and does + nothing. If you want the wrapper pushed straight into a Blender + extensions folder for manual testing, do the copy yourself (see the + README's example path) rather than relying on `try`. +- **`stop`/`down` removes the container**, it does not pause it (it's + literally `docker compose down`). Named volumes (ccache) and the + bind-mounted `build/` survive, so nothing is really lost - `up` just has + to recreate the container - but don't expect `docker ps -a` to still + show it afterwards. +- **The final "Package .zip archives" step of `build()` has a pre-existing + bash syntax error** unrelated to compilation - the actual build already + succeeded by that point (look for `Built IfcOpenShell...` in the output), + so this is safe to ignore if you only need the raw artifacts under + `build/.../install/`, not packaged release zips. +- **`ready_repo` originally cloned the third-party dependency cache one + directory level too shallow** (`../build` instead of `build`, relative to + the repo root), so `nix/build-all.py` would never find it and silently + rebuild every dependency (boost, OCCT, CGAL, ...) from source - "did the + build finish in ~1 minute, or is it grinding for 40+ minutes reconfiguring + OCCT" is the tell. Fixed in this copy; if `up` seems to be building + dependencies that should already be cached, check `ready_repo`'s `cd` + targets first. +- **On a brand-new `UNIQUE_ID`/folder, `up` used to fail on the very first + run** because `ready_repo` tried to `docker exec` into the container + before `docker compose up -d` had created it. Also fixed in this copy + (container creation now happens first); if you see + `Error response from daemon: No such container` right after "Getting the + repo ready to build...", just run `up` again. +- **Root-partition disk space**: only the bind-mounted `/build` lives + on the host filesystem your repo is checked out on. Anything the + container writes *outside* that mount (stray files, apt/dnf state, etc.) + lives in the container's own writable layer under Docker's data root + (commonly `/var/lib/docker`, i.e. usually your root partition) - keep an + eye on `df -h /` if you're running several of these containers at once. diff --git a/docker/compose.yaml b/docker/compose.yaml new file mode 100644 index 0000000000..005a660e85 --- /dev/null +++ b/docker/compose.yaml @@ -0,0 +1,13 @@ +name: ifcopenshell-${UNIQUE_ID} +services: + ifcopenshell: + container_name: ifcopenshell-${UNIQUE_ID} + image: ifcopenshell-build-env:updated + volumes: + - type: bind + source: ../ + target: /__w/IfcOpenShell/IfcOpenShell + - ccache:/ccache + +volumes: + ccache: diff --git a/docker/ifcos_env b/docker/ifcos_env new file mode 100755 index 0000000000..3e91819114 --- /dev/null +++ b/docker/ifcos_env @@ -0,0 +1,233 @@ +#!/bin/bash + +# ================== CONFIG ================== +SCRIPT_NAME=$(basename "$0") +ENV_FILE=".env" +WORKDIR="/__w/IfcOpenShell/IfcOpenShell" +NAMEPREFIX=ifcopenshell + +function set_env() { + # Load .env file if it exists + if [[ -f "$ENV_FILE" ]]; then + set -a + source "$ENV_FILE" + set +a + echo "✅ Loaded environment variables from $ENV_FILE" + else + echo "⚠️ No $ENV_FILE found, proceeding without it." + fi +} + +set_env + +# ================ FUNCTIONS ================= + +function create() { + echo "⭐ Creating image: ifcopenshell-build-env" + docker build -f Dockerfile_init -t ifcopenshell-build-env:updated . +} + +function update() { + echo "⚡ Updating image: ifcopenshell-build-env" + docker build -f Dockerfile_update -t ifcopenshell-build-env:updated . +} + +function up() { + echo "🚀 Starting stack: ifcopenshell-${UNIQUE_ID}" + unique # Update UNIQUE_ID first + docker compose up -d "$@" # Container must exist before ready_repo can exec into it. + ready_repo # Ensure repo is recursive, and the build repo is in place. +} + +function down() { + echo "🛑 Stopping stack: ifcopenshell-${UNIQUE_ID}" + docker compose down "$@" +} + +function restart() { + echo "🔄 Restarting stack..." + down + up +} + +function logs() { + echo "📜 Showing logs..." + docker compose logs -f "$@" +} + +function ps() { + docker compose ps +} + +function config() { + echo "🔍 Validated compose configuration:" + docker compose config +} + +function remove() { + echo "🔥 Removing stack: ifcopenshell-${UNIQUE_ID}" + docker compose rm "$@" +} + +function unique() { + echo "🔧 Making stack name folder specific..." + + REGEX="^UNIQUE_ID=" + + if [[ ! -f "$FILE" ]] || ! grep -qE "$REGEX" "$ENV_FILE"; then + echo -e "\nUNIQUE_ID=dummy\n" >> "$ENV_FILE" + fi + + export UNIQUE_ID="$(pwd | sha256sum | cut -c -8)" && sed -sin "s/^UNIQUE_ID=.*$/UNIQUE_ID=${UNIQUE_ID}/" .env + + set_env +} + +function ready_repo() { + echo "👍 Getting the repo ready to build..." + docker exec -i -w "${WORKDIR}" "${NAMEPREFIX}-${UNIQUE_ID}" bash -c ' + set -euo pipefail # Recommended for robustness + + git submodule update --init --recursive + + if [[ ! -d "build" ]]; then + git clone -b rockylinux9-x64 https://github.com/IfcOpenShell/build-outputs.git build + else + cd build + git pull + cd .. + fi + + if [[ ! -d "build/Linux/x86_64/install/boost-1.86.0/" ]]; then + cd build + uv run ../nix/cache_dependencies.py unpack + cd .. + fi + ' +} + +function build() { + echo "☕ Execute the build, go make yourself a cuppa... I'll be a while" + local BUILD_TARGET="$1" + + docker exec -i -w "${WORKDIR}" -e PY_TGT="${PY_TGT}" -e BUILD_TARGET="${BUILD_TARGET}" "${NAMEPREFIX}-${UNIQUE_ID}" bash -c ' + set -o pipefail + CXXFLAGS="-O3" CFLAGS="-O3 ${DARWIN_C_SOURCE}" ADD_COMMIT_SHA=1 BUILD_CFG=Release uv run ./nix/build-all.py -v ${PY_TGT:+-$PY_TGT} --diskcleanup ${BUILD_TARGET} 2>&1 | tee build.log + ' + echo "🎒 Pack Dependencies" + docker exec -i -w "${WORKDIR}" "${NAMEPREFIX}-${UNIQUE_ID}" bash -c ' + cd build + uv run ../nix/cache_dependencies.py pack + ' + + echo "🎁 Package .zip archives" + docker exec -i -w "${WORKDIR}" -e GITHUB_SHA="$(git rev-parse HEAD)" "${NAMEPREFIX}-${UNIQUE_ID}" bash -c ' + OUTPUT_DIR=${PWD}/output + VERSION=v`cat VERSION` + mkdir -p ${OUTPUT_DIR} + cd ./build/`uname`/*/install/ifcopenshell + + ls -d python-* | while read py_version; do + postfix=`echo ${py_version: -1} | sed s/[0-9]//` + numbers=`echo $py_version | grep -oE "[0-9]+\.[0-9]+" | tr -d "."` + py_version_major=python-${numbers}$postfix + pushd . > /dev/null + cd $py_version + if [ ! -d ifcopenshell ]; then + mkdir ../ifcopenshell_ + mv * ../ifcopenshell_ + mv ../ifcopenshell_ ifcopenshell + fi + [ -d ifcopenshell/__pycache__ ] && rm -rf ifcopenshell/__pycache__ + find ifcopenshell -name "*.pyc" -delete + zip -r -qq ifcopenshell-${py_version_major}-${VERSION}-${GITHUB_SHA:0:7}-linux64.zip ifcopenshell/* + mv *.zip ${OUTPUT_DIR}/ + popd > /dev/null + done + + cd bin + if compgen -G "./*.zip" > /dev/null; then + rm *.zip 2>&1 >/dev/null || true + ls | while read exe; do + zip -qq -r ${exe}-${VERSION}-${GITHUB_SHA:0:7}-linux64.zip $exe + done + mv *.zip ${OUTPUT_DIR}/ + cd .. + ' +} + +function attach() { + echo "🔦 Connect to interactive shell" + docker exec -it -w "${WORKDIR}" "${NAMEPREFIX}-${UNIQUE_ID}" /bin/bash +} + +function try() { + echo "🚴 Push artefacts to the Blender so you can test" + # check if SRC and TGT set if not explain what to do. + #if [[ ! -f "$FILE" ]] || ! grep -qE "$REGEX" "$ENV_FILE"; then + # echo -e "\nUNIQUE_ID=dummy\n" >> "$ENV_FILE" + #fi +} + +function clean() { + echo "💎 Clean the build and output folder up" + if [[ -d "../build" ]]; then + rm -rf ../build + fi + if [[ -d "../output" ]]; then + rm -rf ../output + fi +} + + +function help() { + cat < + +Available commands: + create Create the image based on rocky9 + update Update installed packages + up Start services (docker compose up -d) + down Stop and remove containers + restart Restart the stack + build Execute the build + attach Connect to interactive shell + try Copy wrapper files to Blender + clean Remove build and output folders + logs Follow logs + ps Show running containers + config Validate and show compose config + remove Remove the stack + help Show this help + +Environment variables from .env are automatically loaded. +EOF +} + +# ================= MAIN ================= + +case "$1" in + create) create ;; + update) update ;; + up|start) up "${@:2}" ;; + down|stop) down "${@:2}" ;; + restart) restart ;; + build) build "${@:2}" ;; + attach) attach ;; + try) try ;; + clean) clean ;; + logs) logs "${@:2}" ;; + ps) ps ;; + config) config ;; + remove) remove ;; + help|-h|--help) help ;; + "") + echo "❌ No command provided." + help + ;; + *) + echo "❌ Unknown command: $1" + echo "Type './$SCRIPT_NAME help' for available commands." + exit 1 + ;; +esac