diff --git a/docker/.dockerignore b/docker/.dockerignore index 2979bddbab..b91616d2c0 100644 --- a/docker/.dockerignore +++ b/docker/.dockerignore @@ -1,4 +1,3 @@ .env *.pyc __pycache__ -redis-data diff --git a/docker/.gitignore b/docker/.gitignore index 2979bddbab..b91616d2c0 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,4 +1,3 @@ .env *.pyc __pycache__ -redis-data diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..fbc4c23baf --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,56 @@ +FROM rockylinux:9 + +# Update system, enable CRB (needed by some EPEL packages) and install EPEL, +# then install required packages + some common tools for a bit of command +# line comfort. Combined into one layer so a later `create` always installs +# against packages from the same dnf update, rather than layering fresh +# installs on top of a stale cached "update" layer. +RUN dnf update -y && \ + dnf install -y epel-release && \ + dnf config-manager --set-enabled crb && \ + dnf install -y --allowerasing --setopt=install_weak_deps=False --setopt=tsflags=nodocs \ + bash-completion vim git curl wget which tree htop sudo \ + gcc gcc-c++ autoconf automake bison make zip cmake \ + python3 python3-pip \ + bzip2 patch mesa-libGL-devel libffi-devel fontconfig-devel \ + sqlite-devel bzip2-devel zlib-devel openssl-devel xz-devel \ + readline-devel ncurses-devel libuuid-devel git-lfs \ + findutils xz byacc ccache && \ + git lfs install --system && \ + dnf clean all && \ + rm -rf /var/cache/dnf + +# Trust bind-mounted repos regardless of which user (root or builder) or host +# UID owns them, rather than a per-user config that only one of them sees. +RUN git config --system --add safe.directory '*' + +# Configure ccache. CCACHE_MAXSIZE (not `ccache -M`) because /ccache is a +# volume mount point at runtime - anything `ccache -M` writes to a config +# file under it during this build gets shadowed once the real volume is +# mounted, so the size cap only actually takes effect via the env var. +ENV CCACHE_DIR=/ccache +ENV CCACHE_MAXSIZE=5G +ENV PATH="/usr/lib/ccache:$PATH" + +# Non-root user matching the host UID/GID that bind-mounts the repo (default +# 1000:1000, the common single-user-Linux-box case), so files the build +# creates under the mount keep sane, non-root ownership on the host side. +# Override with --build-arg USER_UID=$(id -u) --build-arg USER_GID=$(id -g) +# if your host user has a different UID/GID. +ARG USER_UID=1000 +ARG USER_GID=1000 +RUN groupadd -g "${USER_GID}" builder \ + && useradd -m -u "${USER_UID}" -g "${USER_GID}" -s /bin/bash builder \ + && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder + +# Copied while still root: /bin is not writable by the builder user. +COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /uvx /bin/ + +USER builder +WORKDIR /__w/IfcOpenShell/IfcOpenShell + +# Installed as builder so managed Python interpreters land under builder's +# $HOME, matching the user that actually runs the build. +RUN uv python install + +CMD ["sleep", "infinity"] diff --git a/docker/Dockerfile_init b/docker/Dockerfile_init deleted file mode 100644 index a48ffe3262..0000000000 --- a/docker/Dockerfile_init +++ /dev/null @@ -1,40 +0,0 @@ -FROM rockylinux:9 - -# Update system -RUN dnf update -y - -# Enable CRB (needed by some EPEL packages) and install EPEL -RUN dnf install -y epel-release && \ - dnf config-manager --set-enabled crb && \ - dnf install -y --setopt=install_weak_deps=False \ - ccache - -# Install required packages + some common tools for a bit of command line comfort -RUN dnf install -y --allowerasing bash-completion vim git curl wget which tree htop \ - gcc gcc-c++ git autoconf automake bison make zip cmake python3 python3-pip \ - bzip2 patch mesa-libGL-devel libffi-devel fontconfig-devel \ - sqlite-devel bzip2-devel zlib-devel openssl-devel xz-devel \ - readline-devel ncurses-devel libffi-devel libuuid-devel git-lfs \ - findutils xz byacc ccache - -# Clean the caches -RUN dnf clean all && \ - rm -rf /var/cache/dnf - -# Configure ccache -ENV CCACHE_DIR=/ccache -ENV PATH="/usr/lib/ccache:$PATH" - -# Optional: Set a reasonable cache size limit (adjust as needed) -RUN ccache -M 5G # e.g. 5 GB max - -# Setup uv -COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /uvx /bin/ - -# Install Python -RUN uv python install - -# Prevent dubious ownership error -RUN git config --global --add safe.directory '*' - -CMD ["sleep", "infinity"] diff --git a/docker/Dockerfile_update b/docker/Dockerfile_update deleted file mode 100644 index a6ee2b62fa..0000000000 --- a/docker/Dockerfile_update +++ /dev/null @@ -1,16 +0,0 @@ -FROM ifcopenshell-build-env:updated - -# Update system -RUN dnf update -y - -# Clean the caches -RUN dnf clean all && \ - rm -rf /var/cache/dnf - -# Setup uv -COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /uvx /bin/ - -# Install Python -RUN uv python install - -CMD ["sleep", "infinity"] diff --git a/docker/SKILL.md b/docker/SKILL.md index 4ddc3ce18c..c9df338a56 100644 --- a/docker/SKILL.md +++ b/docker/SKILL.md @@ -13,13 +13,11 @@ description: >- # Building IfcOpenShell locally with docker/ifcos_env -`docker/` is a small toolchain (see `docker/README.md` for the original -author's own description and design rationale - read that first for the -*why*; this file is the practical *how*, distilled from actually driving it -end-to-end) that mirrors the project's GitHub Actions build environment -locally, with a persistent container and ccache so repeat builds are fast. -Pure-Python changes don't need any of this - only reach for it when you need -a real compiled `_ifcopenshell_wrapper*.so` or `IfcConvert` binary. +`docker/` mirrors the project's GitHub Actions build environment locally, +in a persistent, non-root container with ccache so repeat builds are fast. +See `docker/README.md` for the design rationale. Pure-Python changes don't +need any of this - only reach for it when you need a real compiled +`_ifcopenshell_wrapper*.so` or `IfcConvert` binary. ## Placement @@ -29,12 +27,12 @@ to build (sibling of `src/`, `cmake/`, etc.) - `compose.yaml` and itself sits, and bind-mount it into the container. If you're setting this up in a fresh clone, copy the whole `docker/` directory there first. -## First-time setup +## Setup ```bash cd docker -./ifcos_env create # build the base image (shared across all your clones/checkouts by name, so usually instant after the first time anywhere) -./ifcos_env up # start the container, clone+unpack the third-party dependency cache (~10GB, one-time per container) +./ifcos_env create # build the image (shared by name across all your clones/checkouts, so usually instant after the first time anywhere) +./ifcos_env up # create + start the container, clone/unpack the third-party dependency cache (~10GB, one-time per container) ./ifcos_env build # full build: all deps + IfcParse + IfcGeom + IfcConvert + the Python wrapper, for one Python version ``` @@ -47,6 +45,25 @@ A full first build takes ~1.5 hours (mostly compiling IfcOpenShell's own C++, not the cached third-party deps). After that, ccache makes incremental rebuilds of a couple of touched `.cpp` files **under a minute**. +## Container lifecycle + +The container is long-lived (`sleep infinity`) so exec'd commands and +ccache state persist between builds. Commands map directly onto Docker +Compose's own container-vs-image distinction: + +```bash +./ifcos_env up # create the container if it doesn't exist, then start it (runs ready_repo too) +./ifcos_env stop # stop the container, keep it around +./ifcos_env start # start it back up (same container, same filesystem layer) +./ifcos_env restart # stop, then start +./ifcos_env down # remove the container (and its network) entirely +./ifcos_env recreate # down, then up - a fresh container +``` + +Named volumes (`ccache`) and the bind-mounted repo/`build/` are unaffected +by `down`/`recreate` - only the container itself goes away, and `up` +recreates it from the image. + ## Fast iteration Pass a target to `build` to skip the parts you don't need: @@ -64,14 +81,15 @@ exercising the Python API, build `IfcOpenShell-Python`. ## Where the artifacts land Build output goes to `/build/Linux/x86_64/install/` on the host -(bind-mounted, not just inside the container): +(bind-mounted, not just inside the container), owned by you (see +"Container user" below): - `ifcopenshell/bin/IfcConvert` - the CLI binary - `python-/lib/python/site-packages/ifcopenshell/_ifcopenshell_wrapper*.so` and `ifcopenshell_wrapper.py` - the compiled wrapper + its generated Python glue -## Wiring the build into a checkout for testing +## Testing against a checkout (automated / AI-driven) `_ifcopenshell_wrapper*.so` and `ifcopenshell_wrapper.py` are already gitignored under `src/ifcopenshell-python/ifcopenshell/`, which is exactly @@ -98,40 +116,71 @@ tests.) You'll need the matching Python version's `pip install`s too modules under test import) since this is a bare interpreter, not the project's pixi env. -## Known gotchas (some fixed in this copy, watch for them if you're on an -## older/different copy of this script) +**This is the pattern to use for automated or AI-driven verification.** +Don't use `try` (below) for that - it overwrites files in a real, live +Blender installation, which isn't something an automated/AI workflow +should ever do without the human explicitly asking for it in the moment. -- **`try` is an unimplemented stub** - it prints a message and does - nothing. If you want the wrapper pushed straight into a Blender - extensions folder for manual testing, do the copy yourself (see the - README's example path) rather than relying on `try`. -- **`stop`/`down` removes the container**, it does not pause it (it's - literally `docker compose down`). Named volumes (ccache) and the - bind-mounted `build/` survive, so nothing is really lost - `up` just has - to recreate the container - but don't expect `docker ps -a` to still - show it afterwards. +## Testing in Blender itself (human only) + +`try` copies the built wrapper straight into your actual Blender/Bonsai +extension install, for manual in-Blender testing: + +```bash +./ifcos_env try +``` + +It reads `BLENDER_USER_RESOURCE` from `.env` - set this to wherever +Blender's user resource folder for the Bonsai extension actually lives on +your system, which depends on your own Blender setup: + +```bash +# in docker/.env +BLENDER_USER_RESOURCE=~/.config/blender/bonsai/ +``` + +`try` figures out the built Python version from `build/.../install/` +(disambiguating with `PY_TGT` if more than one version was built) and +copies the wrapper to +`$BLENDER_USER_RESOURCE/extensions/.local/lib/python/site-packages/ifcopenshell/`. + +## Container user + +The image runs as a non-root `builder` user, UID/GID matching your host +account (passed as `--build-arg` by `create` from `id -u`/`id -g`, so it +adjusts automatically - no manual flag needed even if you're not 1000:1000). +Files the build creates under the bind mount come out owned by you, not +root. Passwordless `sudo` is available inside the container (e.g. via +`attach`) for the rare case you need root for something ad hoc. + +If you're picking up an existing checkout that was previously built with +an older, root-based image, you may hit `Permission denied` the first time +you run `up`/`build` under the new image - `build/`, `.git/modules/`, the +`ccache` volume, `output/`, and `build.log` can all be left root-owned from +before. Fix it once via the container's own root (no host `sudo` needed): + +```bash +docker exec -u root -w /__w/IfcOpenShell/IfcOpenShell \ + chown -R "$(id -u)":"$(id -g)" .git/modules build output build.log /ccache +``` + +(`` is `ifcopenshell-` - see `docker ps -a`.) + +## Other things worth knowing + +- **Linux x64 only.** `compose.yaml` pins `platform: linux/amd64`; on an + ARM host (e.g. Apple Silicon) this build isn't available. - **The final "Package .zip archives" step of `build()` has a pre-existing - bash syntax error** unrelated to compilation - the actual build already + bash syntax error**, unrelated to compilation - the actual build already succeeded by that point (look for `Built IfcOpenShell...` in the output), so this is safe to ignore if you only need the raw artifacts under `build/.../install/`, not packaged release zips. -- **`ready_repo` originally cloned the third-party dependency cache one - directory level too shallow** (`../build` instead of `build`, relative to - the repo root), so `nix/build-all.py` would never find it and silently - rebuild every dependency (boost, OCCT, CGAL, ...) from source - "did the - build finish in ~1 minute, or is it grinding for 40+ minutes reconfiguring - OCCT" is the tell. Fixed in this copy; if `up` seems to be building - dependencies that should already be cached, check `ready_repo`'s `cd` - targets first. -- **On a brand-new `UNIQUE_ID`/folder, `up` used to fail on the very first - run** because `ready_repo` tried to `docker exec` into the container - before `docker compose up -d` had created it. Also fixed in this copy - (container creation now happens first); if you see - `Error response from daemon: No such container` right after "Getting the - repo ready to build...", just run `up` again. -- **Root-partition disk space**: only the bind-mounted `/build` lives - on the host filesystem your repo is checked out on. Anything the - container writes *outside* that mount (stray files, apt/dnf state, etc.) - lives in the container's own writable layer under Docker's data root - (commonly `/var/lib/docker`, i.e. usually your root partition) - keep an - eye on `df -h /` if you're running several of these containers at once. +- **`test_mmaped_stream` and similar `USE_MMAP`-dependent tests will fail** + against this build - `nix/build-all.py` is invoked with `USE_MMAP=OFF` + here. Not a bug in your code if you see it fail. +- Only the bind-mounted `/build` lives on the host filesystem your + repo is checked out on. Anything the container writes *outside* that + mount lives in the container's own writable layer under Docker's data + root (commonly `/var/lib/docker`, i.e. usually your root partition) - + keep an eye on `df -h /` if you're running several of these containers + at once. diff --git a/docker/compose.yaml b/docker/compose.yaml index 005a660e85..9bc79234d1 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -3,6 +3,7 @@ services: ifcopenshell: container_name: ifcopenshell-${UNIQUE_ID} image: ifcopenshell-build-env:updated + platform: linux/amd64 volumes: - type: bind source: ../ diff --git a/docker/ifcos_env b/docker/ifcos_env index 3e91819114..4286421572 100755 --- a/docker/ifcos_env +++ b/docker/ifcos_env @@ -24,28 +24,60 @@ set_env function create() { echo "⭐ Creating image: ifcopenshell-build-env" - docker build -f Dockerfile_init -t ifcopenshell-build-env:updated . + docker build -f Dockerfile \ + --build-arg USER_UID="$(id -u)" --build-arg USER_GID="$(id -g)" \ + -t ifcopenshell-build-env:updated . } function update() { + # The Dockerfile always builds FROM a clean rockylinux:9 and does + # `dnf update -y` as its first step, so re-running create() is enough + # to get fresh packages. echo "⚡ Updating image: ifcopenshell-build-env" - docker build -f Dockerfile_update -t ifcopenshell-build-env:updated . + create } function up() { - echo "🚀 Starting stack: ifcopenshell-${UNIQUE_ID}" + # Creates the container if it doesn't exist yet (and starts it either + # way) - this is the one that needs ready_repo, since a freshly created + # container has no submodules/dependency cache in place yet. + echo "🚀 Creating/starting stack: ifcopenshell-${UNIQUE_ID}" unique # Update UNIQUE_ID first docker compose up -d "$@" # Container must exist before ready_repo can exec into it. ready_repo # Ensure repo is recursive, and the build repo is in place. } function down() { - echo "🛑 Stopping stack: ifcopenshell-${UNIQUE_ID}" + # Removes the container (and its network) entirely. Named volumes + # (ccache) and the bind-mounted repo/build/ survive; up() will recreate + # the container from scratch next time. + echo "🔥 Removing stack: ifcopenshell-${UNIQUE_ID}" docker compose down "$@" } +function stop() { + # Stops the existing container without removing it - the container, + # its filesystem layer, and its exec history all remain intact. + echo "🛑 Stopping stack: ifcopenshell-${UNIQUE_ID}" + docker compose stop "$@" +} + +function start() { + # Starts a previously-stopped container back up. Does nothing (and + # won't create anything) if the container doesn't exist - use up() for + # that. + echo "▶️ Starting stack: ifcopenshell-${UNIQUE_ID}" + docker compose start "$@" +} + function restart() { - echo "🔄 Restarting stack..." + echo "🔄 Restarting stack (stop, then start)..." + stop + start +} + +function recreate() { + echo "♻️ Recreating stack (down, then up)..." down up } @@ -65,21 +97,23 @@ function config() { } function remove() { - echo "🔥 Removing stack: ifcopenshell-${UNIQUE_ID}" + # Lower-level than down(): removes already-stopped containers without + # touching the compose network. Mostly useful after a plain stop(). + echo "🗑️ Removing stopped containers: ifcopenshell-${UNIQUE_ID}" docker compose rm "$@" } function unique() { echo "🔧 Making stack name folder specific..." - + REGEX="^UNIQUE_ID=" - - if [[ ! -f "$FILE" ]] || ! grep -qE "$REGEX" "$ENV_FILE"; then + + if [[ ! -f "$ENV_FILE" ]] || ! grep -qE "$REGEX" "$ENV_FILE"; then echo -e "\nUNIQUE_ID=dummy\n" >> "$ENV_FILE" fi - - export UNIQUE_ID="$(pwd | sha256sum | cut -c -8)" && sed -sin "s/^UNIQUE_ID=.*$/UNIQUE_ID=${UNIQUE_ID}/" .env - + + export UNIQUE_ID="$(pwd | sha256sum | cut -c -8)" && sed -si "s/^UNIQUE_ID=.*$/UNIQUE_ID=${UNIQUE_ID}/" "$ENV_FILE" + set_env } @@ -162,19 +196,70 @@ function attach() { } function try() { - echo "🚴 Push artefacts to the Blender so you can test" - # check if SRC and TGT set if not explain what to do. - #if [[ ! -f "$FILE" ]] || ! grep -qE "$REGEX" "$ENV_FILE"; then - # echo -e "\nUNIQUE_ID=dummy\n" >> "$ENV_FILE" - #fi + # Copies the freshly built wrapper into your actual Blender/Bonsai + # installation for manual, in-Blender testing. This is a human-only + # convenience: it overwrites files in your live Blender setup, so it's + # not something that should run unattended as part of an automated or + # AI-driven build/test loop (which should instead copy the wrapper into + # the repo's own src/ifcopenshell-python/ifcopenshell/ - see SKILL.md). + echo "🚴 Copying build artifacts into your Blender resource folder for testing" + + if [[ -z "${BLENDER_USER_RESOURCE:-}" ]]; then + echo "❌ BLENDER_USER_RESOURCE is not set in .env." + echo " Add a line pointing at wherever Blender's user resource folder for" + echo " the Bonsai extension actually is on your system, e.g.:" + echo " BLENDER_USER_RESOURCE=~/.config/blender/bonsai/" + return 1 + fi + + # Normalise: expand a leading ~ (in case it was quoted in .env and so + # never went through shell tilde-expansion when set_env sourced it), + # then resolve to an absolute, symlink-free path. + local resource="${BLENDER_USER_RESOURCE/#\~/$HOME}" + resource="$(realpath -m "$resource")" + + local install_dir="../build/Linux/x86_64/install" + local py_dirs=("$install_dir"/python-*) + if [[ ${#py_dirs[@]} -gt 1 && -n "${PY_TGT:-}" ]]; then + # PY_TGT is compact (py-311); the install dirs are dotted + # (python-3.11.8) - reinsert the dot (assumes a single-digit major + # version, true for the Python 3.x line) before matching. + local py_tgt_digits="${PY_TGT#py-}" + local py_tgt_dotted="${py_tgt_digits:0:1}.${py_tgt_digits:1}" + local filtered=() d + for d in "${py_dirs[@]}"; do + [[ "$(basename "$d")" == "python-${py_tgt_dotted}."* ]] && filtered+=("$d") + done + [[ ${#filtered[@]} -gt 0 ]] && py_dirs=("${filtered[@]}") + fi + if [[ ${#py_dirs[@]} -ne 1 || ! -d "${py_dirs[0]}" ]]; then + echo "❌ Expected exactly one built python-* dir under $install_dir, found ${#py_dirs[@]}." + echo " Run 'build' first, or set PY_TGT in .env to disambiguate a multi-version build." + return 1 + fi + + local py_minor + py_minor="$(basename "${py_dirs[0]}" | grep -oE '[0-9]+\.[0-9]+')" + local wrapper_dir="${py_dirs[0]}/lib/python${py_minor}/site-packages/ifcopenshell" + if [[ ! -f "$wrapper_dir/ifcopenshell_wrapper.py" ]]; then + echo "❌ Built wrapper not found at $wrapper_dir - run 'build' first." + return 1 + fi + + local target="$resource/extensions/.local/lib/python${py_minor}/site-packages/ifcopenshell" + mkdir -p "$target" + cp "$wrapper_dir"/_ifcopenshell_wrapper*.so "$target/" + cp "$wrapper_dir"/ifcopenshell_wrapper.py "$target/" + echo "✅ Copied wrapper into $target" } function clean() { + # Host-side only - doesn't touch the container, image, or ccache volume. echo "💎 Clean the build and output folder up" - if [[ -d "../build" ]]; then + if [[ -d "../build" ]]; then rm -rf ../build fi - if [[ -d "../output" ]]; then + if [[ -d "../output" ]]; then rm -rf ../output fi } @@ -185,22 +270,31 @@ function help() { Usage: ./$SCRIPT_NAME Available commands: - create Create the image based on rocky9 - update Update installed packages - up Start services (docker compose up -d) - down Stop and remove containers - restart Restart the stack - build Execute the build - attach Connect to interactive shell - try Copy wrapper files to Blender - clean Remove build and output folders - logs Follow logs + create Build the rocky9-based image + update Rebuild the image fresh, picking up OS package updates + up Create the container if it doesn't exist yet, and start it + down Remove the container entirely (docker compose down) + stop Stop the container without removing it + start Start a previously-stopped container + restart stop, then start (same container, no recreation) + recreate down, then up (fresh container) + build Execute the IfcOpenShell build + attach Connect to an interactive shell in the container + try Copy the built wrapper into your Blender resource folder + (human-only - see BLENDER_USER_RESOURCE below, and SKILL.md + for the AI/automated-testing equivalent) + clean Remove the build and output folders + logs Follow container logs ps Show running containers config Validate and show compose config - remove Remove the stack + remove Remove stopped containers (docker compose rm) help Show this help -Environment variables from .env are automatically loaded. +Environment variables from .env are automatically loaded, including: + PY_TGT Restrict the build to one Python version, e.g. py-311 + UNIQUE_ID Recalculated automatically on every 'up', don't set by hand + BLENDER_USER_RESOURCE Where 'try' copies the wrapper for manual testing, e.g. + ~/.config/blender/bonsai/ EOF } @@ -209,9 +303,12 @@ EOF case "$1" in create) create ;; update) update ;; - up|start) up "${@:2}" ;; - down|stop) down "${@:2}" ;; + up) up "${@:2}" ;; + down) down "${@:2}" ;; + stop) stop "${@:2}" ;; + start) start "${@:2}" ;; restart) restart ;; + recreate) recreate ;; build) build "${@:2}" ;; attach) attach ;; try) try ;;