chore: switch Dockerfile to slim Python + uv lockfile

.github/workflows/deploy-image.yml

@@ -3,7 +3,6 @@ on: [push, workflow_dispatch]
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  BRANCH: ${{ github.ref }}
 jobs:
   build-and-push-image:
     runs-on: ubuntu-latest
@@ -26,25 +25,86 @@ jobs:
         with:
           driver-opts: network=host
 
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
+      # Two parallel tag sets. `dev` is the default (no suffix, e.g. `:latest`,
+      # `:develop`); `runtime` carries a `-runtime` suffix.
+      - name: Tags — dev (default)
+        id: meta-dev
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Build and export to Docker
+      - name: Tags — runtime
+        id: meta-runtime
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=-runtime,onlatest=true
+
+      # Build runtime first (smaller, used to smoke-test pipeline binaries)
+      - name: Build runtime (load)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: runtime
+          load: true
+          tags: ${{ steps.meta-runtime.outputs.tags }}
+          labels: ${{ steps.meta-runtime.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Smoke-test the binaries baked into the runtime image. Catches the
+      # class of regression where the image builds but a runtime tool
+      # (sextractor, weightwatcher) is missing or unrunnable.
+      - name: Test runtime — binaries
+        run: |
+          IMAGE=$(echo "${{ steps.meta-runtime.outputs.tags }}" | head -n1)
+          docker run --rm "$IMAGE" source-extractor --version
+          docker run --rm "$IMAGE" weightwatcher --version
+          docker run --rm "$IMAGE" psfex --version
+
+      - name: Test runtime — shapepipe entry point
+        run: |
+          IMAGE=$(echo "${{ steps.meta-runtime.outputs.tags }}" | head -n1)
+          docker run --rm "$IMAGE" shapepipe_run -c /app/example/config.ini
+
+      # Build dev (reuses cached `base` layer)
+      - name: Build dev (load)
         uses: docker/build-push-action@v6
         with:
+          context: .
+          target: dev
           load: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-dev.outputs.tags }}
+          labels: ${{ steps.meta-dev.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Verify the dev-only additions are present and runnable.
+      - name: Test dev — interactive tools and test extras
+        run: |
+          IMAGE=$(echo "${{ steps.meta-dev.outputs.tags }}" | head -n1)
+          docker run --rm "$IMAGE" vim --version | head -n1
+          docker run --rm "$IMAGE" rg --version | head -n1
+          docker run --rm "$IMAGE" pytest --version
 
-      - name: Test
-        run: docker run --rm ${{ steps.meta.outputs.tags }} shapepipe_run -c /app/example/config.ini
+      # Push both targets
+      - name: Push runtime
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          target: runtime
+          push: true
+          tags: ${{ steps.meta-runtime.outputs.tags }}
+          labels: ${{ steps.meta-runtime.outputs.labels }}
+          cache-from: type=gha
 
-      - name: Push
+      - name: Push dev
         uses: docker/build-push-action@v6
         with:
+          context: .
+          target: dev
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta-dev.outputs.tags }}
+          labels: ${{ steps.meta-dev.outputs.labels }}
+          cache-from: type=gha

Dockerfile

@@ -1,53 +1,130 @@
-FROM images.canfar.net/skaha/astroml:latest
+# syntax=docker/dockerfile:1.7
+#
+# Two-target image:
+#   --target runtime  →  minimal, for canfar batch jobs and downstream stacks
+#   --target dev      →  runtime + everyday CLI tools + all extras (test,
+#                        lint, doc, …); default if --target is omitted
+#
+# Both share the `base` stage (system deps + uv + lockfile copy), so the
+# heavy apt + wheel-resolution work is cached once.
+
+# ----------------------------------------------------------------------
+# base — system deps shared by every target
+# ----------------------------------------------------------------------
+FROM python:3.12-slim-bookworm AS base
 
-# Metadata
 LABEL maintainer="martin.kilbinger@cea.fr"
-LABEL description="ShapePipe base image with common dependencies"
 
-# Install system dependencies needed for ShapePipe and WeightWatcher
-RUN apt-get update -o Acquire::ForceIPv4=true -y --quiet && \
+ENV SHELL=/bin/bash \
+    QT_QPA_PLATFORM=offscreen \
+    PIP_NO_CACHE_DIR=1 \
+    DEBIAN_FRONTEND=noninteractive \
+    LANG=C.UTF-8
+
+# System dependencies — three categories:
+#  - astromatic binaries (psfex, source-extractor, weightwatcher) ship as
+#    Debian packages on bookworm; preferred over building from source.
+#  - compilers and dev libs needed to build the heavier wheels (galsim,
+#    mpi4py, python-pysap, fitsio).
+#  - libgl1, proj, fftw at runtime for skyproj/PyQt5/galsim.
+RUN apt-get update -y --quiet && \
     apt-get install -y --no-install-recommends \
-    psfex source-extractor \
-    libproj-dev proj-bin && \
+        build-essential \
+        cmake \
+        gfortran \
+        git \
+        wget \
+        pkg-config \
+        libfftw3-dev libfftw3-bin \
+        libgsl-dev \
+        libcfitsio-dev \
+        libopenmpi-dev openmpi-bin \
+        libproj-dev proj-bin \
+        libgl1-mesa-glx \
+        psfex source-extractor weightwatcher && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# Build and install WeightWatcher from source
-ARG WW_VERSION=1.12
-RUN cd /tmp && \
-    wget --no-check-certificate https://github.com/astromatic/weightwatcher/archive/refs/tags/${WW_VERSION}.tar.gz && \
-    tar -xzf ${WW_VERSION}.tar.gz && \
-    rm ${WW_VERSION}.tar.gz
-RUN cd /tmp/weightwatcher-${WW_VERSION} && \
-    sed -i 's/^  prefstruct\tprefs;/extern prefstruct\tprefs;/' src/prefs.h && \
-    sed -i 's/^char\t\tgstr\[MAXCHAR\];/extern char\t\tgstr[MAXCHAR];/' src/globals.h && \
-    sed -i 's/^int\t\tbswapflag;/extern int\t\tbswapflag;/' src/fits/fitscat.h && \
-    sed -i '/preflist\.h/a prefstruct\tprefs;' src/prefs.c && \
-    sed -i '/xml\.h/a char\t\tgstr[MAXCHAR];' src/main.c && \
-    sed -i '/fitscat\.h/a int\t\tbswapflag;' src/fits/fitscat.c && \
-    ./configure --quiet && \
-    make --quiet && \
-    make install
-
-# Ensure astroml:latest conda Python 3.12 is used (Docker RUN does not source conda init)
-ENV PATH /opt/conda/bin:$PATH
-
-# Upgrade pip and install tools not part of the ShapePipe package
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir \
-        ipython==8.18.1 \
-        jupyterlab==4.3.1 \
-        snakemake==8.27.1
-
-# Set working directory and copy source code
+# uv — fast reproducible Python deps installer. pyproject.toml + uv.lock
+# are the SSOT; `uv sync --frozen` installs exactly what uv.lock specifies,
+# so upstream changes only land when we deliberately regenerate the lockfile.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
 WORKDIR /app
+COPY pyproject.toml uv.lock /app/
+
+# ----------------------------------------------------------------------
+# runtime — minimal target for batch jobs and downstream FROM clauses
+# ----------------------------------------------------------------------
+FROM base AS runtime
+LABEL description="ShapePipe runtime — slim Python + uv-frozen deps"
+
+# Lockfile-frozen Python deps + jupyter + fitsio. Test/lint/doc extras
+# are intentionally left out here; they live in the dev target.
+RUN uv sync --frozen --no-install-project --extra jupyter --extra fitsio
+
+# Copy the source and install shapepipe into the same venv.
 COPY . /app/.
-RUN chown -R root:root /app && chmod -R u+rwX /app
+# go+rwX so non-root users on canfar/skaha can read/traverse /app and
+# write into the venv when they need to (e.g. uv add for ad-hoc deps).
+RUN chmod -R go+rwX /app && \
+    uv pip install --no-deps -e . && \
+    for ext in .py .sh .bash; do \
+        for script in /app/scripts/*/*$ext; do \
+            link_name=$(basename $script $ext); \
+            ln -s $script /usr/local/bin/$link_name; \
+        done; \
+    done
+
+# Activate the uv-managed venv on container start so shapepipe_run etc
+# resolve against it without explicit activation.
+ENV PATH="/app/.venv/bin:${PATH}" \
+    VIRTUAL_ENV=/app/.venv
+
+# ----------------------------------------------------------------------
+# dev — everyday working environment (default target)
+# ----------------------------------------------------------------------
+FROM base AS dev
+LABEL description="ShapePipe dev — runtime + interactive CLI tools + all extras"
+
+# Interactive tools for working inside the container. Curated subset of
+# cailmdaley/containers focused on the search/edit/process loop; heavier
+# tooling (neovim, polspice, quarto, zellij) is intentionally not here.
+RUN apt-get update -y --quiet && \
+    apt-get install -y --no-install-recommends \
+        vim \
+        less \
+        tmux \
+        htop \
+        procps \
+        ripgrep \
+        fd-find \
+        jq \
+        bat \
+        curl \
+        ca-certificates \
+        git-lfs \
+        rsync \
+        unzip zip \
+        openssh-client \
+        locales && \
+    if command -v batcat >/dev/null; then ln -sf "$(command -v batcat)" /usr/local/bin/bat; fi && \
+    if command -v fdfind >/dev/null; then ln -sf "$(command -v fdfind)" /usr/local/bin/fd; fi && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# All extras pre-installed (dev = doc + jupyter + lint + release + test +
+# fitsio). Pre-installing avoids the read-only-fs failure Martin hit when
+# trying to live `uv sync --extra test` inside the runtime image on canfar.
+RUN uv sync --frozen --no-install-project --extra dev
 
-# Install ShapePipe and its dependencies (including fitsio optional extra)
-RUN pip install --no-cache-dir -e ".[fitsio]" && \
+COPY . /app/.
+RUN chmod -R go+rwX /app && \
+    uv pip install --no-deps -e . && \
     for ext in .py .sh .bash; do \
         for script in /app/scripts/*/*$ext; do \
             link_name=$(basename $script $ext); \
             ln -s $script /usr/local/bin/$link_name; \
         done; \
     done
+
+ENV PATH="/app/.venv/bin:${PATH}" \
+    VIRTUAL_ENV=/app/.venv

pyproject.toml

@@ -10,32 +10,39 @@ authors = [
 ]
 license = { "file" = "LICENSE" }
 requires-python = ">=3.12"
+# Abstract runtime constraints. Exact versions live in `uv.lock` — pyproject
+# is the single source of truth for *what* shapepipe needs, the lockfile for
+# *which versions*. Minimums signal the major-version target; uv.lock pins
+# the actual installed versions. Bump minimums only when crossing a major
+# version line or when a feature actually requires a specific release.
 dependencies = [
-    "astropy>=5.2",
+    "astropy>=7.0",          # major 6 → 7
     "astroquery",
     "canfar",
-    "cs_util>=0.1",
-    "galsim>=2.5.3",
+    "cs_util>=0.1.9",
+    "galsim>=2.8",
     "h5py",
-    "joblib>=0.13",
-    "matplotlib>=3.8.4",
+    "joblib>=1.4",
+    "matplotlib>=3.10",
     "mccd>=1.2.4",
-    "modopt>=1.2",
-    "mpi4py",
-    "numba>=0.58.1",
-    "numpy>=1.14",
-    "pandas",
+    "modopt>=1.6",
+    "mpi4py>=4.0",
+    "numba>=0.59",           # numpy 2 support
+    "numpy>=2.0",            # major 1 → 2
+    "pandas>=3.0",           # major 2 → 3
     "python-dateutil",
-    "python-pysap>=0.2.1",
+    "python-pysap>=0.3",
     "PyQt5",
     "pyqtgraph",
-    "reproject",
-    "sf_tools",
-    "skaha",
-    "sqlitedict",
+    "reproject>=0.19",
+    "sf_tools>=2.0.4",
+    "skaha>=1.7",
+    "sqlitedict>=2.0",
     "termcolor",
-    "tqdm",
-    "vos",
+    "tqdm>=4.63",
+    "vos>=3.6",
+    # ngmix is pinned to Axel's stable_version branch until upstream ngmix
+    # absorbs the fixes — tracked separately, do not modernize this line.
     "ngmix @ git+https://github.com/aguinot/ngmix@stable_version",
 ]
 
@@ -48,6 +55,11 @@ doc = [
     "sphinxcontrib-bibtex",
     "sphinx-book-theme"
 ]
+jupyter = [
+    "ipython>=8.18",
+    "jupyterlab>=4.3",
+    "snakemake>=8.27",
+]
 lint = [
     "black",
     "isort"
@@ -57,13 +69,13 @@ release = [
     "twine",
 ]
 test = [
-    "pytest",
-    "pytest-cov",
-    "pytest-pycodestyle",
-    "pytest-pydocstyle"
+    "pytest>=8.3",
+    "pytest-cov>=5.0",
+    "pytest-pycodestyle>=2.4",
+    "pytest-pydocstyle>=2.4",
 ]
 fitsio = ["fitsio"]
-dev = ["shapepipe[doc,lint,release,test]"]
+dev = ["shapepipe[doc,jupyter,lint,release,test,fitsio]"]
 
 [project.scripts]
 shapepipe_run = "shapepipe.shapepipe_run:main"