From 5a637a404e0a3afee951c210ce3635d744c63c70 Mon Sep 17 00:00:00 2001 From: Rob Boerman Date: Mon, 28 Sep 2026 08:41:11 +0200 Subject: [PATCH] fix(podman): credit bounds only where podman reaches the user manager, and three venue leftovers (#453) Bounds. podmanBoundsDelegated read podman info's host.cgroupControllers, which is the caller's own cgroup, not what is delegated to the account: it listed all five controllers with nothing applied, and memory and pids where cpu was applied. Measured on Fedora 44 / Podman 5.8.1 and Ubuntu 24.04 / 4.9.3 with the venue's own argv, the bounds apply when user@.service runs with pids, memory and cpu delegated and Podman reaches it; with no user manager, or with Podman unable to reach it over D-Bus (it then falls back to cgroupfs under the caller's root-owned cgroup), they do not. Credit now needs that cgroup's controllers and either the systemd cgroup manager or the worker itself inside user@.service. Doctor names each cause with its own fix, reads linger and warns when the manager lives only for a session; the conformance job carries an isolation floor. up, doctor and the preflight. A proxy counts as present only when running, on the pinned digest, with its measured entrypoint and command (Podman 4.9's string form normalised) and with this folder's two mounts, each judged on its own (unknown only under a VM path prefix or on macOS/Windows; on Linux a missing source is stale). A current stopped proxy is started, a paused one unpaused, a stale one replaced only with consent (never under --yes while job networks are attached), quietly reusing the egress network. The worker's pre-spend preflight admits only a running proxy; paused, exited, dead and created refuse; any other state is retried once, then fails, naming the proxy and its state. doctor reads the deployment's venue keys, VALKEY_URL, PI_PROVIDER and the provider key from .env through one allowlist and the service's own loader, says the source, never prints a secret (URLs show only scheme://host:port/db) and prints fleet names without control bytes; the podman image fix lines no longer suggest docker save | podman load. init. The next steps follow up's venue decision: the podman ladder (linger, pull, up, doctor, service install, doctor --live) on a podman-only host, none when the venue is unknown, today's docker text otherwise. Specs: DES-PODMAN-NATIVE-ROOTLESS-BACKEND, DES-PODMAN-STACK-AS-QUADLET-UNITS, REQ-DEPLOYMENT-BOOTSTRAP, INT-LIVE-PROBE-CONTRACT, INT-EGRESS-POLICY-CONTRACT. Signed-off-by: Rob Boerman --- .github/scripts/podman-conformance.mjs | 9 +- docs/backends.md | 6 +- docs/podman.md | 64 +++- specs/design.md | 80 ++++- specs/interfaces.md | 13 +- specs/requirements.md | 33 +- worker/src/backend-podman.mjs | 103 +++++- worker/src/backends.mjs | 19 +- worker/src/cli.mjs | 4 +- worker/src/deployment-venue.mjs | 94 ++++++ worker/src/doctor.mjs | 320 +++++++++++++++++-- worker/src/egress-proxy-state.mjs | 151 +++++++++ worker/src/egress.mjs | 21 +- worker/src/init.mjs | 65 +++- worker/src/live-probes.mjs | 20 +- worker/src/processor.mjs | 10 +- worker/src/up.mjs | 207 ++++++------ worker/test/backend-podman.test.mjs | 115 ++++++- worker/test/cli.test.mjs | 17 + worker/test/doctor.test.mjs | 400 +++++++++++++++++++++--- worker/test/egress-proxy-state.test.mjs | 98 ++++++ worker/test/egress.test.mjs | 34 +- worker/test/init.test.mjs | 100 ++++++ worker/test/processor.test.mjs | 14 + worker/test/sandbox.test.mjs | 11 +- worker/test/start-wiring.test.mjs | 26 +- worker/test/up.test.mjs | 293 +++++++++++++++-- 27 files changed, 2046 insertions(+), 281 deletions(-) create mode 100644 worker/src/deployment-venue.mjs create mode 100644 worker/src/egress-proxy-state.mjs create mode 100644 worker/test/egress-proxy-state.test.mjs diff --git a/.github/scripts/podman-conformance.mjs b/.github/scripts/podman-conformance.mjs index 228ff96e..102cdb38 100644 --- a/.github/scripts/podman-conformance.mjs +++ b/.github/scripts/podman-conformance.mjs @@ -132,6 +132,10 @@ const backend = makePodmanBackend({ onOutput: () => {}, euid, egid, + // An isolation floor (issue #453, gate round 1), so the bounds observation must HOLD for the probe to be admitted: + // without it `observationPreflight` admitted every probe whatever `podmanBoundsDelegated` said, and an observation + // reading the wrong path would have stayed green here. The runner's pdjob has linger, a user bus and the drop-in. + backendFloor: { isolation: "enforced" }, }); let probes = 0; @@ -205,8 +209,9 @@ const withBrokenEnumeration = () => makeReaper({ log: () => {}, bin: `pi-dispatc */ async function egressCanary() { if (armed !== true) return { results: [], proxyRunning: null }; - const state = await podman(["inspect", "--format={{.State.Running}}", proxy]); - const proxyRunning = state.code === 0 && state.stdout.trim() === "true"; + // `.State.Status` "running", as the worker's egress preflight reads it (issue #453). + const state = await podman(["inspect", "--format={{.State.Status}}", proxy]); + const proxyRunning = state.code === 0 && state.stdout.trim() === "running"; if (!proxyRunning) return { results: [], proxyRunning }; const canary = await runEgressCanary({ run: liveRunVia(spawn, { bin: "podman" }), bin: "podman", proxy, image, pid: process.pid, user: decision.user }); for (const check of canary.checks) if (!check.ok) console.error(`podman-conformance: ${check.label}`); diff --git a/docs/backends.md b/docs/backends.md index 3dd604fe..b293670a 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -123,8 +123,10 @@ ones adapters get wrong: worker runs the `podman` CLI as its own account and each job as that account's uid with `--userns=keep-id`. Rootless Podman through its Docker API is still refused. - **`podman` declares every word `enforced`**, three of them only while observed from `podman info` and this - account's Podman files: `isolation` (`podmanBoundsDelegated`: cgroup v2 with the pids, memory and cpu - controllers delegated, since without delegation rootless Podman accepts the bounds and applies none), + account's Podman files: `isolation` (`podmanBoundsDelegated`: cgroup v2, the account's systemd user manager + running with the pids, memory and cpu controllers delegated to it, and Podman putting containers under it, through + the `systemd` cgroup manager or a worker running inside that manager; otherwise rootless Podman accepts the bounds + and applies none, measured in issue #453 and `docs/podman.md` step 3), `mountSet` (`podmanAddsNoMounts`: the mounts.conf that applies is empty and nothing else adds a mount) and `credentialTransit` (`podmanServiceLocal`: `serviceIsRemote` is false). `nonRoot` is `enforced` there, where `local` only asserts it, because the argv always carries the worker's own non-zero uid and keep-id maps it diff --git a/docs/podman.md b/docs/podman.md index 91733a12..6a0b99d9 100644 --- a/docs/podman.md +++ b/docs/podman.md @@ -342,13 +342,48 @@ an unprivileged account (uid 1234), on 2026-09-25. Run everything below as the w environment, since Podman then falls back to `/run/user/`. That hand-written system unit still works, but `pi-dispatch service install` installs this venue's worker in USER scope (it refuses `--system` here, step 6), and linger is then also what starts the user manager, and with it the worker and the Quadlet units, at boot. -3. **cgroup v2 with the controllers delegated.** `podman info --format '{{.Host.CgroupControllers}}'` must list `cpu`, - `memory` and `pids`. Fedora delegates all of them to user sessions (measured: `cpuset cpu io memory pids`); where a - distribution does not, a drop-in for `user@.service` with `Delegate=cpu cpuset io memory pids` does. Leave `cgroups` - unset in every containers.conf the account reads: measured with `--cgroups=disabled`, `--memory` and `--pids-limit` - are accepted and silently not applied, and the bounds read back as `max`. The worker credits `isolation` only while - it observes the delegation (`podmanBoundsDelegated`), and `pi-dispatch doctor --live` reads `pids.max` and - `memory.max` back off a real container. + A hand-written SYSTEM unit with `User=` must also be ordered after that user manager, which nothing else orders + it after: add `Wants=user@.service` and `After=user@.service` to its `[Unit]` section. Measured (issue + #453, Fedora 44): without them, with `PI_BACKEND_FLOOR=isolation=enforced`, the worker started before + `user@.service`, found no user manager, refused to boot with exit 2, and stayed failed after the manager came + up, since `RestartPreventExitStatus=2` keeps a configuration refusal from restarting; with the two lines it started + after the manager and ran. Without an isolation floor the same race costs the jobs picked up before the manager + runs their bounds (step 3). The user-scope worker `service install` writes runs inside the manager and needs neither. +3. **cgroup v2, a systemd user manager running for the account with the controllers delegated to it, and Podman + putting its containers under it.** `cat /sys/fs/cgroup/user.slice/user-.slice/user@.service/cgroup.controllers` + must list `cpu`, `memory` and `pids`. Fedora delegates all of them to user managers (measured: `cpuset cpu io + memory pids`); where a distribution does not, a drop-in for `user@.service` with `Delegate=cpu cpuset io memory + pids` does. With a controller missing a job cannot have that bound: measured for cpu, Podman 5.8.1 refused to + start a container carrying `--cpus` (exit 126, "controller `cpu` is not available"). The file is absent while the + account's `user@.service` is inactive (measured, issue #453; systemd removes a stopped unit's cgroup). What + decided the bounds in every case measured (Fedora 44, Podman 5.8.1, the venue's own argv; issue #453): + - **no user manager** (linger off, the account reached through `sudo -iu`, which starts none): Podman put the job in + the caller's own root-owned cgroup, `pids.max`, `memory.max` and `cpu.max` read back `max`, and the run exited 0, + whether Podman's cgroup manager was `systemd` (it fell back to `cgroupfs` with a warning) or `cgroupfs`; + - **a user manager Podman reaches** (`podman info --format '{{.Host.CgroupManager}}'` says `systemd`): applied. A + plain ssh login started the manager (pam_systemd), and so does linger (step 2); + - **a user manager Podman does not reach** over the account's user bus (no `/run/user//bus` socket, as without + the dbus-user-session package on Debian and Ubuntu; a `DBUS_SESSION_BUS_ADDRESS` pointing nowhere; a system unit + with `User=` and no user bus): Podman fell back to `cgroupfs` and the job again landed in the caller's own cgroup, + unbounded; + - **`cgroupfs` from a process inside `user@.service`** (the worker as a systemd user service): applied. + So the worker credits `isolation` (`podmanBoundsDelegated`) only while that file lists the three controllers AND + either Podman reports the `systemd` cgroup manager or the worker itself runs inside the user manager. One measured + case is applied and still not credited, since nothing the worker can read tells it apart from the unbounded ones: + an explicit `cgroup_manager = "cgroupfs"` used from a shell outside the manager. Not measured, and not credited: no + user manager, with the worker in a cgroup of its own that the account owns (a system unit with `User=` and + `Delegate=yes`, linger off). `podman info`'s `CgroupControllers` answers none of this: it is the calling process's + own cgroup, which listed every controller with nothing applied. Leave `cgroups` unset in every containers.conf the + account reads: measured with `--cgroups=disabled`, `--memory` and `--pids-limit` are accepted and silently not + applied, and the bounds read back as `max`. + + `pi-dispatch doctor`'s isolation line names which of these it saw, with its fix (⚠; the `PI_BACKEND_FLOOR` line is + the ✗ when the floor asks for `isolation`), and `pi-dispatch doctor --live` reads `pids.max` and `memory.max` back + off a real container. With linger off, doctor's ✓ holds only while a login session of the account is open, from + whatever shell it runs in: measured, a `sudo -iu` doctor saw the manager another login session had started and + gave ✓. The manager stops with the last session, so doctor says linger is off in a ⚠ of its own. Without a floor + on `isolation` a missed observation changes only the word: podman jobs still run, unbounded. With one (say + `PI_BACKEND_FLOOR=isolation=enforced`) the worker refuses to boot, naming the cause. 4. **No mounts of Podman's own.** `mkdir -p ~/.config/containers && : > ~/.config/containers/mounts.conf`. For a rootless account the user's `mounts.conf` replaces `/etc/containers/mounts.conf`, and an EMPTY file at whichever of the two applies stops the default `/run/secrets` mount (both measured). Leave `volumes`, `mounts`, `devices` @@ -382,7 +417,9 @@ an unprivileged account (uid 1234), on 2026-09-25. Run everything below as the w --fix` run as the account offers the pull for you: `podman pull ghcr.io/edgehero/pi-job:latest`, then `podman tag` to `pi-job:latest`, which is the default `PI_JOB_IMAGE`, so there is nothing to set. It offers nothing while `local` is also listed, or for a `PI_JOB_IMAGE` you chose. The image must declare `anyUid` unless the account is uid 1001; releases - since issue #341 do. + since issue #341 do. Pull it as the account; doctor's fix line names only that, since a podman-only host has no + docker to save an image from (issue #453). `pi-dispatch init` run with `PI_BACKENDS=podman` (in the shell or the + folder's `.env`) prints these steps as its next steps, in place of the docker ones. 6. **The egress proxy and Valkey, as Quadlet units in this account's user manager.** Put `PI_BACKENDS=podman` in `.env` first (step 8), then from the deployment folder run `pi-dispatch up`, which pulls the job image into this account's store and offers the stack, or `pi-dispatch service install`, which installs the same units beside the @@ -485,8 +522,13 @@ an unprivileged account (uid 1234), on 2026-09-25. Run everything below as the w `doctor --fix` offers no `docker run` for it. 8. **`PI_BACKENDS=podman`** in `.env` (the setup wizard, `/dispatch setup`, writes it when you choose rootless Podman at its runtime step), then start the worker, and run `pi-dispatch doctor` and - `pi-dispatch doctor --live` as the same account. doctor's podman section checks that `podman info` answered, that - the service is rootless and not remote, that the controllers are delegated, whether SELinux relabelling applies, + `pi-dispatch doctor --live` as the same account, from the deployment folder. doctor reads `PI_BACKENDS`, + `PI_EGRESS` and `PI_EGRESS_PROXY` as `up` and `service install` do: this shell's value where it sets one, else + `.env`'s, and a line naming the file when it supplied one; a shell and a `.env` that set one differently is a ✗, + since the service runs the file (issue #453). It takes the service's `VALKEY_URL`, `PI_PROVIDER` and the provider + key's presence from `.env` the same way where this shell does not set them, and says so; it never prints a key. doctor's podman section checks that `podman info` answered, that + the service is rootless and not remote, that a user manager runs with the controllers delegated and Podman puts + containers under it (step 3), whether SELinux relabelling applies, that the job image is in this account's store, and, with the egress policy armed, that the proxy is running under this Podman. `--live` reads the declarations back off real containers and says it read them back on podman, `egress` included: with the policy armed it first runs doctor's egress canary under this account's Podman, two @@ -591,7 +633,7 @@ same host. | Property | Docker Engine, rootful | Podman rootful | Podman rootless, keep-id, Docker API | Podman rootless, Docker API | podman-docker, rootful | podman-docker, rootless | podman (native, rootless) | |---|---|---|---|---|---|---|---| -| isolation | enforced | asserted (bounds applied, not credited) | refused: rootless | refused: rootless | asserted | refused: rootless | enforced while the pids, memory and cpu controllers are observed delegated, else asserted | +| isolation | enforced | asserted (bounds applied, not credited) | refused: rootless | refused: rootless | asserted | refused: rootless | enforced while this account's systemd user manager is observed running with the pids, memory and cpu controllers delegated to it and Podman putting containers under it, else asserted | | ephemeral | enforced | enforced | refused: rootless | refused: rootless | enforced | refused: rootless | enforced | | mountSet | enforced | enforced with the empty override, else asserted | refused: rootless | refused: rootless | enforced with the empty override, else asserted | refused: rootless | enforced while the mounts.conf that applies is observed empty, else asserted | | egress | enforced (PI_EGRESS) | enforced (PI_EGRESS) | refused: rootless | refused: rootless | enforced (PI_EGRESS) | refused: rootless | enforced (PI_EGRESS) | diff --git a/specs/design.md b/specs/design.md index 4a229cd3..75e41204 100644 --- a/specs/design.md +++ b/specs/design.md @@ -4846,9 +4846,11 @@ a tunnel. the default one included, which is why an egress-off job is not given a bridge of its own. The run, the stop, the reaper, the image and egress preflights and the job networks are the `local` venue's own factories with `bin: "podman"` (issue #354's first part, the seams). The venue decides from - one bounded `podman info --format json`, never from a container, kept once it answers (rootlessness, remoteness - and delegation are the account's and the process's, fixed for a worker's life) while the files the observations - read are read again before each job. In this order (`decidePodmanJobUser`): + one bounded `podman info --format json`, never from a container, kept once it answers (rootlessness and remoteness + are the account's and the process's, fixed for a worker's life) while the files the observations read are read + again before each job, the account's user-manager cgroup among them since issue #453, because that manager can stop + under a running worker. The cgroup manager that read reported is kept with it, a residual: a user bus that appears or + goes away later is not seen until a restart. In this order (`decidePodmanJobUser`): 1. a worker that is not on Linux: refused, `podman-platform` (Podman machine is unmeasured); 2. no answer: `unknown`, retried like any unanswered read, except the two determinate ones, no `podman` to run (`podman-not-found`) and an answer nothing parses (`podman-unreadable`); @@ -4938,16 +4940,50 @@ a tunnel. `XDG_RUNTIME_DIR`, falling back to `/run/user/`. - **`podman info --format json`** carries `host.security.rootless`, `host.serviceIsRemote` (false for a plain `podman`, true under `CONTAINER_HOST` or `--remote`), `host.security.selinuxEnabled`, `host.cgroupVersion`, - `host.cgroupControllers` and `version.Version`: one read decides the job user and all three observations. + `host.cgroupControllers` and `version.Version`: one read decides the job user and all three observations, the + bounds one together with a host file (below). `host.cgroupControllers` is the CALLING process's own cgroup, not + what is delegated to the account (issue #453, measured): it listed all five controllers where no bound was applied, + `memory pids` inside a user unit without `Delegate=` where cpu was applied, and flipped with a sibling unit's + `Delegate=`. `host.cgroupManager` does not decide the bounds either (below). + - **Bounds and the user manager** (issue #453, Fedora 44, Podman 5.8.1; round-446 M0-e with the venue's own argv, + and the gate-456 adversary's L rows). With no user manager (linger off, reached through `sudo -iu`, which starts + none) the configured `systemd` cgroup manager fell back to `cgroupfs` with a warning, an explicit `cgroupfs` + behaved the same, the container landed in the caller's root-owned session scope, and `pids.max`, `memory.max` and + `cpu.max` read `max` with exit 0. With a user manager running, the bounds applied where Podman reached it over the + account's user bus (`podman info` then reports the `systemd` cgroup manager), from a shell or a system unit alike, + and where the caller itself ran inside `user@.service` under an explicit `cgroupfs`; they did NOT apply where + Podman could not reach it (no user bus socket, a `DBUS_SESSION_BUS_ADDRESS` pointing nowhere, a system unit with + `User=` and no bus): Podman fell back to `cgroupfs` and the container landed in the caller's own root-owned cgroup. + An explicit `cgroupfs` from another user's session with the bus reachable applied too (M0-e E3c). A plain ssh + login started the user manager (pam_systemd). The `user@.service` cgroup path was absent with the unit + inactive and present with it active (measured), which is systemd's removal of a stopped unit's cgroup. - **`mounts.conf`**: a rootless user's `~/.config/containers/mounts.conf` replaces `/etc/containers/mounts.conf`, and an empty file at whichever applies suppresses `/usr/share/containers/mounts.conf`'s `/run/secrets`. - **SELinux** confines rootless containers as it does rootful ones: an unlabelled source is denied, `:z` and `:Z` work, and `label=disable` works by running the container `spc_t`. - **The words**, each checked against how `local` words the same property: - - `isolation` **enforced**, `observedBy` `podmanBoundsDelegated` (rootless, cgroup v2, `pids`, `memory` and `cpu` - among the controllers, and no containers.conf setting `cgroups`): the flags are this worker's argv, and the - observation is what makes the bounds real rather than accepted and dropped. `local`'s equivalent (`daemonAppliesBounds`) is false on every Podman daemon because its - Docker API hard-codes the booleans; `podman info`'s controller list is a different and readable fact. + - `isolation` **enforced**, `observedBy` `podmanBoundsDelegated` (rootless, cgroup v2, the account's systemd user + manager running with `pids`, `memory` and `cpu` in `/sys/fs/cgroup/user.slice/user-.slice/user@.service/ + cgroup.controllers`, Podman putting containers under it, and no containers.conf setting `cgroups`): the flags are + this worker's argv, and the observation is what makes the bounds real rather than accepted and dropped. "Under it" + is observed as either `podman info` reporting the `systemd` cgroup manager or the worker's own `/proc/self/cgroup` + inside `user@.service`, each of which the measurements above show applying the bounds, and neither of which + held in a row that did not. The file's absence is a determinate `false` (`no-user-manager`); a manager Podman does + not reach from outside it is a determinate `false` of its own (`user-manager-unreachable`); a read that failed for + a moment is a retry. Not credited, stated: an explicit `cgroupfs` from a shell outside the manager (measured + applied, but indistinguishable from the unbounded rows by anything the worker reads), and no user manager with the + worker in a cgroup of its own the account owns (a system unit with `User=` and `Delegate=yes`, linger off; not + measured). A hand-written system unit must be ordered after `user@.service` (`Wants=`/`After=`): measured, a + worker that started first refused to boot with exit 2 under an isolation floor and stayed failed. The observation changes a job only through + `PI_BACKEND_FLOOR`: with a floor on `isolation` the worker refuses to boot on it (tagged, exit 2) and each podman + job is refused `backend-floor-unobserved`; without one the word falls to `asserted` and jobs run as before, + unbounded. Doctor says it once, in the backend section's `isolation` line (⚠, with the cause's own fix; the floor + line is the ✗ under such a floor), its ✓ line lists the user manager's controllers, and with linger off a ⚠ says the + ✓ holds only while a login session lasts (measured: a `sudo -iu` doctor saw a manager another session had + started), so the static read agrees with `--live`, whose podman isolation fix names the same causes. The forge comment for a refusal + on this observation is cause-neutral: it says the bounds are not observed applied, never which cause, since the + cause is evidence and stays in the operator's log. `local`'s equivalent (`daemonAppliesBounds`) is false on every + Podman daemon because its Docker API hard-codes the booleans. - `ephemeral` **enforced**: `--rm` and a name carrying the job id, measured removing the container and its cidfile. - `mountSet` **enforced**, `observedBy` `podmanAddsNoMounts`: the argv's mounts are `containerSpec`'s, and the one runtime mount measured (`/run/secrets`) is observed absent through the mounts.conf chain this account's Podman @@ -5110,7 +5146,32 @@ a tunnel. of the plan for its `PODMAN_SYSTEMD_UNIT` label; one that does not name our unit is foreign, `service install` refuses unless `--force` (and then says it replaces it), and `up`, which has no `--force`, installs nothing and names both ways out. `up` reads a running proxy from `{{.State.Running}}`'s OUTPUT: an exited container prints - `false` with exit 0 (measured), and the exit code alone called a stopped hand-started proxy present. + `false` with exit 0 (measured), and the exit code alone called a stopped hand-started proxy present. Since issue + #453 it reads `{{.State.Status}}` and counts only `running`: Podman reports a paused container `paused` and one + between restarts `stopped` (measured on 4.9.3 and 5.8.1). The docker venue's proxy step reads the same Status, + stdout alone, since the status word is what names a paused or restarting container (measured on Docker Engine + 29.8.1: both read `.State.Running` true), and it also asks whether the shipped `pi-dispatch-egress-proxy` is CURRENT: the + pinned squid digest, the image's own entrypoint and command (read from its registry config; neither compose nor + `up` sets them), this folder's `deploy/egress-proxy.conf` and `egress-allowlist.conf` mounted, and no mount but + those and the image's two anonymous volumes (`egress-proxy-state.mjs`). The mounts are compared only where every + bind source resolves on this host: a runtime that reports its VM's own paths (Docker Desktop's shape, not measured + here) would otherwise have every proxy read stale, so that half is UNKNOWN, said as a ⚠ and never the ground for a + removal: UNKNOWN only for a source under a VM prefix (`/host_mnt/`, `/run/desktop/mnt/host/`) or on a macOS or + Windows host (gate round 3's simple rule); on Linux a source that does not resolve is gone, and stale. The argv is + normalised whatever the runtime's spelling: Podman 4.9.3's native inspect renders the entrypoint as a string + (measured), Docker Engine and Podman 5.8.1, natively and through its Docker API, as a list (measured). Measured on + Docker Engine 29.8.1 with compose 5.5.1 (rootful): a compose-made and an `up`-made proxy, from a real and a + symlinked folder, all read current (the symlink's path is kept as given, and resolves to the folder). A current + stopped one is + offered `docker start` of that same container (a paused one `docker unpause`): that is what `docker compose + --profile egress up -d` does to a stopped container whose configuration is unchanged, it keeps a compose-created + container compose's, and it destroys nothing. A stale one, running or not, would bring back a policy this + deployment did not ship, so it is offered `docker rm -f` and the shipped run, shown line for line, naming the job + and sandbox networks attached to it, which the removal cuts off mid-run, and saying to stop the worker first; a + container whose state `docker inspect` answers unreadably is left alone rather than run over; doctor's egress + section says the same ✗ for a running stale one, judging the mounts only when run from a folder holding the two + files. A container `PI_EGRESS_PROXY` names is the operator's: reported, never started or judged stale, and doctor's + fix for it names `docker start`, not compose, which starts only the shipped name. - **A replaced file restarts its unit.** `--force` over a changed `.container` file restarts that unit, because `start` on an active unit does nothing, and warns first when it is the proxy. A changed `.network` file restarts nothing: its unit runs `podman network create --ignore`, which cannot change an existing network. @@ -5375,3 +5436,4 @@ a tunnel. | 2026-09-27 | Issue #432. **`DES-PODMAN-NATIVE-ROOTLESS-BACKEND` AMENDED**, one sentence: its CI job, `.github/workflows/podman-conformance.yml`, is a required check on `main`, no longer advisory. It was promoted after it had passed on every PR and main push of the podman follow-up round (#428 to #437; 38 of 39 runs green, the one red a real defect on the branch that built the venue), renamed first to drop "(advisory)" because the job name is the context string, and then added to the branch protection as the sixth required context. The decision itself is UNCHANGED, checked. | | 2026-09-27 | Issue #451. **`DES-TERMINAL-COMMENTS-AND-FAILURE-HOOK` AMENDED**, the `provider-auth-refused` paragraph: the quoted sentence's "(HTTP 401 or 403)" becomes "(an authentication or permission error)", and the stop is described as a provider's refusal of the credential rather than a 401/403, because issue #451 admits a Google 400 `API_KEY_INVALID` and Bedrock's `UnrecognizedClientException: 403: ` form (`INT-RUNNER-EXIT-CODE-PROTOCOL`). The token, its once-ness and its hook behaviour are UNCHANGED, checked. **Code evidence**: worker/src/processor.mjs -> TERMINAL_COMMENTS. | | 2026-09-27 | Issue #446, folding its PR #457 gate rounds 1 to 3 into this one row. **`DES-RETENTION-SWEEPS-ON-A-TIMER` UNCHANGED, checked**: the sandbox sweep's leftover-tombstone removal runs at the start of the closure boot already built, so boot and every tick share it, nothing new is constructed, and the per-tree yield covers each tombstone as it covers each directory; the "window is a floor" cost is unchanged, the window now being the earlier of the worker's current one and the run's recorded `retainUntil` (`INT-SANDBOX-CONTRACT`). **`DES-SANDBOX-IS-A-FRESH-CONTAINER` UNCHANGED, checked**: a sandbox is still a fresh `--rm` container from the run's image and workspace; the post-launch look only removes one whose run was swept as it started, and a loss later in the session is reported, never acted on. | +| 2026-09-27 | Issue #453, three podman venue leftovers, with the PR's gate rounds 1 to 3 and the round-cap re-review. **`DES-PODMAN-NATIVE-ROOTLESS-BACKEND` AMENDED, and CORRECTED twice**: `podmanBoundsDelegated` read `podman info`'s `host.cgroupControllers`, which a measurement on Fedora 44 (Podman 5.8.1, the venue's own argv) showed is the CALLING process's cgroup, not the account's delegation: it listed all five controllers where an account with no systemd user manager (linger off, a `sudo -iu` shell) got no bound at all, `pids.max` and `memory.max` reading `max` with exit 0; it listed `memory pids` inside a user unit without `Delegate=` where cpu was applied; and it flipped with a sibling unit's `Delegate=`. The first correction read the fact that decided those rows, `user@.service`'s `cgroup.controllers`, and dropped the cgroup manager; the gate's adversary then measured a manager running that Podman could not reach (no user bus socket, a `DBUS_SESSION_BUS_ADDRESS` pointing nowhere, a system unit with `User=` and no bus), where Podman fell back to `cgroupfs` and the job again landed in the caller's own root-owned cgroup, unbounded, while the observation said applied. The rule now: the manager's cgroup with `pids`, `memory` and `cpu`, AND either the `systemd` cgroup manager reported by `podman info` or the worker's own `/proc/self/cgroup` inside `user@.service`. It credits no row measured unbounded; one measured-applied row (an explicit `cgroupfs` from a shell outside the manager) is not credited, and one unmeasured case fails closed, both named. A missing controller's evidence says what was measured (Podman 5.8.1 refuses a `--cpus` container, exit 126), not "applies nothing". The issue's "a plain `ssh`" case was corrected by measurement: an ssh login starts the user manager and the bounds apply; the failing case is `sudo -iu` with linger off. The residual the info cache carries (the cgroup manager of the first answered read) is stated. The consequence lands only through `PI_BACKEND_FLOOR` on `isolation` (the worker's boot, exit 2, and each podman job as `backend-floor-unobserved`); without one the word falls to `asserted`. A hand-written system unit must be ordered after `user@.service` (measured: without it the worker refused to boot and stayed failed). Doctor says the cause once, in the backend section's `isolation` line with that cause's fix (the ✗ under a floor stays the floor line's), its ✓ line lists the user manager's controllers, and with linger off a ⚠ says the ✓ lasts only as long as a login session (measured: a `sudo -iu` doctor saw a manager another session had started). The forge comment for this observation (`OBSERVATION_COMMENT`) says the bounds are not observed applied, true for every cause. Doctor's podman image fix lines no longer offer `docker save ... \| podman load`, which a podman-only host cannot run. **`DES-PODMAN-STACK-AS-QUADLET-UNITS` AMENDED**, one rule and its docker twin: a running proxy is `{{.State.Status}}` `running` on both venues (docker's `.State.Running` is also true for a paused or crash-looping container; Podman's states measured on 4.9.3 and 5.8.1). On docker the shipped proxy must also be current, the pinned squid with its own entrypoint and command and this folder's two files mounted and no other mount but its volumes: a current stopped one is offered `docker start` (a paused one `docker unpause`, compose's semantics for an unchanged container), a stale one `docker rm -f` and the shipped run, shown and consented with the job networks it would cut off named, and doctor says ✗ for a running stale one. Round 2 made the mount half conditional: it is compared only where every bind source resolves on this host, and is otherwise UNKNOWN (a ⚠, never a removal), since a runtime reporting its VM's own paths, as Docker Desktop is expected to, would have read every proxy stale; measured on Docker Engine 29.8.1 with compose 5.5.1, a compose-made and an `up`-made proxy from a real and a symlinked folder all read current, and a paused and a restarting container both read `.State.Running` true (pinned in `egress-proxy-state.test.mjs`). An unreadable inspect answer on an existing container is left alone rather than run over, and doctor's fix for a STOPPED shipped proxy that is stale is `up`'s replace offer, never compose (which would start it as it is). Round 3, the cap, chose the simplest rules: the proxy's states split three ways (running admits; paused, exited and dead refuse; every other word is a retry that names the state, in the worker, `up` and doctor alike); the mounts are unknown only under a VM prefix or on a macOS or Windows host, and a source gone on Linux is stale; the argv is normalised (Podman 4.9.3 renders the entrypoint as a string, measured); `--yes` never replaces a running proxy that job networks are attached to; and doctor's `.env` reads pass one allowlist and one loader mapping. The re-review: `created` refuses (it stays so, measured), a retry is "once, then failed" (two attempts), doctor says ✗ for `restarting` (a crash loop fails every job) and ⚠ only for a transient word; each bind is judged on its own, one under a VM prefix unknown for itself while the others and any extra mount are still compared, and nothing is removed while one of the two expected binds is unknown; doctor prints a URL as scheme, host, port and database only, and blanks C1 controls as well as C0. From the executing review of round 3: doctor's fix for a paused shipped proxy is `docker unpause`, as `up` offers; a running stale proxy gets no "✓ running" line before its ✗, and the canary through it names that proxy, not the allowlist, as what to fix; `up` creates the proxy's network only where it does not exist, so a recreate no longer prints the daemon's "already used". The `user-manager-unreachable` evidence says the bounds are not observed from here rather than not applied, since an explicit cgroupfs with the bus reachable measured them applied. An operator's `PI_EGRESS_PROXY` container is never started or judged stale, and doctor's fix for it now names `docker start` rather than compose. `up` no longer offers a proxy from a folder without `deploy/egress-proxy.conf` (measured: the runtime mounts a directory there). **UNCHANGED, checked**: `DES-EGRESS-DENY-ON-A-DEDICATED-NETWORK` (no network, attach or proxy image moves) and `DES-CLI-SURFACE` (no command or flag is added; `init` prints different next steps, and `init` and `doctor` decide the venue with `up`'s function). **Code evidence**: worker/src/backend-podman.mjs -> parsePodmanInfo, observePodmanBounds, podmanUserManagerControllersPath, podmanNoUserManagerEvidence, observePodman; worker/src/backends.mjs -> OBSERVATIONS, OBSERVATION_FIX; worker/src/doctor.mjs -> runDoctor, backendChecks, podmanChecks, egressChecks, PODMAN_BOUNDS_FIX, LIVE_FAIL_FIX_PODMAN, liveFailFix; worker/src/processor.mjs -> OBSERVATION_COMMENT; worker/src/egress-proxy-state.mjs; worker/src/deployment-venue.mjs -> deploymentVenueEnv; worker/src/up.mjs -> runUp (steps e2 and e3); worker/src/init.mjs -> runInit, nextSteps; .github/scripts/podman-conformance.mjs (an isolation floor, so a wrong observation fails the job). | diff --git a/specs/interfaces.md b/specs/interfaces.md index 9b01ff2b..19860d3c 100644 --- a/specs/interfaces.md +++ b/specs/interfaces.md @@ -1666,7 +1666,9 @@ contract governs the argv of one container, this governs the estate that argv jo |---|---| | `PI_EGRESS=0` | no network, no flag, no proxy variables, **zero docker spawns** -- byte-identical to a deployment before this contract | | proxy absent | POLICY refusal `egress-proxy-missing`, pre-spend, not retried | - | proxy stopped | POLICY refusal `egress-proxy-stopped`, pre-spend, not retried | + | proxy stopped (`{{.State.Status}}` `paused`, `exited`, `dead` or `created`, a container never started staying so; issue #453) | POLICY refusal `egress-proxy-stopped`, pre-spend, not retried | + | proxy in any other state (`restarting`, `stopping`, `removing`, Podman's `stopped` between restarts, a word no runtime prints yet; issue #453) | INFRA, pre-spend, **retried once, then failed** (the queue's two attempts), the message naming the proxy and its state: a state on its way somewhere is not a policy, and a crash loop fails its jobs rather than holding them | + | proxy running but not this deployment's (another squid digest, entrypoint, command or mount) | **admitted**: the gate reads the status word alone and is no guarantee of which proxy it is. `pi-dispatch doctor` says ✗ for a running stale shipped proxy and `pi-dispatch up` offers to replace it (issue #453) | | daemon silent | INFRA `container-never-started`, **retried**, reservation refunded | | network create fails | INFRA `container-never-started`, retried; a network the proxy could not join is torn down rather than left half-built | | allowlist missing a host | the job runs and the agent is refused by the proxy at that host. **Not** pre-spend detectable, and said so plainly rather than implied away. | @@ -2328,7 +2330,13 @@ is its only entry point (`worker/src/live-probes.mjs`, driven from `doctor.mjs`) rather than reading it. `.github/scripts/podman-conformance.mjs` runs the same canary instead of a copy of its own. The verdicts are this module's, unchanged; the few whose detail - named the docker CLI name the runtime they ran on, and their wording on `local` is byte-identical. The same + named the docker CLI name the runtime they ran on, and their wording on `local` is byte-identical. A failed + verdict's fix is `local`'s text with the CLI's name swapped, except `isolation` on podman (issue #453), which has + its own: rootless Podman's cause is the account's systemd user manager not running, Podman not reaching it over the + account's user bus, or a controller not delegated to it (all measured, issue #453), which doctor's `isolation` + line names when the static read sees it, and the fix leads with linger, never "cgroup delegation" alone. A probe + container that did not start is reported with the runtime's own first line (control bytes blanked, at most 300 + characters), so the reason is on the page rather than behind a rerun. The same verdicts are what `.github/scripts/podman-conformance.mjs` hands the harness as its `readBack`. - **Other venues** read back through the conformance harness: `runBackendConformance(backend, { readBack })` takes the same verdicts, keyed by property or as this module's array, for `READ_BACK_BY_A_LIVE_PROBE`. A @@ -5132,3 +5140,4 @@ onFailureTimeoutMs; worker/test/on-failure.test.mjs; worker/test/start-wiring.te | 2026-09-27 | Issue #449 (a retry turn is not a budget turn), with PR #455's gate rounds 1 and 2 folded in. **`INT-RUNNER-EXIT-CODE-PROTOCOL` AMENDED**, two places: the provider 429 row now says the code stays `1` under `--max-turns 1` too, because pi's own auto-retry of a turn that made no progress (no tool ran, no reply completed) is not a budget turn; and the telemetry paragraph names the decided exit line's new `retryTurns` field beside `turns`, diagnostic only, omitted on the catch path, parsed by nothing host-side. **`INT-RUN-HISTORY-FILE-CONTRACT` AMENDED**, one sentence: the record's `turns` is the budgeted count and `retryTurns` is deliberately not recovered into it; the record shape is UNCHANGED (no new field). **`parseExitTurns`' cites corrected** in `worker/src/run-history.mjs` (the exit lines moved from `run-job.mjs:263/:277` to `:431/:446`). **Pre-existing wording fixed in the same entry**: the `tokens.calls` key is defined precisely (stream dispatches the process-wide meter observed, including one pi-ai ended as aborted before sending, excluding a call the meter's own hard stop answered), matching `image/runner/src/usage-meter.mjs`'s `observe()` and `wrapProviderStreams`; no field changed. | | 2026-09-27 | Issue #451. **`INT-RUNNER-EXIT-CODE-PROTOCOL` AMENDED**, the provider-refusal row and its named residuals, admitting only shapes MEASURED against a real endpoint with a real bogus credential, because the costly mistake is a false refusal. (1) Refusals now read as `2` / `provider-auth-refused`: google-generative-ai and google-vertex (api key) by PARSING @google/genai's double-wrapped body (the whole message is JSON, the outer `error.code` is 400, 401 or 403, `error.message` is JSON again, and, reading own properties only, an ErrorInfo `reason` is `API_KEY_INVALID` or `ACCESS_TOKEN_TYPE_UNSUPPORTED`; the gRPC `status` is never read), and bedrock-converse-stream by the anchored `UnrecognizedClientException: 403: ` prefix (a runner rule: a shipped worker refuses a built-in `amazon-bedrock` job pre-spend as `provider-unconfigured`, since pi gives that provider no API-key variable to forward, and on the one reachable route, a custom `models.json` provider on that api, the key goes out as a bearer token whose bogus answer is a residual, so the admitted prefix is SigV4's and is reached only when SigV4 keys arrive another way; see (6)). Every one still passes pi-ai's `isRetryableAssistantError` guard. Why parse and not match: Google's inner key order differed between two identical real requests, and a status quoted inside another string must not count. Why these tokens: each is exactly what a real endpoint sent for a bogus credential (M0-d; the Google key is HTTP 400 `INVALID_ARGUMENT`, so its reason is the only signal). (2) The loopback table's Google cells answer `text/event-stream`, the format the real endpoints use; the old `application/json` cells tested a message production never produces. Must-stay-infra rows join it. (3) Named residuals, each a decision: openai-codex-responses (no status on either transport, only server prose); Google's gRPC status without a measured reason, `UNAUTHENTICATED` (a class, not a cause: "Authentication backend unavailable, try again later" is not transient to pi's predicate) and `PERMISSION_DENIED` (stub-only, and `SERVICE_DISABLED` and IAM propagation are transient windows pi's predicate misses); Bedrock's `AccessDeniedException` (also an intermittent cross-region SCP/IAM denial or a propagating fix, told apart only by the lost message) and `ExpiredTokenException` (never seen from real AWS); google-vertex under ADC in its three library forms, `invalid_grant: ...`, bare `invalid_grant`/`invalid_client`, and `Error code invalid_grant: ...` (`invalid_grant` also covers a clock-skewed assertion and a propagating service account, and a bare `invalid_client` is too generic); Google's quota 429 with the `billing` wording that pi calls non-transient; the single-layer Google JSON form. Gate round 1 of the PR narrowed a first draft that also admitted the `UNAUTHENTICATED` and `PERMISSION_DENIED` statuses, `AccessDeniedException` and `ExpiredTokenException`, and CORRECTED its ADC ground ("no loopback can drive the family's stream()"), which review refuted: an external_account credential names its own token URL, and a pinned test now drives the family's own `stream()` through it. (4) Recorded only: pi-ai 0.80.7 loses Bedrock's message, so nothing past the prefix is read, and the table pins the lost-body message so a fix is noticed. The residual "Google's bogus key is HTTP 400 and retries" is REMOVED, closed by (1). (5) **`INT-RUN-HISTORY-FILE-CONTRACT` and `INT-ON-FAILURE-HOOK-CONTRACT` AMENDED in wording only**: their prose restatements of the reason (the runner-reason paragraph, the paid-terminals bullet) say "refused the credential" instead of "answered 401 or 403", since a Google 400 `API_KEY_INVALID` now reaches it; the token and every rule around it are UNCHANGED. **`CONST-RETRY-INFRA-ONLY` statement UNCHANGED, checked; its note AMENDED** (see constitution.md): every admitted shape has a row driving its family's own `stream()` and a real-endpoint measurement. **Code evidence**: image/runner/src/outcome.mjs -> PROVIDER_AUTH_REFUSED, googleRpcAuthRefused, providerAuthRefused; image/runner/test/pinned-api.test.mjs -> AUTH_REFUSAL_TABLE, the two google-vertex ADC tests; image/runner/test/outcome.test.mjs -> AUTH_REFUSED_451, NOT_AUTH_REFUSED_451. (6) Round 2 of the gate, reachability and one admitted-shape residual: a shipped worker refuses a built-in `amazon-bedrock` job pre-spend as `provider-unconfigured`, and on the one reachable route (a custom `models.json` provider on `bedrock-converse-stream`) pi-ai sends the key as a bearer token, whose bogus answer (`AccessDeniedException`) is a residual, so a bogus key there ends infra; the admitted `UnrecognizedClientException` is SigV4's, reached only when SigV4 keys arrive another way. ADC Vertex likewise reaches the runner only through `auth.json` or a custom provider (pi's one key variable for `google-vertex` is `GOOGLE_CLOUD_API_KEY`). NEW named residual: a Google key used within minutes of creation or undelete can be answered `API_KEY_INVALID` before it propagates and ends `provider-auth-refused`; kept admitted, since the operator is at the keyboard and the stop names the reason. The own-properties read is pinned by a polluted-prototype test and described as defense in depth. | | 2026-09-27 | Issue #446 (the two windows left after #429), folding its PR #457 gate rounds 1 to 3 into this one row. **`INT-SANDBOX-CONTRACT` AMENDED**, seven places. (1) The manifest gains `retainUntil`, the worker's own deadline written at retention, and ONE rule (`sandboxDeadline`) is now read by the sweep, the CLI's open and `--list`, and the panel: the pin while set, else the EARLIER of `retainUntil` (a string, or nothing) and `createdAt` plus the reader's CURRENT window. Shortening applies (decided): `0` or a lowered window still clears what an earlier setting kept, raising it does not extend a run already retained, and `retainUntil` makes a reader with a LONGER window than the worker's end the run where the worker does. The Retention bullet writes down the accepted residual exactly: the sweep re-asks each expired run's runtime right before the rename (`opened-during-pass`, `runtime-unanswered`), and what is left, a container starting between that re-ask and the rename, is removed at the look's launch check or reported after the shell exits. It also states two supported configurations nothing stated before: one worker per retention root, and one runtime endpoint for the worker and the opener (documented, not refused: the manifest records no docker endpoint). (2) The sweep deletes through a TOMBSTONE, `.reap---` in the same root with no job id in the name (ENAMETOOLONG otherwise), re-reads the manifest there and restores a changed one, displacing an EMPTY directory at the name when it must (Docker's auto-created bind source; Podman refuses such a bind instead); a tombstone holding a live pin is never deleted (`tombstone-pinned`) and goes back only under a name its own `jobId` maps to (held with no usable `jobId`); a failed rename holds the run (`rename-failed`), a failed delete stays a tombstone (`tombstone-stuck`, retried each pass, said once a day), leftovers are removed at the start of every pass except a young tombstone of another LIVE pid (`tombstone-foreign`, once a day; a future-stamped name counts as old), and `.reap-` is reserved from every reader of the root. The one unseen case is stated: an open bound to an empty directory that a restore displaced shows an empty mount while the pinned run is intact. (3) A new bullet on opening a run at the end of its window: the `past-window` refusal inside `SANDBOX_OPEN_GRACE_MS` (five minutes) naming `--pin`, the pin written before any runtime call with its failure a refusal (`pin-failed`), the panel inheriting the refusal with no pin, and the post-launch look, concurrent with the shell: only its LAUNCH check removes the container (`rm -f --time=0` on podman, whose plain `rm -f` outlasted its own bound on pd-fedora; `rm -f` on docker), a loss later is reported after the shell exits (`lost`: `swept` or `replaced`) and never acted on, only an exit 125 from a container never listed, with the run gone, is `swept-at-launch` (Podman's refused bind), every other exit is checked for a loss at exit, a container never listed is watched but never touched, only a missing manifest or a replaced directory (another inode) is a verdict, and the look stops at once when the shell returns (its in-flight `ps` aborted). (4) The Name and the layout go through `sandboxEntryName`, `sanitizeJobId` plus an escape (one more `_` before a leading `.` or `_`), so no id names a tombstone, the root or its parent, and `.x` and `_x` never share a directory; a run retained before the escape is still read, named and held under its old directory (also when the escaped name holds another id's run); `resolveSandbox` refuses a directory holding another id's run (`id-mismatch`). (5) `directory-not-removed` and `blocked` are narrowed to a directory that could not be moved aside; a stuck tombstone's id leaves `keep` from the next pass; doctor warns about pinned-held tombstones apart from every other old one, never classifying by pid, and names only the unpinned ones for removal, each by path. (6) The absent refusal is neutral about which window swept a run, and `--list` says a run past its window or inside the grace needs `--pin`. (7) The Acceptance gains the #446 clauses. **`INT-CONTAINER-RUNTIME-CONTRACT` UNCHANGED, checked**: no job argv, mount or env var moves, and the sandbox argv is untouched. **`INT-EGRESS-POLICY-CONTRACT` UNCHANGED, checked**: the session network's creation and removal are where they were; only the keep set's inputs lost the tombstone names. **Code evidence**: worker/src/sandbox-store.mjs -> retainJobDir, readManifest, sandboxEntryName, sandboxDeadline, sandboxExpiry, sandboxTombstonePid, defaultPidAlive, makeSandboxReaper (isOpen, pidAlive, clearTombstone, restoreTombstone, restoreName), listSandboxes; worker/src/sandbox.mjs -> SANDBOX_OPEN_GRACE_MS, sandboxWindowRefusal, resolveSandbox, openSandbox, sandboxRemovalArgs, stopSandbox, listRunningSandboxes, makeSandboxRuntimeWatch (isOpen), sandboxContainerName; worker/src/start.mjs -> startWorker (isOpen wired); worker/src/sandbox-cli.mjs -> runSandbox, renderList, remaining; worker/src/doctor.mjs -> render, countRetained, sandboxTombstoneChecks; admin/src/index.ts -> readSandboxInfo, openSandboxSession; .github/scripts/conflict-marker-check.mjs (every tracked text file). | +| 2026-09-27 | Issue #453, with its gate rounds 1 to 3 and the round-cap re-review. **`INT-LIVE-PROBE-CONTRACT` AMENDED**, two sentences: on the native `podman` venue a failed `isolation` read-back carries its own fix, naming the causes measured (the account's user manager not running, Podman not reaching it, a controller not delegated) and leading with linger, where it used to be `local`'s "cgroup delegation" text with the CLI's name swapped, which sent an operator whose controllers already read delegated after the wrong thing; and a probe container that did not start is reported with the runtime's own first line (control bytes blanked, at most 300 characters) rather than "did not start" alone. No verdict, probe, argv or mutation moves. **`INT-EGRESS-POLICY-CONTRACT` AMENDED**, the failure table: the proxy counts as running only while `{{.State.Status}}` is `running`, in the worker's pre-spend gate, `up` and doctor alike, the status word being what names a paused or crash-looping container (measured: Docker Engine 29.8.1 reads `.State.Running` true for both). Gate round 3's simple rule: only `running` admits; `paused`, `exited` and `dead` are the `egress-proxy-stopped` refusal; every other word (restarting, and any word no runtime prints yet) is an INFRA retry, which the queue's two attempts make one retry and then a failure, whose message names the proxy and its state rather than blaming the runtime; the re-review moved `created` to the refusals (a container never started stays so, measured) and said "retried once, then failed" where the text had said "until" (round 2 had made `restarting` alone a retry, after a refusal there refused a job for good on one squid crash-restart). A new row says what the gate is not: a running proxy that is not this deployment's is admitted, and doctor and `up` are where it is flagged. The proxy component's name and attach rules are UNCHANGED, checked. **UNCHANGED, checked**: `INT-CONTAINER-RUNTIME-CONTRACT` (no flag, mount or env var moves; the bounds observation reads two host files, `user@.service`'s `cgroup.controllers` and, when Podman does not use the `systemd` cgroup manager, `/proc/self/cgroup`, in place of `podman info`'s caller-cgroup list). **Code evidence**: worker/src/doctor.mjs -> liveFailFix, LIVE_FAIL_FIX_PODMAN, PODMAN_BOUNDS_FIX; worker/src/live-probes.mjs -> runLiveProbes, runtimeErrorLine; worker/src/egress.mjs -> makeEgressPreflight; worker/src/egress-proxy-state.mjs. | diff --git a/specs/requirements.md b/specs/requirements.md index ef67a102..a5d7208f 100644 --- a/specs/requirements.md +++ b/specs/requirements.md @@ -1957,7 +1957,22 @@ and nothing about the box itself (`INT-CONTAINER-RUNTIME-CONTRACT`). does not list `local` (`DES-PODMAN-STACK-AS-QUADLET-UNITS`). `up` reads `PI_BACKENDS`, `PI_EGRESS` and `PI_EGRESS_PROXY` from this shell where it sets them and otherwise from the deployment's `.env`, the file `service install` reads, refuses when the two set one differently, and never replaces a container it did not - start. + start without showing that it does and asking: on the podman venue not at all, and on the docker venue only the + shipped proxy's name, as below (issue #453). On the docker venue `up` counts the egress proxy as present only while + it is RUNNING (`{{.State.Status}}` is `running`, which a paused or crash-looping container is not) and CURRENT (the + pinned squid with its own entrypoint and command, with this deployment folder's `deploy/egress-proxy.conf` and + `egress-allowlist.conf` mounted and no other mount but the image's volumes; mounts whose sources do not resolve on + this host are unknown, said, and never a reason to replace it). A + current shipped proxy that is stopped is offered `docker start` of that same container (a paused one `docker + unpause`), never a `docker run` on its taken name; a stale one, running or not, is offered `docker rm -f` and the + shipped run, all three lines shown before consent, with the job networks the removal would cut off named, and `--yes` never covering the removal of a RUNNING proxy that job networks are attached to (the lines are printed and a person must answer); a container `PI_EGRESS_PROXY` names is the operator's, reported + and never started. Neither is offered from a folder that lacks either file, which a start would mount as a + directory. **`init` and `doctor` decide the venue as `up` does** (issue #453): this shell where it sets a venue key, + else the deployment's `.env` read as `service install` reads it, and a disagreement between the two is said + (doctor's ✗; init shows no steps). `init`'s next steps follow that venue: with podman the only venue, on Linux, + the podman ladder (linger named first, then the job image into this account's store, `.env`, `up`, `doctor`, + `service install`, `doctor --live`); off Linux a line saying the venue refuses the host; otherwise the docker text, + unchanged byte for byte. They are printed text: `init` still runs nothing and remains create-only. **`up` also fills EMPTY keys in an existing `.env`** (issue #357), which is the one mutation it makes without a prompt and is bounded to exactly that: the file must already exist, so `init`'s create-only rule is untouched; a key with any value is left alone, so "never touch an @@ -2011,7 +2026,20 @@ and nothing about the box itself (`INT-CONTAINER-RUNTIME-CONTRACT`). all are removed when the read ends, or by the next `--live` when that run was interrupted, which names what it removed (`INT-LIVE-PROBE-CONTRACT`). Nothing else reaches it: not `up`, not `--fix`, not the panel. - **`doctor` reads the deployment's `.env` for two named keys, to decide what to say AND whether to fail** - (issues #357 and #384). The + (issues #357 and #384), **and, since issue #453, for the three venue keys `up` and `service install` already read** + (`PI_BACKENDS`, `PI_EGRESS`, `PI_EGRESS_PROXY`, through the same `readStackKeys` and the same shell-wins, + disagreement-is-a-✗ rule as `up`), **and for the service's own `VALKEY_URL`, `PI_PROVIDER` and the provider's key** + (its presence, and whether it is whitespace or an OAuth token; never its value), each taken from a plain line with + the service's loader only where this shell does not set it, and each said with its source. Every such key passes + ONE structural allowlist (`SERVICE_ENV_KEYS` plus pi's own key names for the provider) and one platform-to-loader + mapping, beside the two-key `ENV_FILE_READABLE_KEYS`; unlike those two, these values STEER doctor: which venue it + judges and so which runtime it spawns, which Valkey it connects to, and which fixes it offers. A URL doctor prints + is printed as scheme, host, port and database only (`` without one, `` when it does not + parse), and the fleet host names a chosen Valkey returns are printed with C0 and C1 controls blanked. That widening is the review this paragraph asks for: without it the plain `doctor` a podman-only + deployment's own ladder runs judged docker while its service ran podman, and reported "✗ Provider key set" and the + default Valkey URL for a deployment whose `.env` held both (both measured). The file is read once per run, a + regular file only; one that is there and unreadable is said, as `up` says it. + The new precedent in this entry, and it is narrow on purpose because the project's stance is that nothing parses `.env`. `up` writes `PI_PAUSE_WINDOWS_FILE` and `PI_SCOPED_LIMITS_FILE` into `/.env`, which configures the SERVICE through `EnvironmentFile=` and the wrappers and configures nothing about a shell @@ -2280,6 +2308,7 @@ instead of drifting. | Date | Change | |---|---| +| 2026-09-27 | Issue #453. **`REQ-DEPLOYMENT-BOOTSTRAP` AMENDED**, three clauses, the first two revised by the PR's gate rounds 1 to 3 and the round-cap re-review. (1) `up`'s docker egress step counts the proxy present only while it is running (`{{.State.Status}}` `running`, the word that names a paused or crash-looping container) and current (the pinned squid with its own entrypoint and command, this folder's two files mounted and no other mount but its volumes; mounts whose sources do not resolve on this host are unknown and never a reason to replace it). A current stopped one is offered `docker start` (a paused one `docker unpause`), a stale one, running or not, `docker rm -f` and the shipped run, naming the job networks the removal would cut off, which `--yes` does not cover for a running proxy carrying them (round 3); an operator's `PI_EGRESS_PROXY` container is reported and never started; and neither is offered from a folder lacking `deploy/egress-proxy.conf` or the allowlist, which a start would mount as a directory (measured). The "never replaces a container it did not start" sentence is CORRECTED to what is true: never without showing it and asking, never at all on the podman venue, and on docker only the shipped proxy's name. (2) `init` and `doctor` decide the venue exactly as `up` does, from this shell or the deployment's `.env`, and a disagreement is said (doctor ✗, init no steps); doctor used to read its shell alone, so the ladder's own `doctor` step judged docker on a podman-only deployment (measured). The paragraph on doctor's narrow `.env` read gains those three keys and, in gate round 2, `VALKEY_URL`, `PI_PROVIDER` and the provider key's presence (the ladder's own `doctor` step failed on a key and a Valkey URL the service did read, measured), each said with its source and no secret printed, stated as the review it asks for; round 3 put every one of those reads behind one allowlist and one loader mapping and blanked control bytes in the fleet host names a chosen Valkey returns, and the re-review has doctor print a URL as scheme, host, port and database only (no userinfo, query or fragment; ``, ``) and blank C1 controls too. (3) `init` prints the podman ladder, linger named first and `doctor --live` last, when podman is the only venue on Linux, a refusal line off Linux, and the docker text otherwise, byte for byte. The consent contract is UNCHANGED, checked: `start`, `unpause`, `rm -f` and the run are each shown before they run and `--yes` runs exactly those lines, and `init` still mutates nothing but its own create-only scaffolds. **UNCHANGED, checked**: `REQ-EGRESS-ALLOWLIST` (the policy, its proxy image and its per-job networks are untouched; a stopped proxy was already a pre-spend refusal, and the worker's own preflight now reads the same Status rule). | | 2026-09-27 | Issue #446 (the two windows left after #429), folding its PR #457 gate rounds 1 to 3 into this one row. **`REQ-RESURRECTABLE-SANDBOX` AMENDED**, the Bounded bullet and the Acceptance. The bullet keeps its promise that `0` sweeps what an earlier setting retained and adds that a lowered window applies to retained runs too (decided under #446: shortening applies) while a raised one does not extend them; it records the manifest's new `retainUntil`, the earlier-of rule that lets a reader with a longer window end a run where the worker does, that a run at the end of its window opens only with `--pin`, and that the one open the refusal cannot see is held by the sweep's re-ask or reported: removed as swept at the launch check, or said after the shell exits and never removed later. The Acceptance gains the #446 clauses: the past-window refusal (also for an opener whose window is larger than the worker's), the pin first and a failed pin refusing, a lowered or zero window sweeping retained runs, a pin during a large delete reporting the run gone, a crash between rename and delete recovered by the next pass, and a stuck delete named by doctor. The Statement and the Scope are UNCHANGED, checked: nothing about who may open a run, or from where, moved. **`REQ-EGRESS-ALLOWLIST` UNCHANGED, checked**: the session network is created and removed as before. | | 2026-09-27 | Issue #451 (the Google, Vertex and Bedrock refusal shapes). **`REQ-OPERATOR-FAILURE-NOTIFICATION` UNCHANGED, checked**: the new shapes end as the existing `provider-auth-refused` token, which the hook already pages once per paid policy terminal; no reason is added. **`REQ-LOCAL-JOB-VISIBILITY` UNCHANGED, checked**: the same token rides the same fixed-enum field. **`REQ-JOB-STATUS-COMMENTS` AMENDED**, its #437 clause and the stop's sentence: the parenthetical "(HTTP 401 or 403)" becomes "(an authentication or permission error)", because a bogus Google AI Studio key is HTTP 400 `API_KEY_INVALID` and now ends as `provider-auth-refused` (`INT-RUNNER-EXIT-CODE-PROTOCOL`); the sentence stays fixed and path-free, and exactly one completion or failure comment still holds. **Code evidence**: worker/src/processor.mjs -> TERMINAL_COMMENTS (pinned by worker/test/processor.test.mjs); docs/notifications.md. | | 2026-09-27 | Issue #449 (a retry turn is not a budget turn), with PR #455's gate rounds 1 and 2 folded in. **`REQ-RUNNER-TURN-BUDGET` AMENDED**: a new **What counts** bullet (every `turn_start` except the one opening pi's own auto-retry of a turn that made NO progress, meaning no tool ran and no reply completed, tallied as `retryTurns`; the flag clears on `auto_retry_end` and on `agent_settled`), the tool guard's reason (a retry-shaped throw after a turn's tools ran is a new turn, and exempting it laundered 9 paid calls at `--max-turns 1`) and the completed-reply guard's (round 2: a fault after a clean reply let a queued follow-up run as exempt new work, 6 paid calls at `--max-turns 1`; with nothing queued the same fault rejects `session.prompt()` as exit `1`, retried, the right class for its likely cause, such as a session-store fault; the `agent_settled` clear is kept as a safeguard), the plain statement that compaction continuations still count including the overflow one that re-runs a turn and what that means under `--max-turns 1`, the bound restated as at most `maxTurns * PI_RETRY_MAX` retry calls uncounted BY THIS EXEMPTION (calls never counted before stay so), an evidence bullet citing the pinned 0.80.7 dist line by line and naming the real-`AgentSession` loopback pins in `pinned-api.test.mjs`, and two Acceptance sentences (a recovered 429 under `--max-turns 1` exits `0`; an unrecovered one exits `1`, not `turn_budget`). **Pre-existing defect fixed in the same entry**: every `turn_start` past the cap re-aborts (one abort ended only the run in flight, and a queued follow-up's new run went on, 7 paid calls at `--max-turns 1` on main); the fallback per-session token budget (`REQ-TOKEN-ACCOUNTING-AND-CAPS`, `image/runner/src/token-budget.mjs`) re-aborts every `turn_start` after its breach for the same reason, since pi retries a failed breaching turn with a fresh signal; the statement of that requirement is UNCHANGED, checked, and the process-wide meter's own brake already answered every later call. | diff --git a/worker/src/backend-podman.mjs b/worker/src/backend-podman.mjs index fd5b3c15..3420c1de 100644 --- a/worker/src/backend-podman.mjs +++ b/worker/src/backend-podman.mjs @@ -16,10 +16,14 @@ * - only Linux was measured; Podman machine on macOS and Windows was not (`podman-platform`). * * THE FACTS COME FROM ONE READ, `podman info --format json`, cached once it answers: it decides the job user, whether - * mounts are relabelled, and the three observations the table's words rest on. Rootless-ness, remoteness and the - * delegated controllers cannot change under a running worker without a restart (they are this account's and this - * process's environment), so re-reading per job would only add a spawn; the FILES an observation reads (mounts.conf, - * containers.conf) are re-read per job, because an operator creating the empty override must not need a restart. + * mounts are relabelled, and, with this host's files, the three observations the table's words rest on. Rootless-ness + * and remoteness cannot change under a running worker without a restart (they are this account's and this process's + * environment), so re-reading them per job would only add a spawn. The FILES an observation reads are re-read per job: + * mounts.conf and containers.conf, because an operator creating the empty override must not need a restart, and the + * account's user-manager cgroup (issue #453), because that manager can stop under a running worker (linger turned off, + * the last login session ending) and the next job must then be judged without its bounds. One residual rides the + * cache: the cgroup manager `podman info` reported at the first answered read is kept, so a user bus that appears or + * disappears later is not seen until a restart. * * `backends.mjs` stays a leaf, so this module imports it and never the other way round. */ @@ -89,6 +93,12 @@ export function parsePodmanInfo(stdout) { selinux: bool(host.security?.selinuxEnabled), // "v2" or "v1"; anything else is no fact rather than a string an operator's terminal is handed. cgroupVersion: typeof host.cgroupVersion === "string" && /^v[0-9]{1,2}$/.test(host.cgroupVersion) ? host.cgroupVersion : null, + // Issue #453: the cgroup manager Podman actually uses for this call. `cgroupfs` when it was configured so, and when a + // configured `systemd` could not reach the account's user manager over D-Bus (Podman then warns and falls back). + // A short lower-case word, else no fact. + cgroupManager: typeof host.cgroupManager === "string" && /^[a-z][a-z0-9_-]{0,31}$/.test(host.cgroupManager) ? host.cgroupManager : null, + // The controllers of the CALLING process's own cgroup, as `podman info` reports them. Kept for what it is and for + // nothing else: issue #453 measured it listing all five where no bound was applied, so no observation reads it. controllers, version: displayVersion(body.version?.Version), // Issue #429: the container STORE this account's Podman uses (`store.graphRoot`), which moves with HOME, @@ -240,20 +250,88 @@ function observePodmanMounts({ fs, home, env, euid }) { } /** - * `podmanBoundsDelegated` from the info and this host's files: rootless, cgroup v2, `pids`, `memory` and `cpu` among the - * controllers delegated to this user, and no containers.conf taking containers out of their cgroup. Rootful Podman is - * refused as a venue, so its (undelegated, whole-tree) controller list earns nothing here either. + * The cgroup of this account's systemd user manager, whose `cgroup.controllers` says which controllers are delegated to + * it. Measured (issue #453, Fedora 44): the path is absent while `user@.service` is inactive and present while it + * runs, which is systemd's own behaviour (the unit's cgroup is removed when it stops). + */ +export function podmanUserManagerControllersPath(euid) { + return `/sys/fs/cgroup/user.slice/user-${euid}.slice/user@${euid}.service/cgroup.controllers`; +} + +/** What `observePodmanBounds` says when no systemd user manager runs for the account (issue #453); doctor keys its fix on the cause. */ +export function podmanNoUserManagerEvidence(euid) { + return `no systemd user manager is running for this account (${podmanUserManagerControllersPath(euid)} does not exist), so Podman leaves a job's pids, memory and cpu bounds unapplied, whichever cgroup manager it uses: with linger off the manager runs only while the account has a login session, and a \`sudo -iu\` shell starts none (measured). On a host without systemd managing cgroups under user.slice this path never exists, and the venue gives no credit there`; +} + +/** Whether a `/proc/self/cgroup` text puts this process inside the account's user manager (a systemd user service). */ +function insideUserManager(text, euid) { + const line = String(text ?? "").split("\n").find((l) => l.startsWith("0::")); + return typeof line === "string" && line.slice(3).startsWith(`/user.slice/user-${euid}.slice/user@${euid}.service/`); +} + +/** + * `podmanBoundsDelegated` from the info and this host's files: rootless, cgroup v2, a RUNNING systemd user manager for + * this account with `pids`, `memory` and `cpu` delegated to it, Podman able to put its containers under that manager, + * and no containers.conf taking containers out of their cgroup. Rootful Podman is refused as a venue, so it earns + * nothing here either. + * + * WHAT DECIDES IT (issue #453, measured on Fedora 44 with rootless Podman 5.8.1, round-446 M0-e and the gate's L rows): + * - no user manager (linger off, a `sudo -iu` shell): the container landed in the caller's root-owned session scope, + * `pids.max`, `memory.max` and `cpu.max` `max`, exit 0, whichever cgroup manager was configured; + * - a user manager running AND Podman reaching it (`podman info` reports the `systemd` cgroup manager): applied; + * - a user manager running but NOT reachable over D-Bus (no user bus socket, a DBUS_SESSION_BUS_ADDRESS pointing + * nowhere, a system unit with `User=` and no user bus): Podman fell back to `cgroupfs`, the container landed in the + * caller's own root-owned cgroup, unbounded (gate L1, L2, L3); + * - `cgroupfs` from a process already inside `user@.service` (a user unit, with or without `Delegate=`): applied. + * So credit needs the manager's cgroup with the three controllers, and then either the `systemd` cgroup manager or this + * process inside that manager. `podman info`'s `host.cgroupControllers` decides nothing: it is the caller's own cgroup, + * which listed all five with nothing applied. Fail-closed where not measured, or where measured applied but not + * provable from here: an explicit `cgroupfs` from a shell outside the manager with a reachable bus applied (M0-e E3c) + * and is not credited; no user manager with the caller in a cgroup of its own that the account owns (a system unit with + * `User=` and `Delegate=yes`, linger off) was not measured and is not credited. */ function observePodmanBounds(info, { fs, home, env, euid }) { if (info.rootless !== true) return { value: false, evidence: "podman info does not report a rootless Podman, which is the only kind this venue runs on" }; if (info.cgroupVersion !== "v2") return { value: false, evidence: `podman info reports cgroup ${info.cgroupVersion ?? "version unknown"}, and rootless bounds need cgroup v2` }; - const missing = ["pids", "memory", "cpu"].filter((c) => !Array.isArray(info.controllers) || !info.controllers.includes(c)); - if (missing.length > 0) return { value: false, evidence: `the ${missing.join(", ")} cgroup ${missing.length === 1 ? "controller is" : "controllers are"} not delegated to this user, so rootless Podman accepts the bound and applies nothing` }; + if (!Number.isInteger(euid) || euid < 0) return { value: false, evidence: "the worker's uid is not known, so whether its systemd user manager runs could not be read" }; + const path = podmanUserManagerControllersPath(euid); + let text; + try { + text = String(fs.readFileSync(path, "utf8")); + } catch (error) { + if (error?.code === "ENOENT") return { value: false, cause: "no-user-manager", evidence: podmanNoUserManagerEvidence(euid) }; + return unreadFileFinding(path, error?.code ?? "error"); + } + const controllers = text.split(/\s+/).filter((c) => /^[a-z][a-z0-9_]{0,31}$/.test(c)); + const missing = ["pids", "memory", "cpu"].filter((c) => !controllers.includes(c)); + // Measured (gate 456, Fedora 44, Podman 5.8.1, a manager delegated only memory and pids): a job carrying `--cpus` + // FAILED to start, crun exit 126 "controller `cpu` is not available", and one without it ran with no cpu.max at all. + // Only the cpu case was measured, so the sentence says what a job gets without claiming more for the other two. + if (missing.length > 0) return { value: false, evidence: `the ${missing.join(", ")} cgroup ${missing.length === 1 ? "controller is" : "controllers are"} not delegated to this account's systemd user manager (user@${euid}.service), so a job cannot have that bound: measured for cpu, Podman 5.8.1 refuses to start a container carrying --cpus (exit 126, "controller \`cpu\` is not available")` }; + if (info.cgroupManager !== "systemd") { + let inside = false; + try { + inside = insideUserManager(fs.readFileSync("/proc/self/cgroup", "utf8"), euid); + } catch (error) { + if (error?.code !== "ENOENT") return unreadFileFinding("/proc/self/cgroup", error?.code ?? "error"); + } + if (!inside) { + return { + value: false, + cause: "user-manager-unreachable", + evidence: `${info.cgroupManager === null ? "podman info does not say which cgroup manager Podman uses" : `Podman is using the ${info.cgroupManager} cgroup manager, not systemd`}, and this worker is not running inside the account's user manager (user@${euid}.service), so whether a job's pids, memory and cpu bounds are applied is not observed from here. Measured: where a configured systemd manager fell back to cgroupfs because Podman could not reach the user manager over D-Bus (no user bus socket, a DBUS_SESSION_BUS_ADDRESS pointing nowhere), the container landed in the worker's own cgroup unbounded; an explicit cgroupfs with the bus reachable had its bounds applied, and nothing read here tells the two apart`, + }; + } + } const listed = podmanConfFiles({ fs, home, env, euid }); if (listed.finding) return listed.finding; const conf = confKeyFinding(fs, listed.files, { key: CGROUPS_KEY, says: "sets a cgroups key, which can run every container outside its cgroup with its bounds unapplied" }); if (conf) return conf; - return { value: true, evidence: "rootless Podman on cgroup v2 with the pids, memory and cpu controllers delegated, and no containers.conf sets cgroups" }; + return { + value: true, + controllers, + evidence: `rootless Podman on cgroup v2, with this account's systemd user manager (user@${euid}.service) running and the pids, memory and cpu controllers delegated to it, ${info.cgroupManager === "systemd" ? "Podman using the systemd cgroup manager" : "this worker running inside that manager"}, and no containers.conf sets cgroups`, + }; } /** @@ -377,6 +455,11 @@ export function observePodman({ read, fs = { statSync, readFileSync, readdirSync return { observations: { [PODMAN_BOUNDS_DELEGATED]: bounds.value, [PODMAN_ADDS_NO_MOUNTS]: mounts.value, [PODMAN_SERVICE_LOCAL]: service.value }, evidence: { [PODMAN_BOUNDS_DELEGATED]: bounds.evidence, [PODMAN_ADDS_NO_MOUNTS]: mounts.evidence, [PODMAN_SERVICE_LOCAL]: service.evidence }, + // The controllers delegated to the account's user manager when the bounds hold (issue #453), for doctor's line: the + // fact the answer rests on, never `podman info`'s caller-cgroup list. + ...(bounds.value === true ? { boundsControllers: bounds.controllers } : {}), + // Which miss it was, when it was the user manager's, so doctor gives that fix and no other. + ...(bounds.cause ? { boundsCause: bounds.cause } : {}), // With an answered read, a `null` here is a host file that could not be read for a moment (`unreadFileFinding`), // and its reason says so, so the retry names a file rather than "unknown". reasons: { diff --git a/worker/src/backends.mjs b/worker/src/backends.mjs index e80bd630..222f187b 100644 --- a/worker/src/backends.mjs +++ b/worker/src/backends.mjs @@ -363,12 +363,15 @@ export const DAEMON_APPLIES_BOUNDS = "daemonAppliesBounds"; export const RUNTIME_ADDS_NO_MOUNTS = "runtimeAddsNoMounts"; /** - * The observation that a ROOTLESS Podman applies a job container's pid, memory and cpu bounds (issue #354), from the one - * `podman info --format json` read the podman venue's job user is decided from: cgroup v2, with `pids`, `memory` and - * `cpu` among the controllers systemd delegated to this user. Without delegation rootless Podman accepts `--pids-limit` - * and `--memory` and applies nothing (measured on Podman 5.8.1, Fedora 44: `--cgroups=disabled` exits 0 and the bounds - * read back `max`), so the word rests on the delegation being OBSERVED, and a `cgroups` key in a containers.conf this - * user's Podman reads withholds it. `pi-dispatch doctor --live` reads `pids.max` and `memory.max` back off a container. + * The observation that a ROOTLESS Podman applies a job container's pid, memory and cpu bounds (issue #354; the facts it + * rests on corrected by issue #453's measurements). From the `podman info` read the venue's job user is decided from + * (rootless, cgroup v2, the cgroup manager Podman uses) and this host's files: the account's systemd user manager + * running with `pids`, `memory` and `cpu` delegated to it (its `user@.service` cgroup), and Podman able to put a + * container under it (the `systemd` cgroup manager, or the worker itself inside that manager). Measured on Podman + * 5.8.1, Fedora 44: without a reachable user manager a container lands in the caller's own cgroup and its bounds read + * back `max`, exit 0, and `--cgroups=disabled` does the same; so the word rests on those facts being OBSERVED, and a + * `cgroups` key in a containers.conf this user's Podman reads withholds it. `pi-dispatch doctor --live` reads + * `pids.max` and `memory.max` back off a container. */ export const PODMAN_BOUNDS_DELEGATED = "podmanBoundsDelegated"; @@ -405,7 +408,7 @@ export const OBSERVATIONS = Object.freeze({ [DOCKER_ENDPOINT_LOCAL]: "the docker endpoint this host's docker CLI resolves is on this host", [DAEMON_APPLIES_BOUNDS]: "the daemon reports that it applies a container's pid and memory bounds, and is neither rootless nor Podman", [RUNTIME_ADDS_NO_MOUNTS]: "the container runtime adds no mounts of its own to a job container", - [PODMAN_BOUNDS_DELEGATED]: "this worker's rootless Podman runs on cgroup v2 with the pids, memory and cpu controllers delegated to it, and no containers.conf it reads sets `cgroups`", + [PODMAN_BOUNDS_DELEGATED]: "this worker's rootless Podman runs on cgroup v2 with a systemd user manager running for its account, the pids, memory and cpu controllers delegated to that manager, and Podman putting containers under it, and no containers.conf it reads sets `cgroups`", [PODMAN_ADDS_NO_MOUNTS]: "this worker's rootless Podman adds no mounts of its own to a job container", [PODMAN_SERVICE_LOCAL]: "the podman CLI this worker spawns runs containers on this host, not through a remote service", }); @@ -415,7 +418,7 @@ export const OBSERVATION_FIX = Object.freeze({ [DOCKER_ENDPOINT_LOCAL]: "Point the docker CLI back at this host (DOCKER_HOST, DOCKER_CONTEXT or `docker context use`), or lower that entry to `asserted` if the redirect is deliberate.", [DAEMON_APPLIES_BOUNDS]: "Run jobs on a rootful Docker Engine that reports PidsLimit and MemoryLimit, or lower that entry to `asserted`: on Podman and rootless daemons `pi-dispatch doctor --live` reads pids.max and memory.max off a real container instead.", [RUNTIME_ADDS_NO_MOUNTS]: "On Podman, create an empty /etc/containers/mounts.conf and remove any `volumes` or `mounts` key from containers.conf (and write it in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at), or lower that entry to `asserted`.", - [PODMAN_BOUNDS_DELEGATED]: "Delegate the cpu, memory and pids controllers to the worker account on cgroup v2 (a systemd `Delegate=` drop-in for user@.service), remove any `cgroups` key from its containers.conf (written in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at), or lower that entry to `asserted`: `pi-dispatch doctor --live` reads pids.max and memory.max off a real container.", + [PODMAN_BOUNDS_DELEGATED]: "Turn on linger for the worker account (`loginctl enable-linger `, which keeps its systemd user manager running with no one logged in) and run the worker as a systemd user service (`pi-dispatch service install`), or give Podman the account's user bus so it uses the systemd cgroup manager; delegate the cpu, memory and pids controllers to that manager on cgroup v2 (a systemd `Delegate=` drop-in for user@.service), remove any `cgroups` key from its containers.conf (written in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at), or lower that entry to `asserted`: `pi-dispatch doctor --live` reads pids.max and memory.max off a real container.", [PODMAN_ADDS_NO_MOUNTS]: "As the worker account, create an empty ~/.config/containers/mounts.conf (or, when it has none, an empty /etc/containers/mounts.conf), remove any `volumes`, `mounts`, `devices` or `hooks_dir` key from every containers.conf it reads (and write those in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at) and any OCI hook, or lower that entry to `asserted`.", [PODMAN_SERVICE_LOCAL]: "Unset CONTAINER_HOST and any containers.conf service destination for the worker account so `podman info` reports serviceIsRemote false, or lower that entry to `asserted`.", }); diff --git a/worker/src/cli.mjs b/worker/src/cli.mjs index 1e92b6dc..fef2be07 100755 --- a/worker/src/cli.mjs +++ b/worker/src/cli.mjs @@ -56,7 +56,9 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr if (cmd === "init") { const { runInit } = await import("./init.mjs"); - return runInit(process.cwd()); + // The environment rides along so the next steps match the venue (issue #453): PI_BACKENDS=podman alone gets + // the podman ladder. + return runInit(process.cwd(), { env }); } if (cmd === "doctor") { diff --git a/worker/src/deployment-venue.mjs b/worker/src/deployment-venue.mjs new file mode 100644 index 00000000..9de9f116 --- /dev/null +++ b/worker/src/deployment-venue.mjs @@ -0,0 +1,94 @@ +/** + * WHICH VENUE A DEPLOYMENT RUNS, decided one way for every command that asks (issue #453): `up` since #430, and `init` + * (its next steps) and `doctor` (what it judges) since #453, which read only their shell before and so answered for a + * different venue than the service they described. The deployment's `.env` is read with `readStackKeys`, exactly as + * `service install` reads it, and a key this shell sets differently stops the command, since the service runs the file. + */ +import { parseBackendList } from "./backends.mjs"; +import { egressArmed, egressProxyName } from "./egress.mjs"; +import { envValueShown } from "./env-file.mjs"; +import { STACK_KEYS, readStackKeys } from "./podman-stack.mjs"; + +/** How each command says what a shell/file disagreement would make it do, the clause before "while the service ...". */ +const COMMAND_SAYS = Object.freeze({ + up: "up would drive the shell's venue", + init: "init's next steps would be for the shell's venue", + doctor: "doctor would judge the shell's venue", +}); + +/** + * The venue keys this pass decides with (D2): this shell's value where it sets one, else the deployment `.env`'s, read + * with `readStackKeys` exactly as `service install` reads them. Returns `{ env, fromFile, disagreements }` or + * `{ error }`. `env` is the whole environment with the file's keys filled in; `fromFile` is only what the file + * supplied, for doctor's layering; `disagreements` names each key both set differently. + */ +export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up", loader: loaderGiven = null, keys: keysWanted = STACK_KEYS }) { + let keys = {}; + const notes = []; + // The loader of the file on THIS platform, as service.mjs reads it (round 2, E6): systemd's EnvironmentFile= on + // Linux, the sh wrapper on macOS, the cmd wrapper on Windows. Off Linux the venue refuses the host anyway, so a + // line that platform's loader reads differently is noted and never stops `up`. + const linux = platform === "linux"; + // A caller with a mapping of its own passes it (doctor, so every `.env` read it makes uses one loader). + const loader = loaderGiven ?? (linux ? "systemd" : platform === "darwin" ? "shell" : "cmd"); + if (fs.existsSync(envPath)) { + let text = null; + try { + text = String(fs.readFileSync(envPath, "utf8")); + } catch (err) { + // Said rather than silent (round 2 nit): the venue then comes from this shell alone. + notes.push(`${envPath} could not be read (${err?.message}), so PI_BACKENDS, PI_EGRESS and PI_EGRESS_PROXY come from this shell alone`); + } + if (text !== null) { + const read = readStackKeys(text, { loader, path: envPath }); + if (read.error && linux) return { error: read.error }; + if (read.error) notes.push(`${read.error}; off Linux the podman venue refuses this host anyway, so this pass reads the venue from this shell`); + else keys = read.keys; + } + } + const merged = { ...env }; + const fromFile = {}; + const conflicts = []; + // Only the keys the caller allows are taken from the file (doctor's `SERVICE_ENV_KEYS`); all of them by default. + for (const key of STACK_KEYS.filter((k) => keysWanted.includes(k))) { + if (!Object.hasOwn(keys, key)) continue; + if (typeof env[key] === "string") { + // Compared as the worker READS them (round 3, D4), not as strings: ` podman` and `podman,podman` are the list + // `podman`, and refusing them sent an operator to reconcile two values that already agree. + if (venueKeyMeaning(key, env[key]) !== venueKeyMeaning(key, keys[key])) conflicts.push(`${key} is ${quotedShown(env[key])} in this shell and ${quotedShown(keys[key])} in ${envPath}`); + continue; + } + merged[key] = keys[key]; + fromFile[key] = keys[key]; + } + // A KNOWN disagreement is a refusal (round 2, E5), not a warning: this pass would stand up one venue's stack and the + // service installed next would run the other. The file is what the service runs, so it is the one to change unless + // the shell's value was a one-off. + if (conflicts.length > 0) { + return { error: `${conflicts.join("; ")}. ${COMMAND_SAYS[command] ?? COMMAND_SAYS.up} while the service runs the file's. Make them agree: change ${envPath} (what the service reads), or unset the key in this shell` }; + } + return { env: merged, fromFile, notes }; +} + +/** + * What a venue key MEANS to the worker, for comparing two spellings of it: the parsed list for PI_BACKENDS, on or off + * for PI_EGRESS, the resolved name for PI_EGRESS_PROXY exactly as egressProxyName resolves it (empty is the default + * name; NOT trimmed, because nothing that reads the name trims it, so " x" and "x" are different proxies to the worker + * and to podman). A value the worker would refuse keeps its raw spelling, so two different unreadable values still + * disagree and the doctor below names them. + */ +function venueKeyMeaning(key, value) { + try { + if (key === "PI_BACKENDS") return `list:${parseBackendList(value).join(",")}`; + if (key === "PI_EGRESS") return `egress:${egressArmed({ PI_EGRESS: value }) ? "on" : "off"}`; + return `proxy:${egressProxyName({ PI_EGRESS_PROXY: value })}`; + } catch { + return `raw:${value}`; + } +} + +/** A value shown in quotes, so a leading space or an empty value is visible; control characters escaped as ever. */ +function quotedShown(value) { + const shown = envValueShown(value); + return shown.startsWith('"') ? shown : `"${shown}"`; +} diff --git a/worker/src/doctor.mjs b/worker/src/doctor.mjs index bc46b6e4..9bb1bcde 100644 --- a/worker/src/doctor.mjs +++ b/worker/src/doctor.mjs @@ -50,7 +50,8 @@ * no fixAction, because what a failed read-back points at is the image or the runtime. */ import { chmodSync, closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, realpathSync, rmSync, statSync } from "node:fs"; -import { homedir, release as osRelease, tmpdir } from "node:os"; +import { homedir, release as osRelease, tmpdir, userInfo } from "node:os"; +import { STACK_KEYS, readLinger } from "./podman-stack.mjs"; import { dirname, isAbsolute, join, delimiter, posix, resolve, win32 } from "node:path"; import { fileURLToPath } from "node:url"; import { spawn as nodeSpawn } from "node:child_process"; @@ -68,12 +69,14 @@ import { copySkillTree } from "./copy-tree.mjs"; import { SKILL_NAME_RE } from "./flow-gate.mjs"; import { GIT_READ_FLAGS } from "./git-hardening.mjs"; import { resolveBackendName } from "./backend-registry.mjs"; +import { deploymentVenueEnv } from "./deployment-venue.mjs"; +import { PROXY_STATE_FORMAT, parseProxyState, shippedProxyDrift } from "./egress-proxy-state.mjs"; import { ABSENT, ASSERTED, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OBSERVATION_FIX, OBSERVATIONS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, PROPERTY_NAMES, RUNTIME_ADDS_NO_MOUNTS, declarationOf, floorShortfall, parseBackendFloor, parseBackendList, unarmedFloor, unobservedFloor, venuesOf } from "./backends.mjs"; import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_FIRST_START_TIMEOUT_MS, PODMAN_INFO_TIMEOUT_MS, PODMAN_JOB_USER_FIX, decidePodmanJobUser, makePodmanInfoReader, observePodman, podmanConfFix, podmanConfWidening, resolvePodmanImageUser } from "./backend-podman.mjs"; import { PODMAN_PINNED_FLAGS, buildPodmanRunArgs, containerSpec, podmanArgsFromSpec } from "./docker-run.mjs"; import { observeHost } from "./runtime-observations.mjs"; import { endpointShown, makeDockerEndpointResolver, quotedShown } from "./backend-local.mjs"; -import { EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEnv, egressProxyName, networkEndpoints, removeNetworkOrSay } from "./egress.mjs"; +import { DEFAULT_EGRESS_PROXY, STOPPED_PROXY_STATES, EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEnv, egressProxyName, networkEndpoints, removeNetworkOrSay } from "./egress.mjs"; import { runLiveProbes } from "./live-probes.mjs"; import { SANDBOX_TOMBSTONE_STUCK_MS, isSandboxTombstone, sandboxTombstoneAge } from "./sandbox-store.mjs"; import { installedUnitPaths, readUnitSeam, readUnitUser } from "./service.mjs"; @@ -92,6 +95,19 @@ const NODE_FLOOR = [22, 19]; // pi's engine floor (22.19.0) // gh login scopes that reach well past what a job should ever hold — called out by name in the fix line. const BROAD_SCOPES = ["admin:org", "delete_repo", "workflow"]; +/** + * What to do when the podman venue's bounds miss for a reason in the account's systemd setup (issue #453, measured on + * Fedora 44 with Podman 5.8.1), per `observePodman`'s `boundsCause`. Linger leads: it is what keeps the user manager + * running for a service, where a login session keeps it only while that session lasts (a plain ssh started one, + * measured, and a `sudo -iu` shell started none). Which manager a doctor sees does not depend on the shell it runs in: + * with linger off, a `sudo -iu` doctor saw a running manager while another login session of the account was open + * (gate 456), and that ✓ lasted only as long as the session did. + */ +const PODMAN_BOUNDS_FIX = Object.freeze({ + "no-user-manager": "turn on linger for the worker's account: `sudo loginctl enable-linger ` starts its systemd user manager and keeps it running with no one logged in, and `pi-dispatch service install` runs the worker as a systemd user service inside it. A hand-written system unit with `User=` also needs `Wants=user@.service` and `After=user@.service` (docs/podman.md); `pi-dispatch doctor --live` reads pids.max and memory.max back off a real container", + "user-manager-unreachable": "run the worker inside the account's user manager, as a systemd user service (`pi-dispatch service install`, with linger on), where a job's container lands under the manager whichever cgroup manager Podman uses; or let Podman reach the manager over the account's user bus, /run/user//bus (on Debian and Ubuntu the dbus-user-session package provides it), with no DBUS_SESSION_BUS_ADDRESS pointing elsewhere and no `cgroup_manager = \"cgroupfs\"` in its containers.conf; `pi-dispatch doctor --live` reads pids.max and memory.max back off a real container", +}); + // The podman venue's pull of the deployment's default job image into this account's store (issue #433): the fix line's // words and `--fix`'s prompt, one string so the two cannot name different commands. const PODMAN_JOB_IMAGE_PULL = "podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest"; @@ -172,6 +188,10 @@ export async function runDoctor(env = process.env, deps = {}) { readEnvFile = (path) => readFileSync(path, "utf8"), // Issue #345: the host files the runtime-mounts observation reads (Podman's mounts.conf and containers.conf). observationFs = { statSync, readFileSync, readdirSync }, + // Issue #453: this shell's account name, asked of loginctl for linger on the podman venue; and whether this folder + // holds the shipped proxy's two files, so its running proxy's mounts can be compared with them. + userName, + proxyFilesExist, isAlive = defaultIsAlive, pid = process.pid, nonce = randomBytes(6).toString("hex"), @@ -188,7 +208,50 @@ export async function runDoctor(env = process.env, deps = {}) { // so it needs both, and a test cannot drive that sweep while the names come from `process.pid` directly. // `live` rides the shared seams since issue #431 for ONE reader: the podman section, whose allowlist line points at // `--live` on a plain run and has nothing to say on a `--live` run, where the read-back's own canary answers it. - const seams = { cwd, out, spawn, probeValkey, readHosts, fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile, observationFs, isAlive, pid, runTimeouts, live: live === true }; + // Issue #453 (gate round 1): the venue keys, decided as `up`, `init` and `service install` decide them. doctor read + // PI_BACKENDS from its shell alone, so on a podman-only deployment whose `.env` says so (the documented setup) its + // plain run judged docker, while the service it was checking ran podman. This shell wins where it sets a key; the + // deployment `.env` fills the rest; a disagreement is its own ✗, and doctor then judges this shell's values. + // + // ONE read of the file, shared: a regular file only (a FIFO would hang a synchronous read, as `envFileKeys` guards + // against), and the text handed to the first `.env` read the checks below make, so a run still reads the file once. + // A file that is there and cannot be read is said, as `up` says it (gate round 2): the venue then comes from this + // shell alone. An absent one (ENOENT) is simply absent. + const envPath = join(cwd, ".env"); + let envText = null; + let envUnread = null; + try { + if (fileExists(envPath)) { + if (!statSync(envPath).isFile()) envUnread = "not a regular file"; + else envText = String(readEnvFile(envPath)); + } + } catch (err) { + if (err?.code !== "ENOENT") envUnread = err?.code ?? err?.message ?? "error"; + } + let envTextUnused = envText !== null; + const readEnvFileShared = (path) => { + if (envTextUnused && path === envPath) { + envTextUnused = false; + return envText; + } + return readEnvFile(path); + }; + const venueChecks = []; + if (envUnread) { + venueChecks.push({ ok: false, warn: true, label: `${envPath} could not be read (${envUnread}), so PI_BACKENDS, PI_EGRESS and PI_EGRESS_PROXY come from this shell alone`, fix: "make it a readable regular file, as the service's loader needs it to be" }); + } + const venue = deploymentVenueEnv({ env, fs: { existsSync: () => envText !== null, readFileSync: () => envText }, envPath, platform, command: "doctor", loader: serviceEnvLoader(platform), keys: serviceEnvKeys(STACK_KEYS) }); + if (venue.error) { + venueChecks.push({ ok: false, label: `which venue this deployment runs is unknown: ${venue.error}`, fix: "doctor judged this shell's values below; make the shell and the deployment's .env agree, then re-run doctor" }); + } else { + for (const note of venue.notes) venueChecks.push({ ok: false, warn: true, label: note, fix: "fix that line so the service and this shell read the same venue" }); + const read = Object.entries(venue.fromFile); + if (read.length > 0) { + venueChecks.push({ ok: true, label: `venue keys read from ${envPath} (${read.map(([key, value]) => `${key}=${quotedShown(value)}`).join(", ")}), as the service reads them: this shell does not set them` }); + env = venue.env; + } + } + const seams = { cwd, out, spawn, probeValkey, readHosts, fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, isAlive, pid, runTimeouts, live: live === true, venueChecks, userName, proxyFilesExist, serviceEnvFile: envText === null ? null : { text: envText, path: envPath, loader: serviceEnvLoader(platform) } }; let checks = await collectChecks(env, seams); let failed = render(checks, out); @@ -295,9 +358,10 @@ export async function defaultPromptFn(question, { input = process.stdin, output } /** - * The values `/.env` sets for NAMED keys, or `{}`. The only place in this project that reads a `.env` - * file's contents at all (issue #357), and the narrowing is what makes it an addition rather than a - * reversal: + * The values `/.env` sets for NAMED keys, or `{}`. Doctor's first read of a `.env` (issue #357), and the + * narrowing is what makes it an addition rather than a reversal. It is NOT the only reader: `up` and `service + * install` read the venue keys since #430 (`readStackKeys`), and since #453 doctor reads `SERVICE_ENV_KEYS` below, + * whose values DO steer what doctor spawns, connects to and offers as fixes. For these two keys: * * - it reads to decide WHAT DOCTOR SAYS, never to configure anything. No value from here reaches a * config, an argv, a container env, or a fix that writes; @@ -316,6 +380,27 @@ export async function defaultPromptFn(question, { input = process.stdin, output */ export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE"]); +/** + * The keys doctor takes from the deployment's `.env` as the SERVICE's values where its own shell sets none (issue + * #453): the venue keys `up` and `service install` read, the Valkey the worker connects to, the provider, and that + * provider's key variables (pi's own names for it, passed in, and judged for presence only, never printed). ONE + * structural allowlist: every `.env` value doctor acts on passes `serviceEnvKeys` first. Unlike + * `ENV_FILE_READABLE_KEYS` these values steer what doctor does: which venue it judges and so which runtime it spawns, + * which Valkey it connects to, and which fixes it offers. + */ +export const SERVICE_ENV_KEYS = Object.freeze([...STACK_KEYS, "VALKEY_URL", "PI_PROVIDER"]); + +/** `keys` narrowed to `SERVICE_ENV_KEYS` plus the provider's own key variables. */ +export function serviceEnvKeys(keys, providerKeys = []) { + return keys.filter((k) => SERVICE_ENV_KEYS.includes(k) || providerKeys.includes(k)); +} + +/** The loader of the deployment's `.env` on this platform, as `service.mjs` renders it: systemd, the cmd wrapper, or a + * sourcing shell everywhere else. ONE mapping for every `.env` read doctor makes. */ +export function serviceEnvLoader(platform) { + return platform === "linux" ? "systemd" : platform === "win32" ? "cmd" : "shell"; +} + export function envFileKeys(path, keys, { fileExists, readEnvFile, statFile = statSync, platform = process.platform }) { if (typeof readEnvFile !== "function" || !fileExists(path)) return {}; // A REGULAR file or nothing. Every other host read in this module is bounded one way or another, and a @@ -347,7 +432,7 @@ export function envFileKeys(path, keys, { fileExists, readEnvFile, statFile = st // `worker-env-wrapper.sh` (a sourcing shell) on darwin, `worker-env-wrapper.cmd` on win32. Reporting a // reading from a loader this deployment does not use is how doctor told half its operators the wrong // file under the previous shape, which asked systemd's grammar on every platform. - const serviceLoader = platform === "linux" ? "systemd" : platform === "win32" ? "cmd" : "shell"; + const serviceLoader = serviceEnvLoader(platform); const service = readEnvAssignments(text, allowed, { loader: serviceLoader }); // The shell reading stays beside it on POSIX, because a `.env` is also sourced by hand // (`set -a; . ./.env`) and because the two disagree about an `export` line, which is the third state @@ -524,8 +609,21 @@ export async function collectChecks(env, seams) { const { cwd, spawn, probeValkey, readHosts = defaultReadHosts, fileExists, nodeVersion, platform, agentDir = agentDirFrom(env), home = safeHomeDir(), underTemp = underOsTempDir, providerOracle = defaultProviderOracle, facts = null, readEnvFile = null, stat: statSeam = statSync, runTimeouts = RUN_TIMEOUTS } = seams; const jobImage = env.PI_JOB_IMAGE ?? "pi-job:latest"; - const valkeyUrl = env.VALKEY_URL ?? "redis://127.0.0.1:6379"; - const provider = env.PI_PROVIDER ?? "anthropic"; + // Issue #453 (gate round 2): the keys the service takes from the deployment's `.env` and a shell running doctor does + // not, read with the service's own loader (`serviceEnvFile`, from runDoctor's one read), so the ladder's own + // `doctor` step judges the deployment it was told to build: VALKEY_URL and PI_PROVIDER here, the provider's key + // below. This shell wins where it sets one. A value is taken only from a plain line the loader reads as written. + const { serviceEnvFile = null } = seams; + const fromServiceFile = (wanted, providerKeys = []) => { + if (!serviceEnvFile) return {}; + const keys = serviceEnvKeys(wanted, providerKeys); + const read = readEnvAssignments(serviceEnvFile.text, keys, { loader: serviceEnvFile.loader }); + return Object.fromEntries(keys.filter((k) => env[k] === undefined && read[k]?.plain && typeof read[k].value === "string").map((k) => [k, read[k].value])); + }; + const fileKeys = fromServiceFile(["VALKEY_URL", "PI_PROVIDER"]); + const valkeyUrl = env.VALKEY_URL ?? fileKeys.VALKEY_URL ?? "redis://127.0.0.1:6379"; + const provider = env.PI_PROVIDER ?? fileKeys.PI_PROVIDER ?? "anthropic"; + const fromFileNote = (keys) => (keys.length > 0 ? ` -- ${keys.join(" and ")} read from ${serviceEnvFile.path}, as the service reads it; this shell does not set ${keys.length === 1 ? "it" : "them"}` : ""); const checks = []; checks.push(nodeCheck(nodeVersion)); @@ -550,10 +648,11 @@ export async function collectChecks(env, seams) { fixAction: { tier: "silent", describe: "pi-dispatch init", - run: async ({ out }) => { + run: async ({ out, platform }) => { // Lazy import: a doctor run without --fix (or without this failure) never loads init. const { runInit } = await import("./init.mjs"); - const code = runInit(cwd, { out }); + // With doctor's environment and platform, so the next steps match the venue (issue #453). + const code = runInit(cwd, { out, env, platform }); return { ok: code === 0, note: "scaffolded by `pi-dispatch init` (create-only: existing files were kept)" }; }, }, @@ -562,6 +661,9 @@ export async function collectChecks(env, seams) { // Right after `.env present`, because it answers the same question that check raises: where DOES this // deployment's environment come from. [] unless a seam is configured (issue #216). checks.push(...(await envSetupChecks(env, seams))); + // Where the venue keys came from (issue #453), beside the other answers to "where does this deployment's environment + // come from". Empty unless the file supplied a key, or the two disagree. + checks.push(...(seams.venueChecks ?? [])); // Issue #354: which venues run jobs here. An unparseable PI_BACKENDS reads as the unset default, so every docker line // below is exactly what it always was, and the backend section is what fails on it (the worker refuses to boot). @@ -885,7 +987,7 @@ export async function collectChecks(env, seams) { checks.push({ ok: run.code === 0, label: run.ended === "timeout" ? `podman: trigger job image present in this account's Podman store (${img}) -- podman did not answer` : `podman: trigger job image present in this account's Podman store (${img})`, - fix: `pull, load or build it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: podman pull ${img} (or \`docker save ${img} | podman load\`) -- a trigger names it in run.image, and jobs run with --pull=never, so the worker never fetches it at job time`, + fix: `pull, load or build it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: podman pull ${img} -- a trigger names it in run.image, and jobs run with --pull=never, so the worker never fetches it at job time`, }); if (run.code !== 0) continue; const entry = await runCmdCapture(spawn, "podman", ["image", "inspect", "--format={{json .Config.Entrypoint}}", img]); @@ -1334,7 +1436,7 @@ export async function collectChecks(env, seams) { checks.push({ ok: await probeValkey(valkeyUrl), - label: `Valkey reachable (${valkeyUrl})`, + label: `Valkey reachable (${urlShown(valkeyUrl)})${fromFileNote(fileKeys.VALKEY_URL !== undefined ? ["VALKEY_URL"] : [])}`, // Issue #433: without `local`, docker is not this deployment's runtime and may not be installed at all, so neither the // compose file nor a `docker run` is offered. The words hold before and after the setup commands learn Podman: the // podman guide says how to start Valkey under the account, whichever command does it. @@ -1369,7 +1471,10 @@ export async function collectChecks(env, seams) { // the image is actually present -- an absent one is already reported above, and a second line saying // its digest is unknown would be noise on a fault the operator has been told about. const fleet = await readHosts(valkeyUrl); - const peers = (fleet.hosts ?? []).filter((h) => h.name !== workerNameOf(env)); + // Printable before anything is compared or said (issue #453, gate round 3): since a folder's `.env` now chooses the + // Valkey doctor reads, a host row is another party's text, and a control byte in a name or zone must not reach the + // terminal. The registry's own charset already refuses them at the source; this is the reader not relying on it. + const peers = (fleet.hosts ?? []).map((h) => ({ ...h, name: printable(h.name), tz: h.tz ? printable(h.tz) : h.tz })).filter((h) => h.name !== workerNameOf(env)); // Read only when there is a peer to compare against. Every line below is gated on a peer existing, and // the SUBPROCESS has to be too: otherwise every `doctor` run on every single-host deployment spawns an // extra docker call whose answer nothing reads. @@ -1442,7 +1547,7 @@ export async function collectChecks(env, seams) { } } else if (fleet.unreachable) { // Said, rather than silently absent: "no peers" and "could not ask" are different facts. - checks.push({ ok: true, label: `Fleet: could not read the host registry (${fleet.unreachable})` }); + checks.push({ ok: true, label: `Fleet: could not read the host registry (${printable(fleet.unreachable)})` }); } // Which variable holds a provider's key is PI'S fact, asked of pi rather than copied (issue #286). @@ -1454,7 +1559,12 @@ export async function collectChecks(env, seams) { // deployments with different remedies. // Resolved ONCE and reused by the trigger-secret clash check further down: one seam call, one answer. const oracle = await providerOracle(); - checks.push(providerKeyCheck({ provider, env, agentDir, oracle, nodeOk: checks[0]?.ok })); + // The key's presence only, from the file where this shell sets none of the provider's variables; never its value. + const keyCandidates = typeof oracle?.providerKeyCandidates === "function" ? oracle.providerKeyCandidates(provider) : []; + const keyFromFile = keyCandidates.some((name) => (env[name] ?? "") !== "") ? {} : fromServiceFile(keyCandidates, keyCandidates); + const keyCheck = providerKeyCheck({ provider, env: { ...env, ...keyFromFile }, agentDir, oracle, nodeOk: checks[0]?.ok }); + const keyNamed = Object.keys(keyFromFile).filter((name) => keyCheck.label?.includes(`: ${name})`)); + checks.push({ ...keyCheck, label: `${keyCheck.label}${fromFileNote([...(fileKeys.PI_PROVIDER !== undefined ? ["PI_PROVIDER"] : []), ...keyNamed])}` }); // REQ-GLOBAL-PI-OVERLAY: read the extensions opt-out through the WORKER's own parser, so doctor reports @@ -3353,7 +3463,10 @@ const CANARY_LEFTOVER_FIX = "remove whatever is still on it first (a probe conta async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpoint = { local: null, reason: "not resolved" } }) { // `pid` and `isAlive` default here as well as riding the seams, so a caller that predates issue #350 still // gets the process's own answer rather than a name ending in `undefined`. - const { spawn, pid = process.pid, isAlive = defaultIsAlive } = seams; + // `proxyFilesExist` asks whether this folder holds the proxy's two files, which is what makes it a deployment folder + // whose mounts the running proxy can be compared with. A seam of its own, on the real disk by default: the shared + // `fileExists` answers yes to everything in most tests, and "doctor run from some other folder" is the common case. + const { spawn, pid = process.pid, isAlive = defaultIsAlive, proxyFilesExist = existsSync } = seams; // The SAME parse the worker boots with (egress.mjs), never a second `=== "1"`: doctor reporting a // policy that is off, or nothing about one that is on, is worse than doctor not checking at all. // A malformed value is the worker's boot failure to report, not doctor's to guess at, so it reads as @@ -3407,18 +3520,78 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo // `docker inspect` on the container, not `ps`: it answers present-vs-absent and running-vs-stopped in // one call, and those are two different fixes. The FIELD_SEP habit is image-preflight.mjs's -- neither // a boolean nor a health word can contain "|". - const state = await runCmdCapture(spawn, "docker", ["inspect", `--format={{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}`, proxy], { stdoutOnly: true }); - const [running, health] = state.code === 0 ? state.output.trim().split("|") : []; - const up = running === "true"; - checks.push({ + // + // Issue #453 (gate rounds 1 and 2): RUNNING is `.State.Status` "running", and for the SHIPPED name the container + // must be CURRENT, the pinned squid with its own entrypoint and command and this deployment folder's two files + // mounted, or its policy is not this deployment's (`egress-proxy-state.mjs`). One inspect, as `up` reads it, with the + // health word in the same answer. + const state = await runCmdCapture(spawn, "docker", ["inspect", PROXY_STATE_FORMAT, proxy], { stdoutOnly: true }); + const parsed = state.code === 0 ? parseProxyState(state.output) : null; + const health = parsed?.health ?? "none"; + const up = parsed?.status === "running"; + const custom = proxy !== DEFAULT_EGRESS_PROXY; + // CURRENT, for the shipped name, judged whether it runs or not (gate round 2): a stopped proxy of another folder is + // one `up` offers to replace, so its fix must say that, not compose, which would start it as it is. The mounts are + // compared only where this folder holds the two files, which is a deployment folder (run elsewhere, doctor cannot + // know which folder the service uses), and only where every bind source resolves on this host; otherwise that half + // is said to be unknown, never stale. + const inFolder = ["egress-allowlist.conf", "deploy/egress-proxy.conf"].every((f) => proxyFilesExist(join(seams.cwd, f))); + const judged = !custom && parsed ? shippedProxyDrift(parsed, { cwd: seams.cwd, platform: seams.platform ?? process.platform, realpath: (p) => realpathSync(p), compareMounts: inFolder }) : { drift: [], unknown: null }; + const replaceFix = `\`pi-dispatch up\` from the deployment folder offers to replace it with the shipped one (docker rm -f ${proxy}, then the shipped run)`; + // The worker's simple rule (egress.mjs): paused, exited and dead refuse every job, so ✗; any other state is one the + // worker retries through (restarting, created, ...), so ⚠, and the fix says jobs wait rather than fail. + // `restarting` is a crash loop that fails every job (one retry, then failed), so ✗; ⚠ only for a transient word. + const retried = parsed !== null && !up && !STOPPED_PROXY_STATES.has(parsed.status) && parsed.status !== "restarting"; + // A RUNNING stale proxy (exec round 3) gets no "✓ running" line: the ✗ below says what it is, and carries the state + // `--live` reads, so the output never reads healthy right before saying the proxy is not this deployment's. + const staleRunning = up && !custom && judged.drift.length > 0; + const proxyCheck = { ok: up, - label: up ? `Egress proxy running (${proxy})` : state.code === 0 ? `Egress proxy is stopped (${proxy})` : `Egress proxy is not on this host (${proxy})`, - fix: "docker compose -f deploy/docker-compose.yml --profile egress up -d -- the egress policy refuses every job pre-spend while this is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)", + ...(retried ? { warn: true } : {}), + label: up + ? `Egress proxy running (${proxy})` + : state.code === 0 + ? parsed + ? `Egress proxy is ${parsed.status !== "exited" ? parsed.status : "stopped"} (${proxy})` + : `Egress proxy exists but its state could not be read (${proxy})` + : `Egress proxy is not on this host (${proxy})`, + // PI_EGRESS_PROXY names the operator's own proxy, which neither compose nor `up` starts (they start the shipped + // name), so its fix is to start that container, not to run compose. + fix: retried + ? `a job meanwhile is retried once, then failed (the worker does not refuse it outright on this state); if it stays ${parsed.status}, \`docker logs ${proxy}\` says why` + : parsed?.status === "restarting" + ? `its squid keeps exiting and its restart policy keeps bringing it back, so every job is retried once, then failed; \`docker logs ${proxy}\` says why` + : custom + ? `PI_EGRESS_PROXY names your own proxy, which neither compose nor \`pi-dispatch up\` starts: ${state.code === 0 ? `\`docker ${parsed?.status === "paused" ? "unpause" : "start"} ${proxy}\`` : `create and start ${proxy} yourself`} -- the egress policy refuses every job pre-spend while it is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)` + : judged.drift.length > 0 + ? `it is not this deployment's either (${judged.drift.join("; ")}): ${replaceFix} -- the egress policy refuses every job pre-spend while it is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)` + : parsed?.status === "paused" + ? `docker unpause ${proxy} -- as \`pi-dispatch up\` offers; the egress policy refuses every job pre-spend while it is paused, which costs no budget but runs nothing (PI_EGRESS=0 opts out)` + : "docker compose -f deploy/docker-compose.yml --profile egress up -d -- the egress policy refuses every job pre-spend while this is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)", // Not rendered: `--live`'s peer probe (issue #344) needs the proxy's name and whether it is up, and reads them here // rather than asking docker a second time. proxyState: { proxy, running: up }, - }); + }; + if (!staleRunning) checks.push(proxyCheck); if (!up) return checks; + if (!custom) { + if (judged.drift.length > 0) { + checks.push({ + ok: false, + label: `Egress proxy is running but is not this deployment's (${proxy}): ${judged.drift.join("; ")}`, + fix: `${replaceFix}; until then every job's egress runs through a policy this deployment did not ship`, + proxyState: proxyCheck.proxyState, + }); + } + if (judged.unknown) { + checks.push({ + ok: false, + warn: true, + label: `Egress proxy's mounts could not be compared on this host (${proxy}): ${judged.unknown}`, + fix: "its image, entrypoint and command were compared; check its two mounts by hand (`docker inspect --format '{{json .Mounts}}' " + proxy + "`): /etc/squid/squid.conf must be this deployment's deploy/egress-proxy.conf and /etc/pi-dispatch/allowlist.conf its egress-allowlist.conf", + }); + } + } // Advisory on purpose, and deliberately NOT what the money gate reads. A healthcheck can flap, and a // pre-spend gate that refuses on a flapping signal drops real work while one that retries on it burns @@ -3449,7 +3622,9 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo // bounded step runner, so docker's spawns, and what a probe that never launched leaves for the teardown, are exactly // what they were: pinned in doctor.test.mjs against the output and argv captured before the move. const canary = await runEgressCanary({ run: liveRunVia(spawn), probeRun: (args) => runCmdCapture(spawn, "docker", args), proxy, image: jobImage, pid }); - checks.push(...canary.checks); + // Through a stale proxy the canary reads THAT proxy's policy (exec round 3): a failure is the proxy's to fix, never + // this deployment's allowlist, which it may not even mount. + checks.push(...(staleRunning ? canary.checks.map((c) => (c.ok ? c : { ...c, fix: `the canary ran through ${proxy}, which is not this deployment's proxy (above), so this says nothing about egress-allowlist.conf: ${replaceFix}, then re-run doctor` })) : canary.checks)); return checks; } @@ -4167,11 +4342,16 @@ export function backendChecks(env, { endpoint = null, daemon = null, fs = { stat const unread = observations[d.observedBy] !== false; // The endpoint is the operator's to point, as docker's is; bounds and mounts are this account's Podman setup. const by = d.observedBy === PODMAN_SERVICE_LOCAL ? "the operator" : "this account's Podman setup"; + // Issue #453: when the bounds miss for a reason in the account's systemd setup (no user manager, or one Podman + // cannot reach), this line is the ONE that says so (the podman section below does not repeat it), and its fix is + // that cause's rather than the floor's generic remedy. Keyed on the observation's own cause, so a miss with + // another cause keeps OBSERVATION_FIX. + const causeFix = d.observedBy === PODMAN_BOUNDS_DELEGATED && !unread ? PODMAN_BOUNDS_FIX[podman.boundsCause] : undefined; checks.push({ ok: false, warn: true, label: `${name}: ${property} is ASSERTED by ${by}, not enforced: ${podman.evidence?.[d.observedBy] ?? "not observed"}`, - fix: !unread ? OBSERVATION_FIX[d.observedBy] : podman.reasons?.[d.observedBy] === "file-unread" ? fileUnreadFix(OBSERVATIONS[d.observedBy]) : `nothing shows whether ${OBSERVATIONS[d.observedBy]}; fix what stops \`podman info\` answering for the worker's account, then re-run doctor`, + fix: causeFix ?? (!unread ? OBSERVATION_FIX[d.observedBy] : podman.reasons?.[d.observedBy] === "file-unread" ? fileUnreadFix(OBSERVATIONS[d.observedBy]) : `nothing shows whether ${OBSERVATIONS[d.observedBy]}; fix what stops \`podman info\` answering for the worker's account, then re-run doctor`), }); continue; } @@ -4418,7 +4598,9 @@ function fileUnreadFix(what) { * or retries each podman job. The image and the proxy are ✗ as they are for local: every job is refused without them. * * What goes wrong with the observations is NOT repeated here: the backend section above names each degraded word with - * what was seen, so this section prints the bounds line only when they hold. + * what was seen, so this section prints the bounds line only when they hold. That includes a missing systemd user + * manager (issue #453): the backend section's isolation line carries it as evidence with its own fix, and the floor + * line there carries the ✗, so a line here would only repeat both. */ export async function podmanChecks(env, seams, { jobImage }) { const { spawn, home = safeHomeDir(), jobUserIdentity: ids = {}, observationFs = { statSync, readFileSync, readdirSync }, runTimeouts = RUN_TIMEOUTS } = seams; @@ -4481,8 +4663,23 @@ export async function podmanChecks(env, seams, { jobImage }) { // The bounds, said when they hold; see the header for why a miss is left to the backend section. if (observed.observations[PODMAN_BOUNDS_DELEGATED] === true) { - const controllers = Array.isArray(info?.controllers) ? info.controllers.filter((c) => typeof c === "string" && /^[a-z_]{1,20}$/.test(c)) : []; + // The user manager's delegated list the answer rests on (issue #453), not `podman info`'s, which is the caller's cgroup. + const controllers = Array.isArray(observed.boundsControllers) ? observed.boundsControllers : []; checks.push({ ok: true, label: `podman: cgroup v2 controllers are delegated to this account (${controllers.join(", ")}), so a job's pid, memory and cpu bounds are applied` }); + // Issue #453 (gate round 1, measured): with linger off the user manager runs only while the account has a login + // session, and a doctor run then reads it running, from any shell, `sudo -iu` included. That ✓ ends with the last + // session, and a service-run worker then gets no bounds. Asked of loginctl as `up` asks it; an answer it cannot + // read says nothing, since the ✓ above is still what this read saw. + const user = userNameOf(seams); + const linger = user ? await readLinger(user, (cmd, args) => runCmdCapture(spawn, cmd, args, { stdoutOnly: true, timeoutMs: 5000 })) : null; + if (linger === false) { + checks.push({ + ok: false, + warn: true, + label: `podman: linger is off for ${user}, so the user manager above runs only while a login session of the account is open: when the last one ends, a worker run as a user service stops with it, and one run as a system unit with User= runs on with no bounds (refused per job under an isolation floor)`, + fix: `sudo loginctl enable-linger ${user} -- it keeps the manager running with no one logged in, which is what a service needs`, + }); + } } // SELinux, and what it means for a job's mounts: the worker's rule (#355) is `:Z` on a job's own directories only. @@ -4518,7 +4715,7 @@ export async function podmanChecks(env, seams, { jobImage }) { checks.push({ ok: false, label: `podman: job image is not in this account's Podman store (${jobImage})`, - fix: `pull or load it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: ${jobImage === "pi-job:latest" ? PODMAN_JOB_IMAGE_PULL : `podman pull ${jobImage}`} (or \`docker save ${jobImage} | podman load\`) -- jobs run with --pull=never, so the worker never fetches it`, + fix: `pull or load it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: ${jobImage === "pi-job:latest" ? PODMAN_JOB_IMAGE_PULL : `podman pull ${jobImage}`} -- jobs run with --pull=never, so the worker never fetches it`, ...(!localUsed && jobImage === "pi-job:latest" ? { fixAction: { @@ -4571,12 +4768,20 @@ export async function podmanChecks(env, seams, { jobImage }) { const proxy = egressProxyName(env); let proxyRunning = null; if (armed !== false) { - const state = await runCmdCapture(spawn, "podman", ["inspect", "--format={{.State.Running}}", proxy], { stdoutOnly: true }); - proxyRunning = state.code === 0 && state.output.trim() === "true"; + // `.State.Status` (issue #453): only `running` carries traffic; Podman reports paused and between-restarts states + // with their own words (measured on 4.9.3 and 5.8.1). + const state = await runCmdCapture(spawn, "podman", ["inspect", "--format={{.State.Status}}", proxy], { stdoutOnly: true }); + const status = state.code === 0 ? state.output.trim() : null; + proxyRunning = status === "running"; + // The worker's simple rule (egress.mjs), as on docker: a state it retries through is ⚠, one it refuses on is ✗. + const retried = state.code === 0 && !proxyRunning && /^[a-z]{1,20}$/.test(status ?? "") && !STOPPED_PROXY_STATES.has(status) && status !== "restarting"; checks.push({ ok: proxyRunning, - label: proxyRunning ? `podman: egress proxy running under this account's Podman (${proxy})` : state.code === 0 ? `podman: egress proxy is stopped under this account's Podman (${proxy})` : `podman: egress proxy is not under this account's Podman (${proxy})`, - fix: "start it as the worker's account, under the same rootless Podman, on a named bridge network (docs/podman.md): a job's --internal network reaches nothing on the host, so a proxy under docker or another account cannot serve it. Every podman job is refused pre-spend while this is down (PI_EGRESS=0 opts out)", + ...(retried ? { warn: true } : {}), + label: proxyRunning ? `podman: egress proxy running under this account's Podman (${proxy})` : state.code === 0 ? `podman: egress proxy is ${status && status !== "exited" ? status : "stopped"} under this account's Podman (${proxy})` : `podman: egress proxy is not under this account's Podman (${proxy})`, + fix: retried + ? `a podman job meanwhile is retried once, then failed (the worker does not refuse it outright on this state); if it stays ${status}, \`podman logs ${proxy}\` says why` + : "start it as the worker's account, under the same rootless Podman, on a named bridge network (docs/podman.md): a job's --internal network reaches nothing on the host, so a proxy under docker or another account cannot serve it. Every podman job is refused pre-spend while this is down (PI_EGRESS=0 opts out)", }); // Said, never implied by the ✓ above: running is not the same as enforcing the allowlist. Issue #431: on this venue // the canary that reads it back runs with `--live` (`podmanLiveChecks`), not here, so a plain run says where it is @@ -4913,7 +5118,7 @@ function readBackChecks({ venue, bin, result, user, ids, relabel, facts, userFix * podman venue is never sent to `docker ps -a`. "Docker Desktop" and "rootful Podman" are proper names and stay. */ function liveFailFix(bin, key) { - const text = LIVE_FAIL_FIX[key]; + const text = (bin === "podman" ? LIVE_FAIL_FIX_PODMAN[key] : undefined) ?? LIVE_FAIL_FIX[key]; return text === undefined ? text : forRuntime(text, bin); } @@ -5043,6 +5248,53 @@ const LIVE_FAIL_FIX = { localFolders: "a bind-mounted host folder did not behave as one a job edits in place", }; +/** + * Where rootless Podman's cause is not docker's with the name swapped (issue #453). A failed isolation read-back on + * podman is, in the measured case, an account with no systemd user manager running, which the isolation line names; + * "cgroup delegation" alone sent the operator to controllers that already read delegated. + */ +const LIVE_FAIL_FIX_PODMAN = { + isolation: `the runtime did not apply a bound the worker passes: on rootless Podman that is the account's systemd user manager not running, Podman not reaching it, or a controller not delegated to it (the "isolation is ASSERTED" line above names which, when the static read sees it) -- jobs on this host do not have the boundary the table declares. ${capitalized(PODMAN_BOUNDS_FIX["no-user-manager"])}`, +}; + +/** This shell's account name, for loginctl; a seam so a test names the account it models. */ +function userNameOf(seams) { + if (typeof seams.userName === "function") return seams.userName(); + try { + return userInfo().username; + } catch { + return null; + } +} + +/** + * A URL as doctor may print it (issue #453, gate round 3 and the re-review): scheme, host, port and database only, + * never the userinfo (a password or a username that is a token), the query (`password=`, a token) or the fragment; + * `` without a host, and `` when it does not parse, since then nothing can say where a + * credential in it starts or ends. + */ +export function urlShown(url) { + let parsed; + try { + parsed = new URL(String(url)); + } catch { + return ""; + } + // scheme://host:port/db and nothing else (round-cap re-review): no userinfo at all, no query, no fragment. + if (!parsed.hostname) return ""; + return printable(`${parsed.protocol}//${parsed.host}${parsed.pathname && parsed.pathname !== "/" ? parsed.pathname : ""}`); +} + +/** Text with every C0 and C1 control blanked, for a line another party's value reaches. */ +function printable(text) { + return String(text ?? "").replace(/[\u0000-\u001f\u007f-\u009f]/g, " "); +} + +/** A fix line opening a sentence of its own. */ +function capitalized(text) { + return `${text.charAt(0).toUpperCase()}${text.slice(1)}`; +} + /** A PID-liveness check for the stale-fixture sweep: EPERM means alive and owned by someone else. */ function defaultIsAlive(pid) { try { diff --git a/worker/src/egress-proxy-state.mjs b/worker/src/egress-proxy-state.mjs new file mode 100644 index 00000000..caec554b --- /dev/null +++ b/worker/src/egress-proxy-state.mjs @@ -0,0 +1,151 @@ +/** + * THE SHIPPED EGRESS PROXY'S STATE, read one way by `up` and `doctor` on the docker venue (issue #453, gate rounds 1 + * and 2). + * + * Two questions, both answered from one `docker inspect`: + * - RUNNING is `{{.State.Status}}` === "running". Measured on Docker Engine 29.8.1: a paused container reads + * `paused` with `.State.Running` TRUE, and one a restart policy is cycling reads `restarting` with `.State.Running` + * TRUE (39 of 40 samples); Podman reports a paused one `paused` with Running false and one between restarts + * `stopped` (4.9.3 and 5.8.1). Only the status word tells them from a proxy that carries traffic. + * - CURRENT: a container of the shipped name made from another image (an older squid digest), with another + * entrypoint or command, or with other files mounted, is not the proxy this deployment ships, and `docker start` + * of it, or a "present" that leaves it running, keeps that policy. Compared: the image's digest, the entrypoint and + * command against the pinned image's own (read from its registry config, identical on every platform in the + * manifest list), and the mounts: the two bind sources against this folder's `deploy/egress-proxy.conf` and + * `egress-allowlist.conf`, the image's two anonymous volumes allowed, anything else stale. + * + * MEASURED (Docker Engine 29.8.1, compose 5.5.1, rootful, gate round 2; pinned in egress-proxy-state.test.mjs): a proxy + * made by `docker compose --profile egress up -d` and one made by `up`'s argv carry the pinned reference exactly as + * written, the image's own entrypoint and command, the two binds and the image's two anonymous volumes; a folder + * reached through a symlink is reported by the symlink's path, which the realpath below makes the same folder. + * + * THE MOUNTS ARE COMPARED WHERE THEY CAN BE. A runtime that reports a source as a path of its own VM (Docker Desktop's + * `/host_mnt/...` and `/run/desktop/mnt/host/...`, not measured here) gives sources nothing here can resolve, and + * calling that proxy stale would have doctor fail forever and `up --yes` remove a healthy proxy on every run. So a + * source under one of those prefixes, or any unresolvable source on a macOS or Windows host, makes the mount + * comparison UNKNOWN, said as such and never the ground for a removal. On a Linux host an unresolvable source is gone, + * and that proxy is stale. The image, entrypoint and command are compared either way. + * + * Only the SHIPPED name is judged current or stale: a container `PI_EGRESS_PROXY` names is the operator's own. + */ +import { resolve } from "node:path"; + +/** The pinned squid, as `deploy/docker-compose.yml` and `up`'s run argv name it (a test binds the three). */ +export const EGRESS_PROXY_IMAGE = "ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029"; + +/** + * The pinned image's own entrypoint and command, and its anonymous volumes, from its registry config (read on + * 2026-09-27 for every linux platform in the manifest list: amd64, arm64, ppc64le, s390x, all identical). Neither + * compose nor `up` sets either, so a container made by them carries these. + */ +export const EGRESS_PROXY_ENTRYPOINT = Object.freeze(["entrypoint.sh"]); +export const EGRESS_PROXY_CMD = Object.freeze(["-f", "/etc/squid/squid.conf", "-NYC"]); +const IMAGE_VOLUMES = Object.freeze(["/var/log/squid", "/var/spool/squid"]); + +/** Where Docker Desktop reports a bind source from its own VM (not measured here; the documented shapes). */ +const VM_PREFIXES = Object.freeze(["/host_mnt/", "/run/desktop/mnt/host/"]); + +/** + * The inspect format both callers use: one JSON object, so no field can be split on a separator a path might hold. + * `health` is doctor's advisory line; the rest is this module's. + */ +export const PROXY_STATE_FORMAT = + '--format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}}'; + +/** Where the proxy's two files are mounted inside it (compose's and `up`'s mounts alike). */ +const SQUID_CONF = "/etc/squid/squid.conf"; +const ALLOWLIST = "/etc/pi-dispatch/allowlist.conf"; + +/** + * `{ status, health, image, entrypoint, cmd, mounts, networks }` from `PROXY_STATE_FORMAT`'s stdout, or null when it + * is not that shape. `mounts` is the list as `[{ type, source, destination }]`, `networks` the attached names. + */ +export function parseProxyState(stdout) { + let body; + try { + body = JSON.parse(String(stdout ?? "").trim()); + } catch { + return null; + } + if (!body || typeof body !== "object" || typeof body.status !== "string" || !/^[a-z]{1,20}$/.test(body.status)) return null; + // An argv as a list. Podman 4.9.3's native inspect renders `.Config.Entrypoint` as a STRING (`"entrypoint.sh"`, and + // `""` for none; measured, gate round 3), where Docker Engine 29.8.1 and Podman 5.8.1 (native and through its Docker + // API) render a list; docker renders none as `null`. Each is normalised, so a correctly shipped proxy never reads + // stale on the runtime's spelling. + const strings = (v) => (Array.isArray(v) && v.every((x) => typeof x === "string") ? v : typeof v === "string" ? (v === "" ? [] : [v]) : v === null || v === undefined ? [] : null); + return { + status: body.status, + // Podman's Docker API answers "" where docker's template says "none" (measured, 5.8.1): both are "no healthcheck". + health: typeof body.health === "string" && body.health !== "" ? body.health : "none", + image: typeof body.image === "string" ? body.image : "", + entrypoint: strings(body.entrypoint), + cmd: strings(body.cmd), + mounts: (Array.isArray(body.mounts) ? body.mounts : []).filter((m) => m && typeof m === "object").map((m) => ({ type: String(m.Type ?? ""), source: String(m.Source ?? ""), destination: String(m.Destination ?? "") })), + networks: body.networks && typeof body.networks === "object" ? Object.keys(body.networks) : [], + }; +} + +/** The digest an image reference pins, or null. */ +function digestOf(image) { + return /@(sha256:[0-9a-f]{64})$/.exec(String(image ?? ""))?.[1] ?? null; +} + +const sameList = (a, b) => Array.isArray(a) && a.length === b.length && a.every((x, i) => x === b[i]); + +/** + * What makes a container of the shipped name not this deployment's proxy, as `{ drift, unknown }`: `drift` the + * sentences that make it stale, `unknown` why the mounts could not be compared here (null when they were). + * `realpath` throws for a path that does not resolve on this host. `compareMounts: false` judges the image, entrypoint and + * command alone, for a caller that does not know which folder the service uses. `platform` decides whether a source + * that does not resolve can be a path of the runtime's own VM (below). + */ +export function shippedProxyDrift(state, { cwd, realpath = (p) => p, compareMounts = true, platform = "linux" }) { + const drift = []; + if (digestOf(state.image) !== digestOf(EGRESS_PROXY_IMAGE)) drift.push(`it was created from ${state.image || "an unnamed image"}, not the pinned ${EGRESS_PROXY_IMAGE}`); + if (!sameList(state.entrypoint, EGRESS_PROXY_ENTRYPOINT)) drift.push(`its entrypoint is ${JSON.stringify(state.entrypoint)}, not the image's ${JSON.stringify(EGRESS_PROXY_ENTRYPOINT)}`); + if (!sameList(state.cmd, EGRESS_PROXY_CMD)) drift.push(`its command is ${JSON.stringify(state.cmd)}, not the image's ${JSON.stringify(EGRESS_PROXY_CMD)}`); + if (!compareMounts) return { drift, unknown: null }; + const binds = state.mounts.filter((m) => m.type === "bind"); + const resolves = (source) => { + try { + realpath(source); + return true; + } catch { + return false; + } + }; + // UNKNOWN only where a source can be a path of the runtime's own VM: under a VM prefix Docker Desktop is known to use, + // or on a macOS or Windows host, where every Linux daemon runs in a VM (gate round 3's simple rule). On a Linux host a + // source that does not resolve is simply gone (its folder deleted, say), and a proxy mounting nothing real is stale. + // EACH BIND ON ITS OWN (round-cap re-review): an unknown one is unknown for itself, and the other expected bind and + // every extra mount are still compared; a caller refuses a removal only while an EXPECTED bind is unknown. + const vmPath = (source) => VM_PREFIXES.some((prefix) => source.startsWith(prefix)) || platform === "darwin" || platform === "win32"; + const expected = { [SQUID_CONF]: resolve(cwd, "deploy/egress-proxy.conf"), [ALLOWLIST]: resolve(cwd, "egress-allowlist.conf") }; + const same = (source, want) => { + try { + return source === want || realpath(source) === realpath(want); + } catch { + return false; + } + }; + const unknownSources = []; + for (const [destination, want] of Object.entries(expected)) { + const bind = binds.find((m) => m.destination === destination); + if (!bind) drift.push(`nothing is mounted at ${destination}, where ${want} belongs`); + else if (!resolves(bind.source) && vmPath(bind.source)) unknownSources.push(bind.source); + else if (!resolves(bind.source)) drift.push(`its ${destination} is ${bind.source}, which does not exist on this host`); + else if (!same(bind.source, want)) drift.push(`its ${destination} is ${bind.source}, not ${want}`); + } + for (const m of state.mounts) { + if (m.type === "bind" && Object.hasOwn(expected, m.destination)) continue; + if (m.type === "volume" && IMAGE_VOLUMES.includes(m.destination)) continue; + drift.push(`it has a ${m.type || "mount"} at ${m.destination} (from ${m.source || "nowhere named"}) that the shipped proxy does not`); + } + const unknown = unknownSources.length > 0 ? `${unknownSources.join(", ")} ${unknownSources.length === 1 ? "is a path" : "are paths"} this host cannot resolve (the runtime's own VM's, as Docker Desktop reports its sources), so ${unknownSources.length === 1 ? "that mount cannot" : "those mounts cannot"} be compared from here` : null; + return { drift, unknown }; +} + +/** The job and sandbox networks attached to the proxy, which a removal would cut off (issue #453, gate round 2). */ +export function jobNetworksOf(state) { + return state.networks.filter((n) => n.startsWith("pi-job-") || n.startsWith("pi-sandbox-")); +} diff --git a/worker/src/egress.mjs b/worker/src/egress.mjs index 2fa8999c..d3623063 100644 --- a/worker/src/egress.mjs +++ b/worker/src/egress.mjs @@ -180,7 +180,7 @@ export function egressEnv({ proxy = DEFAULT_EGRESS_PROXY, armed }) { * * ZERO spawns when unarmed, which is what makes a deployment without a policy pay nothing at all. * - * The gate gets `Running`, not `Health`. A healthcheck is advisory and can flap; a money gate that refuses + * The gate gets `Status` ("running"), not `Health`. A healthcheck is advisory and can flap; a money gate that refuses * on a flapping signal silently drops real work, and one that retries on it burns the second budget slot * this whole requirement exists to save. `doctor` reports health, where a human is reading. * @@ -189,13 +189,28 @@ export function egressEnv({ proxy = DEFAULT_EGRESS_PROXY, armed }) { * party, which is not a thing to do before every job on every deployment. `doctor` does it once, when * asked. What is left unproven is stated where an operator reads it rather than implied away. */ +/** The proxy states that refuse a job (gate round 3's simple rule, beside `makeEgressPreflight`). */ +// `created` among them (round-cap re-review): a container created and never started stays so until someone acts +// (measured on Docker Engine 29.8.1: a create that failed its mount stayed `created`, ExitCode 127). +export const STOPPED_PROXY_STATES = new Set(["paused", "exited", "dead", "created"]); + export function makeEgressPreflight({ proxy = DEFAULT_EGRESS_PROXY, armed = false, spawnFn = spawn, bin = "docker" } = {}) { // Issue #354: `bin` is the venue's CLI, for both probes, so the proxy is looked for where the job will run. return async function egressPreflight() { if (!armed) return { ok: true }; - const probe = await runDocker(spawnFn, ["inspect", "--format={{.State.Running}}", proxy], true, bin); + // `.State.Status` "running" (issue #453): measured on Docker Engine 29.8.1, a paused and a restarting container both + // read `.State.Running` true, and neither carries a job's traffic; the status word tells them apart. + const probe = await runDocker(spawnFn, ["inspect", "--format={{.State.Status}}", proxy], true, bin); if (probe.code === 0) { - return probe.stdout.trim() === "true" ? { ok: true, proxy } : { proxyStopped: proxy }; + const status = probe.stdout.trim(); + // THE SIMPLE RULE (issue #453, gate round 3 and the re-review): only `running` admits; `paused`, `exited`, `dead` + // and `created` are a stopped proxy, which stays so until someone acts, so the job is refused; EVERY other word + // (restarting, stopping, removing, initialized, podman's `stopped` between restarts, one no runtime prints yet) + // is an infra retry with the state in its words. BullMQ gives a job two attempts, so that is one retry, then + // the job fails: a retry buys a moment, not a wait. + if (status === "running") return { ok: true, proxy }; + if (STOPPED_PROXY_STATES.has(status)) return { proxyStopped: proxy }; + return { unavailable: proxy, state: /^[a-z]{1,20}$/.test(status) ? status : "unreported" }; } if ((await runDocker(spawnFn, ["info"], false, bin)).code === 0) return { proxyMissing: proxy }; return { unavailable: proxy }; diff --git a/worker/src/init.mjs b/worker/src/init.mjs index 7643f0a7..994d44b5 100644 --- a/worker/src/init.mjs +++ b/worker/src/init.mjs @@ -7,9 +7,11 @@ * scoped pauses, an empty packages list stages nothing, an empty subscriptions list declares no plan * prices — so a fresh deployment starts inert and is opted into feature by feature. */ -import { existsSync, copyFileSync, writeFileSync } from "node:fs"; +import { existsSync, copyFileSync, readFileSync, writeFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { join } from "node:path"; +import { parseBackendList, venuesOf } from "./backends.mjs"; +import { deploymentVenueEnv } from "./deployment-venue.mjs"; const EMPTY_TRIGGERS = `${JSON.stringify({ triggers: [] }, null, 2)}\n`; /** @@ -60,8 +62,15 @@ api.anthropic.com registry.npmjs.org `; +/** + * `deps.env` is the caller's environment (the CLI's and doctor's pass theirs), and `deps.venues` a venue set the caller + * already decided (`up` passes its own, so the two never disagree). Otherwise the venue is decided as `up` decides it + * (`deploymentVenueEnv`): this shell where it sets a key, else the deployment `.env`, and a disagreement between the two + * is said instead of any next steps. The default `env` is `{}` and not `process.env` so a test is never steered by the + * shell it runs in. + */ export function runInit(cwd = process.cwd(), deps = {}) { - const { fs = { existsSync, copyFileSync, writeFileSync }, out = (s) => process.stdout.write(s) } = deps; + const { fs = { existsSync, copyFileSync, readFileSync, writeFileSync }, out = (s) => process.stdout.write(s), env = {}, platform = process.platform } = deps; const results = []; // .env from the example. Prefer the copy in cwd (the clone's repo root); fall back to the copy @@ -90,7 +99,25 @@ export function runInit(cwd = process.cwd(), deps = {}) { for (const [verb, name, note] of results) { out(`${verb.padEnd(7)} ${name.padEnd(20)} ${note}\n`); } - out(nextSteps()); + if (deps.venues) { + out(nextSteps(deps.venues, { platform })); + return 0; + } + // Decided exactly as `up` decides it (issue #453, gate round 1): a next-steps ladder for a venue `up` would refuse to + // guess is a ladder for the wrong venue. The files above are scaffolded either way; only the steps wait. + const venue = deploymentVenueEnv({ env, fs, envPath, platform, command: "init" }); + if (venue.error) { + out(`\nNext: which venue this deployment runs is unknown, so no steps are shown: ${venue.error}. Then run \`pi-dispatch init\` again for them (it keeps every file above).\n`); + return 0; + } + for (const note of venue.notes) out(`⚠ ${note}\n`); + try { + parseBackendList(venue.env.PI_BACKENDS); + } catch (err) { + out(`\nNext: which venue this deployment runs is unknown, so no steps are shown: ${err.message}. Fix PI_BACKENDS, then run \`pi-dispatch init\` again for them (it keeps every file above).\n`); + return 0; + } + out(nextSteps(venuesOf(venue.env), { platform })); return 0; } @@ -104,7 +131,14 @@ function scaffold(fs, results, path, content, note) { } } -function nextSteps() { +/** + * The docker text is unchanged byte for byte for every set that includes `local`. A podman-only deployment (issue + * #453) gets the podman ladder instead, the steps of docs/podman.md "Setup" in the order a fresh folder needs them: + * the job image into this account's own store, the venue key and provider key in `.env` (what `up` and `service + * install` read), the stack as Quadlet units, doctor, and the worker as a user service. + */ +export function nextSteps(venues = { localUsed: true, podmanUsed: false }, { platform = "linux" } = {}) { + if (venues.podmanUsed && !venues.localUsed) return platform === "linux" ? PODMAN_NEXT_STEPS : PODMAN_OFF_LINUX; return ` Next: 1. docker pull ghcr.io/edgehero/pi-job:latest && docker tag ghcr.io/edgehero/pi-job:latest pi-job:latest @@ -118,3 +152,26 @@ Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then / (or let the panel do all of the above: /dispatch setup walks these steps with a consent per action) `; } + +// Off Linux the podman venue refuses the host (`podman-platform`: Podman machine on macOS and Windows was never +// measured), so its ladder would walk an operator into Quadlet steps that cannot work here. +const PODMAN_OFF_LINUX = ` +Next: PI_BACKENDS lists only the podman venue, which runs on Linux alone: on this host the worker refuses it +(podman-platform), so there are no podman steps to run here. Run this deployment on a Linux host, or add \`local\` to +PI_BACKENDS to run jobs on Docker here (then run \`pi-dispatch init\` again for those steps). +`; + +const PODMAN_NEXT_STEPS = ` +Next (the podman venue; run these as the worker's own account. First set the account up as docs/podman.md "Setup" +steps 1-4 say, linger included: sudo loginctl enable-linger , without which a job gets no bounds): + 1. podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest + # the prebuilt job image, in this account's own store + 2. edit .env # PI_BACKENDS=podman, and ANTHROPIC_API_KEY (or your provider's key) + 3. pi-dispatch up # Valkey and the egress proxy as Quadlet units in your user manager + 4. pi-dispatch doctor # verify Podman, Valkey, image, and key + 5. pi-dispatch service install # the worker as a user service, after those units (also installs them) + 6. pi-dispatch doctor --live # read the bounds, egress and job user back off real containers + +Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch + (or let the panel do all of the above: /dispatch setup walks these steps with a consent per action) +`; diff --git a/worker/src/live-probes.mjs b/worker/src/live-probes.mjs index 238a3fef..ad545ff1 100644 --- a/worker/src/live-probes.mjs +++ b/worker/src/live-probes.mjs @@ -556,6 +556,21 @@ export async function awaitRemoved({ step, id, now, delay, deadlineMs = LIVE_REM } } +/** + * The first line the runtime printed on a failed run, stderr first, as one printable line of at most 300 characters, + * or null. Control bytes become spaces, so nothing a runtime prints can move an operator's cursor. + */ +export function runtimeErrorLine(result) { + for (const stream of [result?.stderr, result?.stdout]) { + const line = String(stream ?? "") + .split(/\r?\n/) + .map((l) => l.replace(/[\u0000-\u001f\u007f]/g, " ").trim()) + .find((l) => l.length > 0); + if (line) return line.length > 300 ? `${line.slice(0, 300)}...` : line; + } + return null; +} + /** * The names and addresses a peer answers at on one network, from `docker inspect --format={{json .NetworkSettings.Networks}}`: * `{ targets, addresses }`, where `addresses` are the `IPAddress` and `GlobalIPv6Address` targets only, taken from @@ -716,7 +731,10 @@ export async function runLiveProbes({ const reading = await start("probe container", names.probe, liveProbeRunArgs({ image, name: names.probe, fixture, sleepSeconds: liveSleepSeconds(stepTimeoutMs), user, relabel, buildArgs })); const probeId = reading.entry.id; if (reading.result?.code !== 0 || probeId === null) { - return { ran: false, reason: "the probe container did not start, so nothing was read back", verdicts: [], notes, swept }; + // The runtime's own words, when it printed any (issue #453, gate round 1): "did not start" alone left the operator + // to reproduce the run to learn why (a missing controller, a name in use, an image the store lacks). + const said = runtimeErrorLine(reading.result); + return { ran: false, reason: `the probe container did not start${said ? ` (${bin} said: ${said})` : ""}, so nothing was read back`, verdicts: [], notes, swept }; } const expected = containerSpec(probeOptions({ image, name: names.probe, fixture, user, relabel })).mounts; diff --git a/worker/src/processor.mjs b/worker/src/processor.mjs index 08437ff2..32eec754 100644 --- a/worker/src/processor.mjs +++ b/worker/src/processor.mjs @@ -18,7 +18,10 @@ export const OBSERVATION_COMMENT = Object.freeze({ [RUNTIME_ADDS_NO_MOUNTS]: "the container runtime is not observed adding no mounts of its own to a job container", // Issue #354: the podman venue's three, in the same fixed register. No path, no controller list and no service URL: // those are the evidence, which goes to the operator's log only. - [PODMAN_BOUNDS_DELEGATED]: "the worker's rootless Podman is not observed having the cgroup controllers that apply a container's pid and memory bounds", + // Cause-neutral (issue #453): the observation misses both when a controller is not delegated and when no systemd user + // manager runs for the account (measured: podman info then still lists every controller). Which one is the evidence's + // to say, in the operator's log. + [PODMAN_BOUNDS_DELEGATED]: "the worker's rootless Podman is not observed applying a container's pid, memory and cpu bounds", [PODMAN_ADDS_NO_MOUNTS]: "the worker's rootless Podman is not observed adding no mounts of its own to a job container", [PODMAN_SERVICE_LOCAL]: "the podman CLI is not observed running containers on this host rather than through a remote service, so the job's credentials could cross a network the deployment does not own", }); @@ -608,7 +611,10 @@ export async function runJob(job, deps) { // The daemon did not answer, so this is indeterminate rather than a refusal -- the same // determinate/indeterminate split the image preflight draws one gate up, and thrown for the same // reason. Pre-reserve, so the refund below is a no-op and still honest if this gate ever moves. - throw new InfraRetry(runtimeUnavailable(resolveBackendName(job, blessedBackends[0]), "egress preflight"), { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null }); + // With a proxy STATE (issue #453, gate round 3): the daemon answered and the proxy is on its way somewhere + // (restarting, created, ...), so the words name the proxy and its state rather than blaming the runtime. + const said = typeof egress.state === "string" ? `egress proxy "${egress.unavailable}" is ${egress.state}, not running; the job is retried once, then failed` : runtimeUnavailable(resolveBackendName(job, blessedBackends[0]), "egress preflight"); + throw new InfraRetry(said, { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null }); } // REQ-RESUMABLE-SESSION's one fail-CLOSED case. Everything else in that feature fails OPEN and diff --git a/worker/src/up.mjs b/worker/src/up.mjs index eb95928d..50d425a4 100644 --- a/worker/src/up.mjs +++ b/worker/src/up.mjs @@ -37,10 +37,12 @@ import { homedir, userInfo } from "node:os"; import { dirname, join, posix, resolve, win32 } from "node:path"; import { fileURLToPath } from "node:url"; import { PODMAN_BOOT_REFUSING_CAUSES, makePodmanInfoReader, decidePodmanJobUser, podmanJobUserRefusal } from "./backend-podman.mjs"; -import { parseBackendList, venuesOf } from "./backends.mjs"; +import { venuesOf } from "./backends.mjs"; import { logsDirPath, settingsFilePath } from "./config.mjs"; import { DEFAULT_EGRESS_PROXY, egressArmed as egressArmedFn, egressProxyName } from "./egress.mjs"; import { envKeyIsBlank, envValueShown, updateEnvFile } from "./env-file.mjs"; +import { deploymentVenueEnv } from "./deployment-venue.mjs"; +import { EGRESS_PROXY_IMAGE, PROXY_STATE_FORMAT, jobNetworksOf, parseProxyState, shippedProxyDrift } from "./egress-proxy-state.mjs"; import { STACK_KEYS, applyStack, describeAction, foreignContainerRefusal, foreignContainers, lingerNote, planStack, readLinger, readStackKeys, stackComponents, unknownContainerRefusal, userBusRefusal } from "./podman-stack.mjs"; // The shipped Quadlet templates, module-relative like service.mjs's: worker/deploy in a checkout, /deploy under npm. @@ -96,6 +98,11 @@ const VALKEY_RUN_ARGS = [ // The per-job networks are NOT here: the worker creates one per job and attaches this container to it for // the life of that run. This is only the proxy and its way out. const EGRESS_NETWORK_ARGS = ["network", "create", "pi-dispatch-egress-out"]; +// A stopped CURRENT shipped proxy is started (a paused one unpaused) as it is; a STALE one is removed and run again from +// the argv below (issue #453; the reasons are at step e2). +const EGRESS_START_ARGS = ["start", "pi-dispatch-egress-proxy"]; +const EGRESS_UNPAUSE_ARGS = ["unpause", "pi-dispatch-egress-proxy"]; +const EGRESS_RM_ARGS = ["rm", "-f", "pi-dispatch-egress-proxy"]; const EGRESS_RUN_ARGS = [ "run", "-d", @@ -112,7 +119,7 @@ const EGRESS_RUN_ARGS = [ "./deploy/egress-proxy.conf:/etc/squid/squid.conf:ro,z", "-v", "./egress-allowlist.conf:/etc/pi-dispatch/allowlist.conf:ro,z", - "ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029", + EGRESS_PROXY_IMAGE, ]; /** @@ -186,7 +193,7 @@ export async function runUp(argv = [], deps = {}) { // PI_BACKENDS=podman before init has written `.env`); `.env` fills the keys the shell leaves unset; a key BOTH set // differently stops `up` (round 2, E5), because the service will run the file's value. A `.env` line touching one // of these keys that the loaders read differently stops `up` before anything runs, as `service install` refuses. - const venue = deploymentVenueEnv({ env, fs, envPath: join(cwd, ".env"), platform }); + const venue = deploymentVenueEnv({ env, fs, envPath: join(cwd, ".env"), platform, command: "up" }); if (venue.error) { out(`✗ ${venue.error}\n\nup: cannot tell which venue this deployment runs: fix the above, then re-run \`pi-dispatch up\`.\n`); return 1; @@ -297,7 +304,8 @@ export async function runUp(argv = [], deps = {}) { // nothing the operator wrote can be lost here. out("\ninit (never overwrites — an existing file is reported and kept):\n"); runInitFn ??= (await import("./init.mjs")).runInit; - runInitFn(cwd, { out }); + // The venues THIS pass decided (issue #453), so init's next steps never name a runtime up just did not drive. + runInitFn(cwd, { out, venues }); summary.push(["init", "ran — existing files were kept untouched, missing ones scaffolded"]); // (e) WEBHOOK_SECRET, only into a .env that exists (init just scaffolded one unless the operator @@ -466,26 +474,110 @@ export async function runUp(argv = [], deps = {}) { // shipped name whatever that key said, so it could report a proxy present that no job attaches to, or start one // beside the one the worker actually uses. It now asks about the name the worker attaches by and starts nothing: // that container is the operator's, and the shipped command below would not produce it. - if ((await runCmd(spawn, "docker", ["inspect", "--format={{.State.Running}}", proxyName])) === 0) { + // RUNNING, read off stdout alone, exactly as the podman path below (issue #453): `docker inspect` exits 0 for an + // exited container too, so the exit code called a stopped proxy "present" while every job was refused pre-spend. + // `.State.Status`, not `.State.Running` (issue #453): the status word is what names a paused or restarting container. + const inspect = await runCmdQuery(spawn, "docker", ["inspect", "--format={{.State.Status}}", proxyName]); + const status = inspect.code === 0 ? inspect.stdout.trim() : null; + if (status === "running") { out(`\n✓ Egress proxy present (${proxyName}, named by PI_EGRESS_PROXY)\n`); summary.push(["egress", `${proxyName} (PI_EGRESS_PROXY) present, left untouched`]); + } else if (inspect.code === 0) { + // It exists and is not running. Still the operator's container: reported, never started, for the reason above. + out(`\n✗ PI_EGRESS_PROXY names ${proxyName}, and that container exists but is not running (${status || "no status"}). up starts only the shipped ${DEFAULT_EGRESS_PROXY}, so start ${proxyName} yourself\n`); + summary.push(["egress", `${proxyName} (PI_EGRESS_PROXY) exists but is not running; every job is refused pre-spend until it runs`]); } else { out(`\n✗ PI_EGRESS_PROXY names ${proxyName}, and docker has no container of that name. up starts only the shipped ${DEFAULT_EGRESS_PROXY}, so start ${proxyName} yourself\n`); summary.push(["egress", `${proxyName} (PI_EGRESS_PROXY) not found; every job is refused pre-spend until it runs`]); } } else if (dockerUsed && egressArmedFn(venueEnv)) { - if ((await runCmd(spawn, "docker", ["inspect", "--format={{.State.Running}}", "pi-dispatch-egress-proxy"])) === 0) { + // RUNNING and CURRENT, from one inspect (issue #453; `egress-proxy-state.mjs` says why each). Running is + // `.State.Status` "running". Current is the pinned image with its own entrypoint and command and THIS folder's two + // files mounted and nothing else: a container of the shipped name that differs is not this deployment's proxy, + // whether it is running or stopped. Mounts whose sources do not resolve on this host are UNKNOWN, never stale. + const inspect = await runCmdQuery(spawn, "docker", ["inspect", PROXY_STATE_FORMAT, DEFAULT_EGRESS_PROXY]); + const state = inspect.code === 0 ? parseProxyState(inspect.stdout) : null; + const judged = state ? shippedProxyDrift(state, { cwd, platform, realpath: (p) => (typeof fs.realpathSync === "function" ? fs.realpathSync(p) : p) }) : { drift: [], unknown: null }; + const drift = judged.drift; + // The files a start or a recreate mounts. Both, not the allowlist alone: a missing egress-proxy.conf is bind-mounted + // as a directory the runtime creates in its place (measured, Docker Engine 29.8.1: the host path became a root-owned + // directory, and the create failed "not a directory: Are you trying to mount a directory onto a file", exit 127). + const missingFile = ["egress-allowlist.conf", "deploy/egress-proxy.conf"].find((f) => !fs.existsSync(join(cwd, f))); + if (judged.unknown) out(`\n⚠ could not compare ${DEFAULT_EGRESS_PROXY}'s mounts on this host: ${judged.unknown}. Everything else about it is still judged, and up replaces nothing while one of its own two mounts is unknown\n`); + if (inspect.code === 0 && !state) { + // It exists and its state could not be read: never started, never replaced, and never a `docker run` on a name + // that is taken. + out(`\n✗ ${DEFAULT_EGRESS_PROXY} exists, but its state could not be read from \`docker inspect\`, so up leaves it as it is; \`docker inspect ${DEFAULT_EGRESS_PROXY}\` shows it\n`); + summary.push(["egress", "proxy exists, state unreadable: left as it is"]); + } else if (state?.status === "running" && drift.length === 0) { out("\n✓ Egress proxy already present (pi-dispatch-egress-proxy)\n"); - summary.push(["egress", "proxy already present — left untouched"]); - } else if (!fs.existsSync(join(cwd, "egress-allowlist.conf"))) { - out("\n✗ the egress policy is on but egress-allowlist.conf is not here — not starting a proxy with no allowlist\n"); - summary.push(["egress", "skipped — no egress-allowlist.conf in this folder; run `pi-dispatch init` here, then `up` again"]); + summary.push(["egress", judged.unknown ? "proxy already present, left untouched (its mounts could not be compared on this host)" : "proxy already present, left untouched"]); + } else if (state && drift.length > 0) { + // STALE, running or not: never started as it is. Its policy is not this deployment's, so the offer is to replace + // it with the shipped one, shown line for line. A running one may be carrying jobs: its job and sandbox networks + // are named, since removing it cuts each of them off mid-run. + out(`\n✗ ${DEFAULT_EGRESS_PROXY} exists (${state.status}) but is not this deployment's proxy: ${drift.join("; ")}\n`); + const attached = jobNetworksOf(state); + // Never removed while one of its two own mounts is unknown (round-cap re-review): the other findings are said. + const unjudged = Boolean(judged.unknown); + if (yes && attached.length > 0 && state.status === "running" && !judged.unknown && !missingFile) out("--yes does not cover replacing a proxy that jobs are using: answer below, or stop the worker and re-run\n"); + if (unjudged) { + out(`not replaced: one of its own mounts could not be compared on this host (above), so up leaves it as it is; \`docker ${EGRESS_RM_ARGS.join(" ")}\` and \`pi-dispatch up\` replace it if you have checked it\n`); + summary.push(["egress", "stale proxy left as it is: one of its mounts could not be compared on this host"]); + } else if (missingFile) { + out(`✗ ${missingFile} is not here, so up cannot recreate it from this folder; run \`pi-dispatch init\` here (or \`up\` from the deployment folder), then \`up\` again\n`); + summary.push(["egress", `stale proxy left as it is: no ${missingFile} in this folder to recreate it from; its policy is not this deployment's`]); + } else if ( + await consent( + `up would replace it with the shipped proxy (the same semantics as deploy/docker-compose.yml --profile egress)${attached.length > 0 ? `. It is attached to ${attached.join(", ")}: removing it cuts those jobs off from their egress mid-run, so stop the worker first (and let running jobs finish)` : ""}:`, + [`docker ${EGRESS_RM_ARGS.join(" ")}`, `docker ${EGRESS_NETWORK_ARGS.join(" ")} (only if it does not exist yet)`, `docker ${quoteArgs(EGRESS_RUN_ARGS)}`], + // `--yes` does not cover cutting live jobs off (gate round 3): with a job network attached to a RUNNING + // proxy the lines are printed and a person must answer, so an unattended `up --yes` never does it. + { yes: yes && !(attached.length > 0 && state.status === "running"), out, prompt }, + ) + ) { + await runStreamed(spawn, "docker", EGRESS_RM_ARGS, out); + await ensureEgressNetwork(spawn, out); + if ((await runStreamed(spawn, "docker", EGRESS_RUN_ARGS, out)) !== 0) { + out("✗ could not start the egress proxy; continuing, doctor below will re-check it\n"); + summary.push(["egress", "recreate failed: every job refuses pre-spend until it is up (costs no budget, runs nothing)"]); + } else { + summary.push(["egress", "replaced the stale pi-dispatch-egress-proxy with the shipped one"]); + } + } else { + out(`skipped: until it is replaced, jobs use a proxy whose policy is not this deployment's (\`docker ${EGRESS_RM_ARGS.join(" ")}\`, then \`pi-dispatch up\`)\n`); + summary.push(["egress", "stale proxy left as it is (declined): its policy is not this deployment's"]); + } + } else if (missingFile) { + out(`\n✗ the egress policy is on but ${missingFile} is not here, so no proxy is started without it\n`); + summary.push(["egress", `skipped: no ${missingFile} in this folder; run \`pi-dispatch init\` here (or \`up\` from the deployment folder), then \`up\` again`]); + } else if (state?.status === "restarting") { + // A crash loop: the restart policy is already starting it, and its squid keeps exiting. Nothing to start. + out(`\n✗ ${DEFAULT_EGRESS_PROXY} is restarting: its squid keeps exiting and its restart policy keeps bringing it back. \`docker logs ${DEFAULT_EGRESS_PROXY}\` says why; meanwhile each job is retried once, then failed\n`); + summary.push(["egress", "proxy crash-looping: see `docker logs pi-dispatch-egress-proxy`"]); + } else if (state) { + // Current, and stopped or paused. `docker run --name` would fail on the name conflict, so the SAME container is + // offered back: `unpause` for a paused one, else `start`, which is what `docker compose --profile egress up -d` + // does for a stopped container whose configuration is unchanged. It destroys nothing, and a compose-created + // container stays compose's. + const args = state.status === "paused" ? EGRESS_UNPAUSE_ARGS : EGRESS_START_ARGS; + if (await consent(`The egress policy is on (PI_EGRESS=0 opts out) and the allowlist proxy exists but is ${state.status}. up would ${args[0]} that same container (as deploy/docker-compose.yml --profile egress up -d does):`, [`docker ${args.join(" ")}`], { yes, out, prompt })) { + if ((await runStreamed(spawn, "docker", args, out)) !== 0) { + out(`✗ could not ${args[0]} the egress proxy; \`docker ${EGRESS_RM_ARGS.join(" ")}\` and re-run \`pi-dispatch up\` recreates it. Continuing; doctor below will re-check it\n`); + summary.push(["egress", `${args[0]} of the ${state.status} proxy failed; every job refuses pre-spend until it is up (costs no budget, runs nothing)`]); + } else { + summary.push(["egress", `${args[0] === "unpause" ? "unpaused" : "started"} the ${state.status} pi-dispatch-egress-proxy`]); + } + } else { + out(`skipped: ${args[0]} it later with \`docker ${args.join(" ")}\`\n`); + summary.push(["egress", "skipped (declined): every job is refused pre-spend until the proxy is up (PI_EGRESS=0 opts out)"]); + } } else if ( - await consent("The egress policy is on (PI_EGRESS=0 opts out) but the allowlist proxy is not running. up would start it (same semantics as deploy/docker-compose.yml --profile egress):", [`docker ${EGRESS_NETWORK_ARGS.join(" ")}`, `docker ${quoteArgs(EGRESS_RUN_ARGS)}`], { yes, out, prompt }) + await consent("The egress policy is on (PI_EGRESS=0 opts out) but the allowlist proxy is not running. up would start it (same semantics as deploy/docker-compose.yml --profile egress):", [`docker ${EGRESS_NETWORK_ARGS.join(" ")} (only if it does not exist yet)`, `docker ${quoteArgs(EGRESS_RUN_ARGS)}`], { yes, out, prompt }) ) { // The network may already exist from a previous run; that is not a failure, so its code is not // checked. The proxy is what matters and it is checked. - await runStreamed(spawn, "docker", EGRESS_NETWORK_ARGS, out); + await ensureEgressNetwork(spawn, out); if ((await runStreamed(spawn, "docker", EGRESS_RUN_ARGS, out)) !== 0) { out("✗ could not start the egress proxy — continuing; doctor below will re-check it\n"); summary.push(["egress", "start failed — every job refuses pre-spend until it is up (costs no budget, runs nothing)"]); @@ -560,81 +652,6 @@ Next: return doctorCode; } -/** - * The venue keys this pass decides with (D2): this shell's value where it sets one, else the deployment `.env`'s, read - * with `readStackKeys` exactly as `service install` reads them. Returns `{ env, fromFile, disagreements }` or - * `{ error }`. `env` is the whole environment with the file's keys filled in; `fromFile` is only what the file - * supplied, for doctor's layering; `disagreements` names each key both set differently. - */ -function deploymentVenueEnv({ env, fs, envPath, platform }) { - let keys = {}; - const notes = []; - // The loader of the file on THIS platform, as service.mjs reads it (round 2, E6): systemd's EnvironmentFile= on - // Linux, the sh wrapper on macOS, the cmd wrapper on Windows. Off Linux the venue refuses the host anyway, so a - // line that platform's loader reads differently is noted and never stops `up`. - const linux = platform === "linux"; - const loader = linux ? "systemd" : platform === "darwin" ? "shell" : "cmd"; - if (fs.existsSync(envPath)) { - let text = null; - try { - text = String(fs.readFileSync(envPath, "utf8")); - } catch (err) { - // Said rather than silent (round 2 nit): the venue then comes from this shell alone. - notes.push(`${envPath} could not be read (${err?.message}), so PI_BACKENDS, PI_EGRESS and PI_EGRESS_PROXY come from this shell alone`); - } - if (text !== null) { - const read = readStackKeys(text, { loader, path: envPath }); - if (read.error && linux) return { error: read.error }; - if (read.error) notes.push(`${read.error}; off Linux the podman venue refuses this host anyway, so this pass reads the venue from this shell`); - else keys = read.keys; - } - } - const merged = { ...env }; - const fromFile = {}; - const conflicts = []; - for (const key of STACK_KEYS) { - if (!Object.hasOwn(keys, key)) continue; - if (typeof env[key] === "string") { - // Compared as the worker READS them (round 3, D4), not as strings: ` podman` and `podman,podman` are the list - // `podman`, and refusing them sent an operator to reconcile two values that already agree. - if (venueKeyMeaning(key, env[key]) !== venueKeyMeaning(key, keys[key])) conflicts.push(`${key} is ${quotedShown(env[key])} in this shell and ${quotedShown(keys[key])} in ${envPath}`); - continue; - } - merged[key] = keys[key]; - fromFile[key] = keys[key]; - } - // A KNOWN disagreement is a refusal (round 2, E5), not a warning: this pass would stand up one venue's stack and the - // service installed next would run the other. The file is what the service runs, so it is the one to change unless - // the shell's value was a one-off. - if (conflicts.length > 0) { - return { error: `${conflicts.join("; ")}. up would drive the shell's venue while the service runs the file's. Make them agree: change ${envPath} (what the service reads), or unset the key in this shell` }; - } - return { env: merged, fromFile, notes }; -} - -/** - * What a venue key MEANS to the worker, for comparing two spellings of it: the parsed list for PI_BACKENDS, on or off - * for PI_EGRESS, the resolved name for PI_EGRESS_PROXY exactly as egressProxyName resolves it (empty is the default - * name; NOT trimmed, because nothing that reads the name trims it, so " x" and "x" are different proxies to the worker - * and to podman). A value the worker would refuse keeps its raw spelling, so two different unreadable values still - * disagree and the doctor below names them. - */ -function venueKeyMeaning(key, value) { - try { - if (key === "PI_BACKENDS") return `list:${parseBackendList(value).join(",")}`; - if (key === "PI_EGRESS") return `egress:${egressArmedFn({ PI_EGRESS: value }) ? "on" : "off"}`; - return `proxy:${egressProxyName({ PI_EGRESS_PROXY: value })}`; - } catch { - return `raw:${value}`; - } -} - -/** A value shown in quotes, so a leading space or an empty value is visible; control characters escaped as ever. */ -function quotedShown(value) { - const shown = envValueShown(value); - return shown.startsWith('"') ? shown : `"${shown}"`; -} - /** * The podman venue's gate: the same `podman info` read and the same ordered job-user rule the worker boots with * (`decidePodmanJobUser`), so `up` cannot pass a host the worker then refuses, or refuse one it would run. Printed in @@ -719,8 +736,10 @@ async function podmanStackStep({ env, venues, spawn, out, yes, prompt, summary, // (measured), so the exit code alone called a stopped hand-started proxy "present" and offered nothing, which is // exactly the upgrade this step exists for. A stopped one is offered the unit, under the foreign-container rule. // stdout ALONE (round 2, E2): podman's stderr warnings on an ordinary account would otherwise make `true` unequal. - const inspect = await runCmdQuery(spawn, "podman", ["inspect", "--format={{.State.Running}}", DEFAULT_EGRESS_PROXY]); - if (inspect.code === 0 && inspect.stdout.trim() === "true") { + // `.State.Status` (issue #453): Podman reports a paused container `paused` and one between restarts `stopped` + // (measured on 4.9.3 and 5.8.1), and only `running` carries a job's traffic. + const inspect = await runCmdQuery(spawn, "podman", ["inspect", "--format={{.State.Status}}", DEFAULT_EGRESS_PROXY]); + if (inspect.code === 0 && inspect.stdout.trim() === "running") { out(`\n✓ Egress proxy already present under this account's Podman (${DEFAULT_EGRESS_PROXY})\n`); summary.push(["egress (podman)", "proxy already present, left untouched"]); components.proxy = false; @@ -871,6 +890,16 @@ function runCmdQuery(spawn, cmd, args) { }); } +/** + * `docker network create pi-dispatch-egress-out` only where it does not exist (exec round 3): a recreate otherwise printed + * the daemon's "network name ... already used" every time. A quiet inspect first; its answer is never a failure, since the + * run that follows is what is checked. + */ +async function ensureEgressNetwork(spawn, out) { + if ((await runCmdQuery(spawn, "docker", ["network", "inspect", EGRESS_NETWORK_ARGS[2]])).code === 0) return; + await runStreamed(spawn, "docker", EGRESS_NETWORK_ARGS, out); +} + /** Like runCmd but with stdout+stderr captured, for read-only lookups (docker ps). */ function runCmdCapture(spawn, cmd, args) { return new Promise((resolve) => { diff --git a/worker/test/backend-podman.test.mjs b/worker/test/backend-podman.test.mjs index 63be30f1..b6f69af4 100644 --- a/worker/test/backend-podman.test.mjs +++ b/worker/test/backend-podman.test.mjs @@ -26,6 +26,7 @@ function infoBody(over = {}) { { host: { cgroupVersion: "v2", + cgroupManager: "systemd", cgroupControllers: ["cpuset", "cpu", "io", "memory", "pids"], serviceIsRemote: false, remoteSocket: { path: "/run/user/1234/podman/podman.sock", exists: false }, @@ -41,7 +42,7 @@ function infoBody(over = {}) { } // An escape byte built at run time: a literal one in source survives a copy and paste and then does not. const ESC = String.fromCharCode(27); -const INFO = () => ({ rootless: true, serviceIsRemote: false, selinux: true, cgroupVersion: "v2", controllers: ["cpuset", "cpu", "io", "memory", "pids"], version: "5.8.1", graphRoot: "/home/op/.local/share/containers/storage" }); +const INFO = () => ({ rootless: true, serviceIsRemote: false, selinux: true, cgroupVersion: "v2", cgroupManager: "systemd", controllers: ["cpuset", "cpu", "io", "memory", "pids"], version: "5.8.1", graphRoot: "/home/op/.local/share/containers/storage" }); const answered = (over = {}) => ({ answered: true, info: { ...INFO(), ...over } }); /** @@ -49,17 +50,22 @@ const answered = (over = {}) => ({ answered: true, info: { ...INFO(), ...over } * every call on it throws. Everything else is ENOENT, which is what an unconfigured host looks like. */ function fakeFs({ files = {}, dirs = {}, errors = {} } = {}) { + // The account's running user manager with the controllers delegated, as on the measured host, unless a test says + // otherwise: `[USER_MANAGER]: null` is an account with none running (issue #453). Added to the caller's own object, + // since a test may edit `files` after this and expect the change read. + if (!Object.hasOwn(files, USER_MANAGER)) files[USER_MANAGER] = DELEGATED; + const has = (path) => Object.hasOwn(files, path) && files[path] !== null; const fail = (path) => { const code = errors[path] ?? "ENOENT"; throw Object.assign(new Error(`${code}: ${path}`), { code }); }; return { statSync(path) { - if (errors[path] || !Object.hasOwn(files, path)) fail(path); + if (errors[path] || !has(path)) fail(path); return { size: Buffer.byteLength(files[path]) }; }, readFileSync(path) { - if (errors[path] || !Object.hasOwn(files, path)) fail(path); + if (errors[path] || !has(path)) fail(path); return files[path]; }, readdirSync(path) { @@ -69,6 +75,9 @@ function fakeFs({ files = {}, dirs = {}, errors = {} } = {}) { }; } const HOME = "/home/op"; +// user@1234.service's cgroup, whose controllers file is the fact podmanBoundsDelegated reads (issue #453). +const USER_MANAGER = "/sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/cgroup.controllers"; +const DELEGATED = "cpuset cpu io memory pids\n"; const USER_MOUNTS = `${HOME}/.config/containers/mounts.conf`; const clean = () => fakeFs({ files: { [USER_MOUNTS]: "" } }); const observe = (read, fs = clean(), over = {}) => mod.observePodman({ read, fs, home: HOME, env: {}, euid: 1234, ...over }); @@ -82,9 +91,9 @@ test("parsePodmanInfo reads the measured shape and nothing else", { skip }, () = } // Every field null when absent or the wrong type: a missing `rootless` is never read as rootless, nor a missing // `serviceIsRemote` as local. - assert.deepEqual(mod.parsePodmanInfo(JSON.stringify({ host: {} })), { rootless: null, serviceIsRemote: null, selinux: null, cgroupVersion: null, controllers: null, version: null, graphRoot: null }); - const odd = mod.parsePodmanInfo(infoBody({ host: { serviceIsRemote: "false", security: { rootless: 1, selinuxEnabled: "true" }, cgroupVersion: `v2${ESC}[2J`, cgroupControllers: ["pids", 7, "memory\n", "cpu"] }, version: { Version: `5.8.1${ESC}]0;x` } })); - assert.deepEqual(odd, { rootless: null, serviceIsRemote: null, selinux: null, cgroupVersion: null, controllers: ["pids", "cpu"], version: null, graphRoot: "/home/op/.local/share/containers/storage" }); + assert.deepEqual(mod.parsePodmanInfo(JSON.stringify({ host: {} })), { rootless: null, serviceIsRemote: null, selinux: null, cgroupVersion: null, cgroupManager: null, controllers: null, version: null, graphRoot: null }); + const odd = mod.parsePodmanInfo(infoBody({ host: { serviceIsRemote: "false", security: { rootless: 1, selinuxEnabled: "true" }, cgroupVersion: `v2${ESC}[2J`, cgroupManager: `systemd${ESC}[2J`, cgroupControllers: ["pids", 7, "memory\n", "cpu"] }, version: { Version: `5.8.1${ESC}]0;x` } })); + assert.deepEqual(odd, { rootless: null, serviceIsRemote: null, selinux: null, cgroupVersion: null, cgroupManager: null, controllers: ["pids", "cpu"], version: null, graphRoot: "/home/op/.local/share/containers/storage" }); // The store is a path or nothing: a relative one, a non-string or one carrying a control byte is no fact. for (const graphRoot of ["relative/path", 7, `/home/op${ESC}[2J`, ""]) assert.equal(mod.parsePodmanInfo(JSON.stringify({ host: {}, store: { graphRoot } })).graphRoot, null, JSON.stringify(graphRoot)); }); @@ -184,15 +193,22 @@ test("observePodman credits a clean rootless host, and the table's words then ho test("podmanBoundsDelegated needs rootless cgroup v2 with pids, memory and cpu delegated, and no cgroups key", { skip }, () => { const bounds = (read, fs) => observe(read, fs).observations[PODMAN_BOUNDS_DELEGATED]; + const manager = (text) => fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: text } }); for (const missing of ["pids", "memory", "cpu"]) { - const o = observe(answered({ controllers: INFO().controllers.filter((c) => c !== missing) })); + const o = observe(answered(), manager(`${["cpuset", "cpu", "io", "memory", "pids"].filter((c) => c !== missing).join(" ")}\n`)); assert.equal(o.observations[PODMAN_BOUNDS_DELEGATED], false, missing); - assert.match(o.evidence[PODMAN_BOUNDS_DELEGATED], new RegExp(`the ${missing} cgroup controller is not delegated`)); + assert.equal(o.evidence[PODMAN_BOUNDS_DELEGATED], `the ${missing} cgroup controller is not delegated to this account's systemd user manager (user@1234.service), so a job cannot have that bound: measured for cpu, Podman 5.8.1 refuses to start a container carrying --cpus (exit 126, "controller \`cpu\` is not available")`); } - assert.equal(bounds(answered({ controllers: ["io"] })), false); - assert.equal(bounds(answered({ controllers: null })), false); + assert.equal(bounds(answered(), manager("io\n")), false); + assert.equal(bounds(answered(), manager("")), false); assert.equal(bounds(answered({ cgroupVersion: "v1" })), false); assert.equal(bounds(answered({ cgroupVersion: null })), false); + assert.equal(observe(answered(), clean(), { euid: null }).observations[PODMAN_BOUNDS_DELEGATED], false, "no uid, no user manager to read"); + // A user manager's cgroup that could not be read for a moment is a retry, and one the account cannot read is refused. + const busy = observe(answered(), fakeFs({ files: { [USER_MOUNTS]: "" }, errors: { [USER_MANAGER]: "EMFILE" } })); + assert.equal(busy.observations[PODMAN_BOUNDS_DELEGATED], null); + assert.equal(busy.reasons[PODMAN_BOUNDS_DELEGATED], "file-unread"); + assert.equal(observe(answered(), fakeFs({ files: { [USER_MOUNTS]: "" }, errors: { [USER_MANAGER]: "EACCES" } })).observations[PODMAN_BOUNDS_DELEGATED], false); assert.equal(bounds(answered({ rootless: false })), false, "a rootful controller list is the whole tree, not a delegation"); // A containers.conf that takes containers out of their cgroup: measured, the bounds then read back `max`. const conf = (text, path = "/etc/containers/containers.conf") => fakeFs({ files: { [USER_MOUNTS]: "", [path]: text } }); @@ -512,14 +528,84 @@ test("the podman reaper keeps the tri-state and enumerates podman's store", { sk assert.equal((await mod.makePodmanReaper({ log: () => {}, spawnFn: broken.spawnFn })()).reaped, false); }); +// Issue #453, measured on Fedora 44 with rootless Podman 5.8.1 (round-446 M0-e with buildPodmanRunArgs' own argv, and +// the gate-456 adversary's L rows), raw files named per row: whether a job's bounds were APPLIED, against what +// `podman info` said (the cgroup manager and its caller-cgroup controller list), whether the account's user@.service +// cgroup existed with the controllers, and the caller's own cgroup (/proc/self/cgroup). The uids of the measured +// accounts (1236, 1238) are written as this suite's 1234; the other users' scopes are kept as measured. `credited` is +// what the observation must answer: never credit where the bounds were not applied, and one measured-applied row (E3c) +// deliberately not credited, since nothing this process can read tells it apart from L1-L3. +const ALL5 = ["cpuset", "cpu", "io", "memory", "pids"]; +const OWN_UNIT = "0::/user.slice/user-1234.slice/user@1234.service/app.slice/run-p412811-i412812.service\n"; +const ROB_SESSION = "0::/user.slice/user-501.slice/session-5.scope\n"; +const M0E_ROWS = [ + // [case, podman info cgroupManager, podman info cgroupControllers, user@1234.service controllers (null: not running), /proc/self/cgroup, applied, credited] + ["E1: linger off, sudo -iu, no user manager: systemd fell back to cgroupfs (raw-M0e-E1-sudo-nolinger.txt, raw-M0e-E1r-E3d-nolinger.txt)", "cgroupfs", ALL5, null, ROB_SESSION, false, false], + ["E1b: linger off, plain ssh, whose pam_systemd session started the manager (raw-M0e-E1b-ssh-nolinger.txt)", "systemd", ALL5, DELEGATED, "0::/user.slice/user-1234.slice/session-114.scope\n", true, true], + ["E2a: linger on, a user unit without Delegate=, whose own cgroup lists memory pids, cpu applied anyway (raw-M0e-E2a-userunit-linger.txt, raw-M0e-controllers.txt)", "systemd", ["memory", "pids"], DELEGATED, OWN_UNIT, true, true], + ["E2c: linger on, sudo -iu with no environment (raw-M0e-E2c-sudo-linger.txt)", "systemd", ALL5, DELEGATED, ROB_SESSION, true, true], + ["E2d: linger on, sudo with XDG_RUNTIME_DIR only (raw-M0e-E2d-sudo-xdg-linger.txt)", "systemd", ALL5, DELEGATED, ROB_SESSION, true, true], + ["E3a: explicit cgroupfs, a user unit with Delegate=yes (raw-M0e-E3a-cgroupfs-delegate.txt)", "cgroupfs", ALL5, DELEGATED, "0::/user.slice/user-1234.slice/user@1234.service/app.slice/run-p421802-i421803.service\n", true, true], + ["E3b: explicit cgroupfs, a user unit without Delegate= (raw-M0e-E3b-cgroupfs-nodelegate.txt)", "cgroupfs", ["memory", "pids"], DELEGATED, "0::/user.slice/user-1234.slice/user@1234.service/app.slice/run-p422479-i422480.service\n", true, true], + ["E3c: explicit cgroupfs, linger on, sudo -iu from another user's session, bus reachable: applied, NOT credited (raw-M0e-E3c-cgroupfs-sudo.txt)", "cgroupfs", ALL5, DELEGATED, ROB_SESSION, true, false], + ["E3d: explicit cgroupfs, linger off, sudo -iu (raw-M0e-E1r-E3d-nolinger.txt)", "cgroupfs", ALL5, null, ROB_SESSION, false, false], + ["the same user unit with a Delegate=yes sibling running, its own list flipped to all five (raw-M0e-controllers-sibling.txt)", "systemd", ALL5, DELEGATED, OWN_UNIT, true, true], + ["L1: linger on, user manager running, no user bus socket, sudo -iu: systemd fell back to cgroupfs (gate456-adv raw-L1-L2.txt)", "cgroupfs", ALL5, DELEGATED, ROB_SESSION, false, false], + ["L2: linger on, DBUS_SESSION_BUS_ADDRESS pointing at no bus: systemd fell back to cgroupfs (gate456-adv raw-L1-L2.txt)", "cgroupfs", ALL5, DELEGATED, ROB_SESSION, false, false], + ["L3: a system unit with User= and no user bus: cgroupfs, the container in the unit's root-owned cgroup, memory.max max (gate456-adv raw-L1-L2.txt)", "cgroupfs", ALL5, DELEGATED, "0::/system.slice/adv4-l3.service\n", false, false], +]; +const NOT_APPLIED = { + "no-user-manager": "no systemd user manager is running for this account (/sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/cgroup.controllers does not exist), so Podman leaves a job's pids, memory and cpu bounds unapplied, whichever cgroup manager it uses: with linger off the manager runs only while the account has a login session, and a `sudo -iu` shell starts none (measured). On a host without systemd managing cgroups under user.slice this path never exists, and the venue gives no credit there", + "user-manager-unreachable": "Podman is using the cgroupfs cgroup manager, not systemd, and this worker is not running inside the account's user manager (user@1234.service), so whether a job's pids, memory and cpu bounds are applied is not observed from here. Measured: where a configured systemd manager fell back to cgroupfs because Podman could not reach the user manager over D-Bus (no user bus socket, a DBUS_SESSION_BUS_ADDRESS pointing nowhere), the container landed in the worker's own cgroup unbounded; an explicit cgroupfs with the bus reachable had its bounds applied, and nothing read here tells the two apart", +}; + +test("podmanBoundsDelegated credits only where the bounds were measured applied: the user manager, and Podman reaching it or the worker inside it (#453)", { skip }, () => { + for (const [label, cgroupManager, controllers, userManager, self, applied, credited] of M0E_ROWS) { + assert.ok(!credited || applied, `${label}: the table itself never credits an unapplied row`); + const o = observe(answered({ cgroupManager, controllers }), fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: userManager, "/proc/self/cgroup": self } })); + assert.equal(o.observations[PODMAN_BOUNDS_DELEGATED], credited, label); + assert.deepEqual(o.reasons, {}, `${label}: determinate, never a retry`); + const cause = credited ? undefined : userManager === null ? "no-user-manager" : "user-manager-unreachable"; + assert.equal(o.boundsCause, cause, label); + assert.equal( + o.evidence[PODMAN_BOUNDS_DELEGATED], + credited + ? `rootless Podman on cgroup v2, with this account's systemd user manager (user@1234.service) running and the pids, memory and cpu controllers delegated to it, ${cgroupManager === "systemd" ? "Podman using the systemd cgroup manager" : "this worker running inside that manager"}, and no containers.conf sets cgroups` + : NOT_APPLIED[cause], + label, + ); + assert.deepEqual(o.boundsControllers, credited ? ALL5 : undefined, `${label}: the manager's list, not podman info's`); + } + assert.equal(mod.podmanUserManagerControllersPath(1234), USER_MANAGER); + // A podman info that names no manager needs the worker inside the manager, as cgroupfs does. + const unnamed = (self) => observe(answered({ cgroupManager: null }), fakeFs({ files: { [USER_MOUNTS]: "", "/proc/self/cgroup": self } })).observations[PODMAN_BOUNDS_DELEGATED]; + assert.equal(unnamed(ROB_SESSION), false); + assert.equal(unnamed(OWN_UNIT), true); + // A /proc/self/cgroup read that failed for a moment is a retry, never a determinate refusal. + const busy = observe(answered({ cgroupManager: "cgroupfs" }), fakeFs({ files: { [USER_MOUNTS]: "" }, errors: { "/proc/self/cgroup": "EMFILE" } })); + assert.equal(busy.observations[PODMAN_BOUNDS_DELEGATED], null); + assert.equal(busy.reasons[PODMAN_BOUNDS_DELEGATED], "file-unread"); + // Another account's user manager does not count: the prefix is this uid's own. + assert.equal(observe(answered({ cgroupManager: "cgroupfs" }), fakeFs({ files: { [USER_MOUNTS]: "", "/proc/self/cgroup": "0::/user.slice/user-12345.slice/user@12345.service/app.slice/x.service\n" } })).observations[PODMAN_BOUNDS_DELEGATED], false); +}); + test("the podman observationPreflight judges the floor on podman's observations, as local's does (#354)", { skip }, async () => { const floor = { isolation: ENFORCED, credentialTransit: ENFORCED }; assert.deepEqual(Object.keys(await bundle({ backendFloor: floor }).observationPreflight(JOB)), ["ok", "podman"]); - const refused = await bundle({ backendFloor: floor, readInfo: async () => answered({ controllers: ["io"] }) }).observationPreflight(JOB); + const refused = await bundle({ backendFloor: floor, fs: fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: "io\n" } }) }).observationPreflight(JOB); assert.equal(refused.refused, true); assert.deepEqual(refused.observations, [PODMAN_BOUNDS_DELEGATED]); assert.match(refused.message, /podman: isolation=enforced holds only while this worker's rootless Podman runs on cgroup v2/); - assert.match(refused.message, /Delegate the cpu, memory and pids controllers/); + assert.match(refused.message, /delegate the cpu, memory and pids controllers/); + // Issue #453: an account with no systemd user manager running, `podman info` still listing every controller: refused + // per job under the floor, the missing manager named, and the fix keeping one running first. + const noManager = fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: null } }); + const noSession = await bundle({ backendFloor: floor, fs: noManager }).observationPreflight(JOB); + assert.equal(noSession.refused, true); + assert.deepEqual(noSession.observations, [PODMAN_BOUNDS_DELEGATED]); + assert.match(noSession.message, /no systemd user manager is running for this account \(\/sys\/fs\/cgroup\/user\.slice\/user-1234\.slice\/user@1234\.service\/cgroup\.controllers does not exist\)/); + assert.match(noSession.message, /Turn on linger for the worker account \(`loginctl enable-linger `, which keeps its systemd user manager running with no one logged in\)/); + assert.deepEqual(Object.keys(await bundle({ fs: noManager }).observationPreflight(JOB)), ["ok", "podman"], "without the floor a job still runs"); assert.deepEqual(await bundle({ backendFloor: floor, readInfo: async () => ({ answered: false, reason: "timeout", transient: true }) }).observationPreflight(JOB), { unavailable: true, reason: "timeout" }); // A refused identity is passed through to jobUserPreflight, which names the one fix, rather than refused here with a // floor fix that is not it. Every unmappable row, each of which misses a floored observation too. @@ -642,7 +728,7 @@ test("through the processor, a refused podman identity is refused before the ima ["remote, image absent, floored", async () => answered({ serviceIsRemote: true }), "absent", floor], ]) assert.deepEqual(await run(readInfo, mode, backendFloor), { reason: "job-user-unmappable", reserved: 0 }, label); // A usable identity still meets its floor first, and an undecided read is still retried, never refused. - assert.deepEqual(await run(async () => answered({ controllers: ["io"] }), "present", floor), { reason: "backend-floor-unobserved", reserved: 0 }); + assert.deepEqual(await run(async () => answered(), "present", floor, fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: "io\n" } })), { reason: "backend-floor-unobserved", reserved: 0 }); assert.deepEqual(await run(async () => ({ answered: false, reason: "timeout", transient: true }), "present", floor), { threw: "retry", reserved: 0 }); assert.deepEqual(await run(async () => answered(), "absent", {}), { reason: "job-image-missing", reserved: 0 }, "and a usable venue's missing image is the image's"); // Issue #428: a widening containers.conf is refused as the venue's own answer, RETURNED (never a retry), before the @@ -676,7 +762,8 @@ test("the bundle's observationPreflight answers exactly what judgePodmanVenue an ["clean", answered(), clean(), floor], ["rootful", answered({ rootless: false }), clean(), floor], ["widened", answered(), fakeFs({ files: { [USER_MOUNTS]: "", [conf]: MEASURED.annotations } }), floor], - ["undelegated", answered({ controllers: ["cpu"] }), clean(), floor], + ["undelegated", answered(), fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: "cpu\n" } }), floor], + ["no user manager", answered(), fakeFs({ files: { [USER_MOUNTS]: "", [USER_MANAGER]: null } }), floor], ["unanswered", { answered: false, reason: "timeout", transient: true }, clean(), floor], ["no floor", answered({ controllers: [] }), clean(), {}], // #428's transient rules, which the extraction must carry: a conf chain that could not be read just now rides diff --git a/worker/test/cli.test.mjs b/worker/test/cli.test.mjs index 4996e54f..c9973627 100644 --- a/worker/test/cli.test.mjs +++ b/worker/test/cli.test.mjs @@ -3,6 +3,7 @@ import { execFileSync } from "node:child_process"; import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { test } from "node:test"; +import { fileURLToPath } from "node:url"; import { main } from "../src/cli.mjs"; import { tempDir } from "./helpers/temp-dir.mjs"; @@ -114,3 +115,19 @@ test("run fails FAST (does not hang) when Valkey is unreachable", { skip: needsD assert.equal(code, 1, "an unreachable Valkey is a clean error, not a hang"); assert.ok(Date.now() - start < 15000, "must fail fast, well under any CI timeout"); }); + +test("`pi-dispatch init` hands the CLI's environment to init: PI_BACKENDS=podman in the shell gets the podman ladder (#453 gate)", { skip: process.platform !== "linux" ? "the podman ladder is Linux's; off Linux init says the venue refuses the host, pinned in init.test.mjs" : false }, () => { + const dir = tempDir("pi-cli-init-"); + const cli = fileURLToPath(new URL("../src/cli.mjs", import.meta.url)); + const printed = execFileSync(process.execPath, [cli, "init"], { cwd: dir, env: { ...process.env, PI_BACKENDS: "podman" }, encoding: "utf8" }); + assert.match(printed, /\nNext \(the podman venue; run these as the worker's own account\./); + const docker = execFileSync(process.execPath, [cli, "init"], { cwd: tempDir("pi-cli-init-"), env: { ...process.env, PI_BACKENDS: "local" }, encoding: "utf8" }); + assert.match(docker, /\nNext:\n {2}1\. docker pull/); +}); + +test("`pi-dispatch init` off Linux with PI_BACKENDS=podman in the shell says the venue refuses the host (#453 gate)", { skip: process.platform === "linux" ? "covered on Linux by the ladder test beside this one" : false }, () => { + const dir = tempDir("pi-cli-init-"); + const cli = fileURLToPath(new URL("../src/cli.mjs", import.meta.url)); + const printed = execFileSync(process.execPath, [cli, "init"], { cwd: dir, env: { ...process.env, PI_BACKENDS: "podman" }, encoding: "utf8" }); + assert.match(printed, /\nNext: PI_BACKENDS lists only the podman venue, which runs on Linux alone/); +}); diff --git a/worker/test/doctor.test.mjs b/worker/test/doctor.test.mjs index f5b020f0..51106e62 100644 --- a/worker/test/doctor.test.mjs +++ b/worker/test/doctor.test.mjs @@ -6,7 +6,7 @@ import { makeWaitChecker } from "../src/wait-check.mjs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { PassThrough } from "node:stream"; -import { CANARY_LINES, CANARY_PROBE_SLUGS, EGRESS_CANARY_RUNNER_MODULE, EGRESS_CANARY_STALE_RUNNER, ENV_FILE_READABLE_KEYS, RUN_TIMEOUTS, backendChecks, collectChecks, defaultPromptFn, dockerRunVia, egressCanaryProbeArgs, egressCanaryScript, envFileKeys, countRetained, githubProtectionPreflight, jobUserChecks, liveChecks, liveRunVia, podmanLiveChecks, render, runDoctor, sandboxTombstoneChecks } from "../src/doctor.mjs"; +import { CANARY_LINES, CANARY_PROBE_SLUGS, EGRESS_CANARY_RUNNER_MODULE, EGRESS_CANARY_STALE_RUNNER, ENV_FILE_READABLE_KEYS, RUN_TIMEOUTS, SERVICE_ENV_KEYS, backendChecks, collectChecks, countRetained, defaultPromptFn, dockerRunVia, egressCanaryProbeArgs, egressCanaryScript, envFileKeys, githubProtectionPreflight, jobUserChecks, liveChecks, liveRunVia, podmanLiveChecks, render, runDoctor, sandboxTombstoneChecks, serviceEnvKeys, serviceEnvLoader, urlShown } from "../src/doctor.mjs"; import { EMPTY_PAUSE_WINDOWS, EMPTY_SCOPED_LIMITS } from "../src/init.mjs"; import { EGRESS_CANARY_NET_PREFIX, egressCanaryProbe } from "../src/egress.mjs"; import { JOB_USER_FIX, parseDaemonFacts } from "../src/job-user.mjs"; @@ -115,11 +115,33 @@ function stripFlowLines(output) { // these specific keys win the prefix match over a later, broader "docker run" a test plans for the // in-image gh probe. A correct policy reaches the provider (exit 0) and is blocked from an unlisted host // (exit 3), which is what makes both directions of the allowlist assertable. +// Issue #453: the proxy's state is one inspect, health first, then `egress-proxy-state.mjs`'s status, image and mounts. +// A container `up` or compose made in the folder doctor runs from carries the pinned squid and that folder's two files. +const PROXY_KEY = 'docker inspect --format={"status":'; +const PINNED_SQUID = "ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029"; +const proxyAnswer = (health, status, { image = PINNED_SQUID, cwd = process.cwd(), conf = join(cwd, "deploy/egress-proxy.conf"), allowlist = join(cwd, "egress-allowlist.conf"), entrypoint = ["entrypoint.sh"], cmd = ["-f", "/etc/squid/squid.conf", "-NYC"], stderr } = {}) => ({ + code: 0, + output: `${JSON.stringify({ + status, + health, + image, + entrypoint, + cmd, + mounts: [ + { Type: "bind", Source: conf, Destination: "/etc/squid/squid.conf" }, + { Type: "bind", Source: allowlist, Destination: "/etc/pi-dispatch/allowlist.conf" }, + { Type: "volume", Source: "/var/lib/docker/volumes/a1/_data", Destination: "/var/log/squid" }, + { Type: "volume", Source: "/var/lib/docker/volumes/b2/_data", Destination: "/var/spool/squid" }, + ], + networks: { "pi-dispatch-egress-out": {} }, + })}\n`, + ...(stderr ? { stderr } : {}), +}); const EGRESS_OK = { // Issue #278: the docker CLI's endpoint, answered as a local socket. Here rather than only in `green`, // because plans that build on EGRESS_OK without `green` would otherwise read an unresolved endpoint. "docker context inspect": { code: 0, output: '"desktop-linux"|"unix:///Users/x/.docker/run/docker.sock"\n' }, - "docker inspect --format={{.State.Running}}": { code: 0, output: "true|healthy\n" }, + [PROXY_KEY]: proxyAnswer("healthy", "running"), "docker network": 0, "docker run --rm --name pi-dispatch-egress-probe-provider": 0, "docker run --rm --name pi-dispatch-egress-probe-unlisted": 3, @@ -1598,9 +1620,13 @@ test("doctor: the .env read is narrowed to the two keys these checks name, and n const { out, text } = capture(); await runDoctor(imgEnv(), { ...scaffoldDeps(out, cwd), readEnvFile: (path) => (asked.push(path), readFileSync(path, "utf8")) }); assert.deepEqual(asked, [join(cwd, ".env")], "one file, read once"); - for (const leaked of ["never-read:from-env-file", "openai", "/tmp/evil.sh", "redis://from-env-file:6379"]) { + // Issue #453 (gate round 2) widened the read by exactly the keys the service takes from this file and a doctor's + // shell does not: the venue keys, VALKEY_URL, PI_PROVIDER and the provider's key (its presence). Each is said with + // its source; everything else in the file still reaches nothing. + for (const leaked of ["never-read:from-env-file", "/tmp/evil.sh"]) { assert.ok(!text().includes(leaked), `${leaked} came out of .env and must reach nothing`); } + assert.match(text(), new RegExp(`Valkey reachable \\(redis://from-env-file:6379\\) -- VALKEY_URL read from ${join(cwd, ".env").replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")}, as the service reads it; this shell does not set it`)); assert.match(text(), /PI_PAUSE_WINDOWS_FILE is set in/, "only the two keys the checks name are read back"); }); @@ -2503,6 +2529,16 @@ test("doctor --fix: a missing .env is delegated to init's create-only scaffolds, assert.equal(code, 0); }); +test("doctor --fix: init's next steps follow doctor's venue: PI_BACKENDS=podman gets the podman ladder (#453 gate)", async () => { + const cwd = tempDir("pi-fix-init-podman-"); + const { out, text } = capture(); + await runDoctor(podmanEnv(), { ...podmanDeps(out, podmanPlan(), []), cwd, fileExists: existsSync, platform: "linux", fix: true, promptFn: async () => false }); + assert.ok(existsSync(join(cwd, ".env")), "init created the .env"); + assert.match(text(), /\nNext \(the podman venue; run these as the worker's own account\. First set the account up as docs\/podman\.md "Setup"\n/); + assert.match(text(), / {2}6\. pi-dispatch doctor --live {26}# read the bounds, egress and job user back off real containers\n/); + assert.doesNotMatch(text(), /docker compose -f deploy\/docker-compose\.yml up -d {2}# the durable queue/); +}); + test("doctor --fix: accepting the overlay auth.json offer deletes the file and converges credential-free", async () => { const dir = overlay({ auth: true }); const cwd = tempDir("pi-fix-auth-"); @@ -2935,7 +2971,7 @@ test("doctor: a proxy that is not on the host FAILS, because every job is refuse const { out, text } = capture(); const code = await runDoctor( ghEnv({ PI_EGRESS: "1" }), - ghDeps(out, egressPlan({ "docker inspect --format={{.State.Running}}": 1, "gh auth status": { code: 0, output: ghStatusOutput } })), + ghDeps(out, egressPlan({ [PROXY_KEY]: 1, "gh auth status": { code: 0, output: ghStatusOutput } })), ); // A ✓ would be a lie and a ⚠ would under-report a deployment that cannot run anything at all. assert.equal(code, 1, "an armed policy with no proxy is a hard failure"); @@ -2947,16 +2983,76 @@ test("doctor: a proxy that exists but is STOPPED says so, because the fix is a d const { out, text } = capture(); await runDoctor( ghEnv({ PI_EGRESS: "1" }), - ghDeps(out, egressPlan({ "docker inspect --format={{.State.Running}}": { code: 0, output: "false|none\n" }, "gh auth status": { code: 0, output: ghStatusOutput } })), + ghDeps(out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "exited"), "gh auth status": { code: 0, output: ghStatusOutput } })), ); assert.match(text(), /✗ Egress proxy is stopped \(pi-dispatch-egress-proxy\)/); }); +test("doctor: a paused or crash-looping proxy is not running, and one named by PI_EGRESS_PROXY is started by hand, never by compose (#453 gate)", async () => { + // The worker's simple rule (gate round 3): paused refuses every job, so ✗; restarting and every other non-running + // state is retried by the worker, so ⚠ with a fix that says so. + // `restarting` is a crash loop that fails every job (one retry, then failed), and `created` never starts: both ✗. + for (const [status, fix] of [["restarting", "its squid keeps exiting and its restart policy keeps bringing it back, so every job is retried once, then failed; `docker logs pi-dispatch-egress-proxy` says why"], ["created", "docker compose -f deploy/docker-compose.yml --profile egress up -d -- "]]) { + const failed = capture(); + assert.equal(await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(failed.out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", status), "gh auth status": { code: 0, output: ghStatusOutput } }))), 1, status); + assert.ok(failed.text().includes(`✗ Egress proxy is ${status} (pi-dispatch-egress-proxy)\n → ${fix}`), `${status}:\n${failed.text()}`); + } + const paused = capture(); + assert.equal(await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(paused.out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "paused"), "gh auth status": { code: 0, output: ghStatusOutput } }))), 1); + assert.match(paused.text(), /✗ Egress proxy is paused \(pi-dispatch-egress-proxy\)/); + for (const status of ["stopping", "removing"]) { + const retried = capture(); + assert.equal(await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(retried.out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", status), "gh auth status": { code: 0, output: ghStatusOutput } }))), 0, status); + assert.match(retried.text(), new RegExp(`⚠ Egress proxy is ${status} \\(pi-dispatch-egress-proxy\\)\\n {4}→ a job meanwhile is retried once, then failed \\(the worker does not refuse it outright on this state\\); if it stays ${status}, \`docker logs pi-dispatch-egress-proxy\` says why\\n`), status); + } + const { out, text } = capture(); + await runDoctor(ghEnv({ PI_EGRESS: "1", PI_EGRESS_PROXY: "my-squid" }), ghDeps(out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "exited"), "gh auth status": { code: 0, output: ghStatusOutput } }))); + assert.match(text(), /✗ Egress proxy is stopped \(my-squid\)\n {4}→ PI_EGRESS_PROXY names your own proxy, which neither compose nor `pi-dispatch up` starts: `docker start my-squid` -- /); + assert.doesNotMatch(text(), /✗ Egress proxy is stopped \(my-squid\)\n {4}→ docker compose/); +}); + +test("doctor: a RUNNING shipped proxy from another squid, command or folder is ✗; mounts are compared only in a deployment folder whose sources resolve here (#453 gate)", async () => { + // Real folders, since the mount comparison resolves every bind source on this host (gate round 2). + const folder = tempDir("pi-proxy-folder-"); + mkdirSync(join(folder, "deploy")); + writeFileSync(join(folder, "deploy/egress-proxy.conf"), "http_port 3128\n"); + writeFileSync(join(folder, "egress-allowlist.conf"), "api.anthropic.com\n"); + const other = tempDir("pi-proxy-other-"); + writeFileSync(join(other, "egress-allowlist.conf"), "evil.example\n"); + const run = async (answer, { inFolder = true, env = {} } = {}) => { + const { out, text } = capture(); + const code = await runDoctor(ghEnv({ PI_EGRESS: "1", ...env }), { ...ghDeps(out, egressPlan({ [PROXY_KEY]: answer, "gh auth status": { code: 0, output: ghStatusOutput } })), cwd: folder, proxyFilesExist: () => inFolder }); + return { code, text: text() }; + }; + const esc = (t) => t.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); + const stale = await run(proxyAnswer("healthy", "running", { cwd: folder, image: `ubuntu/squid@sha256:${"0".repeat(64)}` }), { inFolder: false }); + assert.equal(stale.code, 1); + assert.match(stale.text, /✗ Egress proxy is running but is not this deployment's \(pi-dispatch-egress-proxy\): it was created from ubuntu\/squid@sha256:0{64}, not the pinned ubuntu\/squid@sha256:6a097f68[0-9a-f]{56}\n {4}→ `pi-dispatch up` from the deployment folder offers to replace it with the shipped one/); + // The entrypoint and command are judged wherever doctor runs (adversary Y7). + const command = await run(proxyAnswer("healthy", "running", { cwd: folder, entrypoint: ["squid"], cmd: ["-N", "-f", "/tmp/open.conf"] }), { inFolder: false }); + assert.match(command.text, /its entrypoint is \["squid"\], not the image's \["entrypoint\.sh"\]; its command is \["-N","-f","\/tmp\/open\.conf"\]/); + const elsewhere = await run(proxyAnswer("healthy", "running", { cwd: folder, allowlist: join(other, "egress-allowlist.conf") })); + assert.equal(elsewhere.code, 1); + assert.match(elsewhere.text, new RegExp(`its /etc/pi-dispatch/allowlist\\.conf is ${esc(join(other, "egress-allowlist.conf"))}, not ${esc(join(folder, "egress-allowlist.conf"))}`)); + // Run from a folder without the two files, doctor cannot know which folder the service uses: the mounts are not judged. + assert.equal((await run(proxyAnswer("healthy", "running", { cwd: folder, allowlist: join(other, "egress-allowlist.conf") }), { inFolder: false })).code, 0); + // Sources that do not resolve on this host: ⚠ unknown, never ✗ (Docker Desktop's shape, not measured here). + const desktop = await run(proxyAnswer("healthy", "running", { conf: "/host_mnt/Users/op/deploy/deploy/egress-proxy.conf", allowlist: "/host_mnt/Users/op/deploy/egress-allowlist.conf" })); + assert.equal(desktop.code, 0, "an unknown never fails doctor"); + assert.match(desktop.text, /⚠ Egress proxy's mounts could not be compared on this host \(pi-dispatch-egress-proxy\): \/host_mnt\/Users\/op\/deploy\/deploy\/egress-proxy\.conf, \/host_mnt\/Users\/op\/deploy\/egress-allowlist\.conf are paths this host cannot resolve/); + assert.doesNotMatch(desktop.text, /is not this deployment's/); + // The operator's own proxy is never judged against the shipped one. + assert.doesNotMatch((await run(proxyAnswer("healthy", "running", { image: "my/squid:1" }), { env: { PI_EGRESS_PROXY: "my-squid" } })).text, /is not this deployment's|could not be compared/); + // Current: silent. + assert.doesNotMatch((await run(proxyAnswer("healthy", "running", { cwd: folder }))).text, /is not this deployment's|could not be compared/); +}); + + test("doctor: a wedged proxy WARNS rather than fails -- health can flap, and the money gate ignores it", async () => { const { out, text } = capture(); const code = await runDoctor( ghEnv({ PI_EGRESS: "1" }), - ghDeps(out, egressPlan({ "docker inspect --format={{.State.Running}}": { code: 0, output: "true|unhealthy\n" }, "gh auth status": { code: 0, output: ghStatusOutput } })), + ghDeps(out, egressPlan({ [PROXY_KEY]: proxyAnswer("unhealthy", "running"), "gh auth status": { code: 0, output: ghStatusOutput } })), ); assert.equal(code, 0, "a flapping healthcheck must not make doctor red"); assert.match(text(), /⚠ Egress proxy health: unhealthy/); @@ -3922,7 +4018,7 @@ test("doctor --live reads the eight back, reports the limits, leaves no fixture, const env = liveEnv(); const { out, text } = capture(); const calls = []; - const code = await runDoctor(env, { ...ghDeps(out, { ...liveOk(), ...green }, calls), live: true, liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); + const code = await runDoctor(env, { ...ghDeps(out, { ...liveOk(), ...green }, calls), live: true, ...instantClock(), liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); assert.equal(code, 0, text()); assert.equal(calls.filter((c) => c.args[0] === "context").length, 2, "the endpoint is read by the collection AND again right before the probe"); for (const property of ["isolation", "ephemeral", "mountSet", "imagePinning", "nonRoot", "localFolders"]) { @@ -3946,7 +4042,7 @@ test("doctor --live reads the eight back, reports the limits, leaves no fixture, test("doctor --live renders a failed read-back as a hard failure with the declared word beside the observed", async () => { const { out, text } = capture(); - const code = await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk({ uid: "0" }), ...green }), live: true, liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); + const code = await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk({ uid: "0" }), ...green }), live: true, ...instantClock(), liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); assert.equal(code, 1); assert.match(text(), /✗ read back on local: nonRoot does NOT hold -- declared asserted, observed: Uid 0 0 0 0/); assert.match(text(), /→ PI_JOB_IMAGE runs as root/); @@ -3956,7 +4052,7 @@ test("doctor --live on a docker CLI not observed local runs no container and say const { out, text } = capture(); const calls = []; const plan = { ...liveOk(), ...green, "docker context inspect": { code: 0, output: '"remote"|"tcp://10.1.2.3:2375"\n' } }; - await runDoctor(liveEnv(), { ...ghDeps(out, plan, calls), live: true, liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false }); + await runDoctor(liveEnv(), { ...ghDeps(out, plan, calls), live: true, ...instantClock(), liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false }); assert.ok(!calls.some((c) => c.args.some((a) => String(a).includes("pi-dispatch-live")))); assert.equal(text().match(/read back on local/g).length, 1); assert.match(text(), /⚠ read back on local: not run -- this shell's docker CLI is not observed to point at this host/); @@ -3965,12 +4061,12 @@ test("doctor --live on a docker CLI not observed local runs no container and say test("doctor --live checks carry no fixAction: a failed read-back is never something doctor fixes", async () => { const env = liveEnv(); const facts = { endpoint: { local: true }, dockerCode: 0, imageCode: 0, jobImage: "pi-job:latest", triggerImages: ["a:1", "b:2"], egress: { armed: false, results: [] } }; - const checks = await liveChecks(env, { spawn: fakeSpawn({ ...liveOk({ uid: "0" }), ...green }), home: "/home/u", liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); + const checks = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...liveOk({ uid: "0" }), ...green }), home: "/home/u", liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); assert.ok(checks.length >= 7); assert.ok(checks.every((c) => c.fixAction === undefined)); assert.match(checks.at(-1).label, /not the 2 image\(s\) your triggers name/); // The not-run path too: a notes-only result must not grow an offer either. - const notRun = await liveChecks(env, { spawn: fakeSpawn(green), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, { ...facts, imageCode: 1 }); + const notRun = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn(green), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, { ...facts, imageCode: 1 }); assert.equal(notRun.length, 1); assert.match(notRun[0].label, /not run -- the job image pi-job:latest is not present/); assert.equal(notRun[0].fixAction, undefined); @@ -3981,7 +4077,7 @@ test("doctor --live gives each localFolders failure its own fix, and names what mkdirSync(join(env.PI_JOBS_DIR, "pi-dispatch-live-99999-aB3xYz")); const facts = { endpoint: { local: true }, dockerCode: 0, imageCode: 0, jobImage: "pi-job:latest", triggerImages: [], egress: { armed: false, results: [] } }; const invisible = { ...liveOk(), [`docker exec ${LIVE_ID} sh -c [ -r /job ]`]: { code: 0, output: "wrote\n" } }; - const checks = await liveChecks(env, { spawn: fakeSpawn({ ...invisible, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); + const checks = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...invisible, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); assert.match(checks[0].label, /✓?read back on local: removed fixture pi-dispatch-live-99999-aB3xYz, left by an interrupted --live run/); const folders = checks.find((c) => /localFolders does NOT hold/.test(c.label)); assert.match(folders.fix, /file sharing/, "a write the host cannot see is not the uid problem"); @@ -3992,7 +4088,7 @@ test("doctor --fix --live reads back ONCE, after the fix pass, from the re-colle const cwd = tempDir("pi-live-fix-"); // no .env, so the silent `init` fix runs and forces a re-collect const { out, text } = capture(); const calls = []; - await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk(), ...green }, calls), fileExists: existsSync, cwd, fix: true, promptFn: async () => false, live: true, liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk(), ...green }, calls), fileExists: existsSync, cwd, fix: true, promptFn: async () => false, live: true, ...instantClock(), liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); assert.match(text(), /re-check after fixes/, "the fixture must actually drive a re-collection"); // `docker info` bare: the job-user read (`info --format=...`, issue #341) is a different question. const infos = calls.map((c, i) => [c, i]).filter(([c]) => c.args[0] === "info" && c.args.length === 1).map(([, i]) => i); @@ -4006,7 +4102,7 @@ test("doctor --live with PI_JOBS_DIR unset builds its fixture under the injected const tmp = tempDir("pi-live-tmpdir-"); const { out, text } = capture(); const env = { PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x", PI_EGRESS: "0", TMPDIR: tmp }; - await runDoctor(env, { ...ghDeps(out, { ...liveOk(), ...green }), live: true, liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(env, { ...ghDeps(out, { ...liveOk(), ...green }), live: true, ...instantClock(), liveFs: liveFsAs(1001), jobUserIdentity: LINUX_1001, isAlive: () => false, pid: 7, nonce: "n" }); assert.ok(text().includes(`with a fixture under ${tmp}/pi-dispatch/jobs;`), text()); assert.deepEqual(readdirSync(join(tmp, "pi-dispatch", "jobs")), [], "and removes it"); }); @@ -4024,17 +4120,25 @@ test("doctor --live's not-run path still names what it swept", async () => { mkdirSync(join(env.PI_JOBS_DIR, "pi-dispatch-live-99998-aB3xYz")); const facts = { endpoint: { local: true }, dockerCode: 0, imageCode: 0, jobImage: "pi-job:latest", triggerImages: [], egress: { armed: false, results: [] } }; const failing = { ...liveOk(), "docker run --name=pi-dispatch-live-probe-": { code: 127, output: `${LIVE_ID}\n` }, "docker rm -f": 1 }; - const checks = await liveChecks(env, { spawn: fakeSpawn({ ...failing, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); + const checks = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...failing, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); assert.match(checks[0].label, /removed fixture pi-dispatch-live-99998-aB3xYz/); assert.ok(checks.some((c) => /not run -- the probe container did not start/.test(c.label))); assert.ok(checks.some((c) => c.warn && new RegExp(`could not be removed: docker rm -f ${LIVE_ID}`).test(c.label)), "a failed removal by ID is said on this path too"); }); +test("doctor --live's probe that did not start carries the runtime's own words, one clean line (#453 gate)", async () => { + const env = liveEnv(); + const facts = { endpoint: { local: true }, dockerCode: 0, imageCode: 0, jobImage: "pi-job:latest", triggerImages: [], egress: { armed: false, results: [] } }; + const failing = { ...liveOk(), "docker run --name=pi-dispatch-live-probe-": { code: 126, output: "", stderr: `\nError: OCI runtime error: crun: controller \`cpu\` is not available${String.fromCharCode(27)}[2J\nsecond line\n` } }; + const checks = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...failing, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); + assert.ok(checks.some((c) => c.label.includes("not run -- the probe container did not start (docker said: Error: OCI runtime error: crun: controller `cpu` is not available [2J), so nothing was read back")), checks.map((c) => c.label).join("\n")); +}); + test("doctor --live's not-writable localFolders failure keeps its own ownership fix", async () => { const env = liveEnv(); const facts = { endpoint: { local: true }, dockerCode: 0, imageCode: 0, jobImage: "pi-job:latest", triggerImages: [], egress: { armed: false, results: [] } }; const notWritable = { ...liveOk(), [`docker exec ${LIVE_ID} sh -c [ -r /job ]`]: { code: 0, output: "not-writable /workspace\n" } }; - const checks = await liveChecks(env, { spawn: fakeSpawn({ ...notWritable, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); + const checks = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...notWritable, ...green }), liveFs, isAlive: () => false, pid: 1, nonce: "n" }, facts); assert.match(checks.find((c) => /localFolders does NOT hold/.test(c.label)).fix, /the account the worker runs as/, "issue #341: the fix names the worker's uid, no longer the image's 1001"); }); @@ -4248,19 +4352,19 @@ test("doctor --live runs its probes as the decided job user and reads the uid ba const { out, text } = capture(); const calls = []; const plan = { ...liveOk({ uid: "1234" }), ...infoPlan(ROOTFUL_INFO), ...imageLabels("anyUid"), ...green }; - const code = await runDoctor(liveEnv(), { ...ghDeps(out, plan, calls), live: true, liveFs: liveFsAs(1234), jobUserIdentity: LINUX_ID(1234), stat: socketStat, isAlive: () => false, pid: 7, nonce: "n" }); + const code = await runDoctor(liveEnv(), { ...ghDeps(out, plan, calls), live: true, ...instantClock(), liveFs: liveFsAs(1234), jobUserIdentity: LINUX_ID(1234), stat: socketStat, isAlive: () => false, pid: 7, nonce: "n" }); assert.equal(code, 0, text()); assert.ok(calls.filter((c) => c.args[0] === "run" && String(c.args[1]).startsWith("--name=pi-dispatch-live-")).every((c) => c.args.includes("--user=1234:1234")), "the reading and the pinning probe"); assert.match(text(), /✓ read back on local: the probe ran as uid 1234, the job user this host decides \(1234:1234\)/); assert.match(text(), /it ran as the job user 1234:1234, decided for this shell \(uid 1234\)/); const wrong = capture(); - assert.equal(await runDoctor(liveEnv(), { ...ghDeps(wrong.out, { ...liveOk({ uid: "1001" }), ...infoPlan(ROOTFUL_INFO), ...imageLabels("anyUid"), ...green }), live: true, liveFs: liveFsAs(1234), jobUserIdentity: LINUX_ID(1234), stat: socketStat, isAlive: () => false, pid: 7, nonce: "n" }), 1); + assert.equal(await runDoctor(liveEnv(), { ...ghDeps(wrong.out, { ...liveOk({ uid: "1001" }), ...infoPlan(ROOTFUL_INFO), ...imageLabels("anyUid"), ...green }), live: true, ...instantClock(), liveFs: liveFsAs(1234), jobUserIdentity: LINUX_ID(1234), stat: socketStat, isAlive: () => false, pid: 7, nonce: "n" }), 1); assert.match(wrong.text(), /✗ read back on local: the probe ran as uid 1001, not the decided job user 1234:1234/); const refused = capture(); const refusedCalls = []; - await runDoctor(liveEnv(), { ...ghDeps(refused.out, { ...liveOk(), ...infoPlan(ROOTFUL_INFO), ...imageLabels("replicas"), ...green }, refusedCalls), live: true, liveFs: liveFsAs(1234), jobUserIdentity: LINUX_ID(1234), stat: socketStat, isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv(), { ...ghDeps(refused.out, { ...liveOk(), ...infoPlan(ROOTFUL_INFO), ...imageLabels("replicas"), ...green }, refusedCalls), live: true, ...instantClock(), liveFs: liveFsAs(1234), jobUserIdentity: LINUX_ID(1234), stat: socketStat, isAlive: () => false, pid: 7, nonce: "n" }); assert.match(refused.text(), /⚠ read back on local: not run -- a local job is refused as this uid \(any-uid-unsupported\)/); assert.ok(!refusedCalls.some((c) => c.args.some((a) => String(a).includes("pi-dispatch-live"))), "no probe for a container no job gets"); }); @@ -4273,7 +4377,7 @@ test("doctor --live runs nothing where no job would run as a decided uid: a root ]) { const { out, text } = capture(); const calls = []; - await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk(), ...plan, ...green }, calls), live: true, liveFs: liveFsAs(1234), jobUserIdentity: ids, stat: () => ({ uid: 0, gid: 2375 }), isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk(), ...plan, ...green }, calls), live: true, ...instantClock(), liveFs: liveFsAs(1234), jobUserIdentity: ids, stat: () => ({ uid: 0, gid: 2375 }), isAlive: () => false, pid: 7, nonce: "n" }); assert.ok(!calls.some((c) => c.args.some((a) => String(a).includes("pi-dispatch-live"))), `${label}: no probe`); assert.match(text(), new RegExp(`⚠ read back on local: not run -- ${reason.source}`), label); } @@ -4288,7 +4392,7 @@ test("doctor --live gives job-unreadable, mount-not-writable and not-yours each ["mount-not-writable", { [`docker exec ${LIVE_ID} sh -c [ -r /job ]`]: { code: 0, output: "not-writable /outbox\n" } }, 1234, /outbox or session mount/], ["not-yours", {}, 999, /owned by another uid, so the worker cannot remove/], ]) { - const checks = await liveChecks(env, { spawn: fakeSpawn({ ...liveOk({ uid: "1234" }), ...over, ...green }), liveFs: liveFsAs(fsUid), isAlive: () => false, pid: 1, nonce: "n", jobUserIdentity: LINUX_ID(1234) }, facts); + const checks = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...liveOk({ uid: "1234" }), ...over, ...green }), liveFs: liveFsAs(fsUid), isAlive: () => false, pid: 1, nonce: "n", jobUserIdentity: LINUX_ID(1234) }, facts); const folders = checks.find((c) => /localFolders does NOT hold/.test(c.label)); assert.ok(folders, `${label}: ${checks.map((c) => c.label).join("\n")}`); assert.match(folders.fix, pattern, label); @@ -4296,10 +4400,10 @@ test("doctor --live gives job-unreadable, mount-not-writable and not-yours each } assert.equal(new Set(fixes).size, 3, "three causes, three fixes: none falls back to the generic one"); // A sudo'd doctor is not the worker, and root can remove anything: no owner comparison, so no false not-yours. - const root = await liveChecks(env, { spawn: fakeSpawn({ ...liveOk({ uid: "1234" }), ...green }), liveFs: liveFsAs(999), isAlive: () => false, pid: 1, nonce: "n", jobUserIdentity: { platform: "darwin", euid: 0, egid: 0 } }, facts); + const root = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...liveOk({ uid: "1234" }), ...green }), liveFs: liveFsAs(999), isAlive: () => false, pid: 1, nonce: "n", jobUserIdentity: { platform: "darwin", euid: 0, egid: 0 } }, facts); assert.ok(!root.some((c) => /not-yours|owned by another uid/.test(`${c.label} ${c.fix ?? ""}`))); assert.match(root.at(-1).label, /the host owner of what the probe wrote was not compared, because doctor ran as root/, "and the limits line says so"); - const shell = await liveChecks(env, { spawn: fakeSpawn({ ...liveOk({ uid: "1234" }), ...green }), liveFs: liveFsAs(1234), isAlive: () => false, pid: 1, nonce: "n", jobUserIdentity: LINUX_ID(1234) }, facts); + const shell = await liveChecks(env, { ...instantClock(), spawn: fakeSpawn({ ...liveOk({ uid: "1234" }), ...green }), liveFs: liveFsAs(1234), isAlive: () => false, pid: 1, nonce: "n", jobUserIdentity: LINUX_ID(1234) }, facts); assert.doesNotMatch(shell.at(-1).label, /was not compared/, "only a root run says it"); }); @@ -4387,7 +4491,7 @@ test("doctor's facts carry the proxy's name and whether it is up, so --live's pe assert.equal(facts.egress.armed, true); assert.deepEqual([facts.egress.proxy, facts.egress.proxyRunning], ["pi-dispatch-egress-proxy", true]); const stopped = {}; - await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, { ...collectSeams({ ...green, "docker inspect --format={{.State.Running}}": { code: 0, output: "false|none\n" } }), facts: stopped }); + await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, { ...collectSeams({ ...green, [PROXY_KEY]: proxyAnswer("none", "exited") }), facts: stopped }); assert.equal(stopped.egress.proxyRunning, false); const off = {}; await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x", PI_EGRESS: "0" }, { ...collectSeams(green), facts: off }); @@ -4408,7 +4512,7 @@ test("doctor --live with the policy armed reads jobToJobIsolation back through t assert.match(failed.fix, /reached another's across their own --internal networks/); assert.match(reached.at(-1).label, /jobToJobIsolation tried one pair of peers/, "a reach needed both peers started, so the limit still applies to it"); - const noAddress = await liveChecks(env, { ...seams({}), spawn: fakeSpawn({ ...liveOk(), ...livePeersOk({}), "docker inspect --format={{json .NetworkSettings.Networks}}": { code: 1, output: "" }, ...green }) }, facts); + const noAddress = await liveChecks(env, { ...instantClock(), ...seams({}), spawn: fakeSpawn({ ...liveOk(), ...livePeersOk({}), "docker inspect --format={{json .NetworkSettings.Networks}}": { code: 1, output: "" }, ...green }) }, facts); assert.ok(noAddress.some((c) => c.warn && /jobToJobIsolation not read back/.test(c.label))); assert.doesNotMatch(noAddress.at(-1).label, /tried one pair of peers/, "peers that were not read back are not claimed as tried"); const proxyDown = await liveChecks(env, seams({}), { ...facts, egress: { ...facts.egress, proxyRunning: false } }); @@ -4535,7 +4639,7 @@ test("doctor names the runtime that answered, and warns on podman-docker, where test("doctor parses the proxy's state and this host's image id from stdout only, so podman-docker's stderr banner cannot flip them (#345)", async () => { const banner = "Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.\n"; const { out, text } = capture(); - await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(out, egressPlan({ "docker inspect --format={{.State.Running}}": { code: 0, output: "true|healthy\n", stderr: banner }, "gh auth status": { code: 0, output: ghStatusOutput } }))); + await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(out, egressPlan({ [PROXY_KEY]: proxyAnswer("healthy", "running", { stderr: banner }), "gh auth status": { code: 0, output: ghStatusOutput } }))); assert.match(text(), /✓ Egress proxy running \(pi-dispatch-egress-proxy\)/); assert.doesNotMatch(text(), /Egress proxy is stopped/); @@ -5747,7 +5851,7 @@ test("doctor --live reads relabel from the collection's own daemon answer, end t for (const [label, body, want] of [["podman with selinux", PODMAN_SELINUX_INFO, true], ["docker", ROOTFUL_INFO, false]]) { const { out } = capture(); const calls = []; - await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk(), ...infoPlan(body), ...green }, calls), live: true, liveFs, jobUserIdentity: LINUX_1001, stat: socketStat, observationFs: noHostFiles, isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv(), { ...ghDeps(out, { ...liveOk(), ...infoPlan(body), ...green }, calls), live: true, ...instantClock(), liveFs, jobUserIdentity: LINUX_1001, stat: socketStat, observationFs: noHostFiles, isAlive: () => false, pid: 7, nonce: "n" }); const probe = calls.find((c) => c.cmd === "docker" && c.args[0] === "run" && c.args.includes("sleep")); assert.ok(probe, `${label}: the probe ran`); assert.equal(probe.args.some((a) => a.endsWith(":/job:ro,Z")), want, label); @@ -5791,7 +5895,7 @@ test("doctor's two docker runners spawn the bin they are given, and docker when // runtime from a variable, the in-image gh probe, which is docker's with `local` and podman's without it. A new podman // spawn, or a docker one turned into a variable, lands here as a diff rather than going unseen. const direct = [...doctorSource.matchAll(/runCmd(?:Capture)?\(\w*[sS]pawn, "podman", \[("[^"]*", "[^"]*")/g)].map((m) => m[1]); - assert.deepEqual(direct, ['"image", "inspect"', '"image", "inspect"', '"pull", "ghcr.io/edgehero/pi-job:latest"', '"tag", "ghcr.io/edgehero/pi-job:latest"', '"inspect", "--format={{.State.Running}}"']); + assert.deepEqual(direct, ['"image", "inspect"', '"image", "inspect"', '"pull", "ghcr.io/edgehero/pi-job:latest"', '"tag", "ghcr.io/edgehero/pi-job:latest"', '"inspect", "--format={{.State.Status}}"']); const chosen = [...doctorSource.matchAll(/runCmd(?:Capture)?\(\w*[sS]pawn, ([a-z]\w*Bin)\b/g)].map((m) => m[1]); assert.deepEqual(chosen, ["ghProbeBin"]); }); @@ -5800,15 +5904,28 @@ test("doctor's two docker runners spawn the bin they are given, and docker when // `podman info --format json` as rootless Podman 5.8.1 on Fedora 44 answers it (measured), reduced to the keys read. const PODMAN_INFO = (over = {}, security = {}) => - JSON.stringify({ host: { security: { rootless: true, selinuxEnabled: false, ...security }, serviceIsRemote: false, cgroupVersion: "v2", cgroupControllers: ["cpuset", "cpu", "io", "memory", "pids"], ...over }, version: { Version: "5.8.1" } }); + JSON.stringify({ host: { security: { rootless: true, selinuxEnabled: false, ...security }, serviceIsRemote: false, cgroupVersion: "v2", cgroupManager: "systemd", cgroupControllers: ["cpuset", "cpu", "io", "memory", "pids"], ...over }, version: { Version: "5.8.1" } }); // The worker account's own mounts.conf, empty: the documented override that earns podmanAddsNoMounts. const PODMAN_HOME = "/home/op"; -const podmanFs = { ...noHostFiles, statSync: (p) => (p === `${PODMAN_HOME}/.config/containers/mounts.conf` ? { size: 0 } : noHostFiles.statSync(p)) }; +// And the account's user manager running with the controllers delegated (issue #453): user@1234.service's cgroup. +const PODMAN_USER_MANAGER = "/sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/cgroup.controllers"; +const podmanFs = { + ...noHostFiles, + statSync: (p) => (p === `${PODMAN_HOME}/.config/containers/mounts.conf` ? { size: 0 } : noHostFiles.statSync(p)), + readFileSync: (p) => (p === PODMAN_USER_MANAGER ? "cpuset cpu io memory pids\n" : noHostFiles.readFileSync(p)), +}; +// The same host with the user manager's controllers given (null: no user manager running for the account). +// `self` is this process's /proc/self/cgroup (null: unread), which decides the bounds only when Podman does not use the +// systemd cgroup manager (issue #453, gate round 1). +const podmanFsWith = (controllers, self = null) => ({ + ...podmanFs, + readFileSync: (p) => (p === PODMAN_USER_MANAGER && controllers !== null ? controllers : p === "/proc/self/cgroup" && self !== null ? self : noHostFiles.readFileSync(p)), +}); // A podman host that answers everything, docker absent altogether. "docker" LAST, so it catches every docker call. -const podmanPlan = ({ info = PODMAN_INFO(), capabilities = "anyUid", image = true, proxy = "true" } = {}) => ({ +const podmanPlan = ({ info = PODMAN_INFO(), capabilities = "anyUid", image = true, proxy = "running" } = {}) => ({ "podman info": { code: 0, output: `${info}\n` }, "podman image inspect --format={{.Id}}": image ? { code: 0, output: `abc|0.80.7||${capabilities}\n` } : { code: 125, output: "" }, - "podman inspect --format={{.State.Running}}": proxy === null ? { code: 125, output: "" } : { code: 0, output: `${proxy}\n` }, + "podman inspect --format={{.State.Status}}": proxy === null ? { code: 125, output: "" } : { code: 0, output: `${proxy}\n` }, docker: "enoent", }); const podmanEnv = (extra = {}) => ({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x", GITHUB_AUTH_SOURCE: "pat", PI_BACKENDS: "podman", PI_EGRESS: "0", ...extra }); @@ -6028,7 +6145,7 @@ const MIXED_PIN = { "✓ podman: cgroup v2 controllers are delegated to this account (cpuset, cpu, io, memory, pids), so a job's pid, memory and cpu bounds are applied", "✓ podman: SELinux does not confine containers here, so nothing a job mounts is relabelled", "✗ podman: job image is not in this account's Podman store (pi-job:latest)", - " → pull or load it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest (or `docker save pi-job:latest | podman load`) -- jobs run with --pull=never, so the worker never fetches it", + " → pull or load it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest -- jobs run with --pull=never, so the worker never fetches it", "✓ podman: jobs run as uid:gid 1234:1234 (passed as --user, with --userns=keep-id) with HOME=/home/pi, once the job image is in this account's store and declares anyUid", "✓ Valkey reachable (redis://127.0.0.1:6379)", "✓ Fleet: 2 workers (mini1, mini2)", @@ -6172,7 +6289,7 @@ test("a widening containers.conf is a podman venue refusal: ✗ where podman is assert.equal(busyCode, 0); // `--live` names the conf line as the one to fix, not the job-user line (the reviewer's R3). const live = capture(); - await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(live.out, { ...podmanLiveOk(), ...podmanPlan() }, [], { observationFs: withConf("annotations = []\n") }), live: true, liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(live.out, { ...podmanLiveOk(), ...podmanPlan() }, [], { observationFs: withConf("annotations = []\n") }), live: true, ...instantClock(), liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n" }); assert.match(live.text(), /⚠ read back on podman: not run -- a podman job is refused here \(podman-conf-widens-job\)[^\n]*\n {4}→ fix the podman containers\.conf line above first/); }); @@ -6220,17 +6337,17 @@ test("with egress armed the podman proxy is read under podman, and the docker eg const code = await runDoctor(podmanEnv({ PI_EGRESS: "1" }), podmanDeps(out, podmanPlan({ proxy }), calls)); return { code, text: text(), calls }; }; - const up = await run("true"); + const up = await run("running"); assert.match(up.text, /✓ podman: egress proxy running under this account's Podman \(pi-dispatch-egress-proxy\)/); // Issue #431: a plain run points at --live, where this venue's canary runs; it starts no canary container itself. assert.match(up.text, /⚠ podman: the egress allowlist is read back by `pi-dispatch doctor --live` on this venue, not by this run/); assert.ok(!up.calls.some((c) => c.args.some((a) => /pi-dispatch-egress-(doctor|probe)-/.test(String(a)))), "no canary object is touched by a plain run"); assert.doesNotMatch(up.text, /Egress (policy|proxy)/, "docker's egress lines are about a proxy no job here is wired to"); - assert.deepEqual(up.calls.find((c) => c.cmd === "podman" && c.args[0] === "inspect")?.args, ["inspect", "--format={{.State.Running}}", "pi-dispatch-egress-proxy"]); + assert.deepEqual(up.calls.find((c) => c.cmd === "podman" && c.args[0] === "inspect")?.args, ["inspect", "--format={{.State.Status}}", "pi-dispatch-egress-proxy"]); const down = await run(null); assert.equal(down.code, 1); assert.match(down.text, /✗ podman: egress proxy is not under this account's Podman \(pi-dispatch-egress-proxy\)\n {4}→ start it as the worker's account, under the same rootless Podman, on a named bridge network/); - const stopped = await run("false"); + const stopped = await run("exited"); assert.match(stopped.text, /✗ podman: egress proxy is stopped under this account's Podman/); assert.doesNotMatch(stopped.text, /allowlist is read back by/, "only said beside a running proxy"); }); @@ -6297,12 +6414,12 @@ test("doctor --live on podman runs nothing when podman is re-read as remote, or const flips = { ...podmanLiveOk(), ...podmanPlan(), "podman info": () => ({ code: 0, output: `${PODMAN_INFO(++reads > 1 ? { serviceIsRemote: true } : {})}\n` }) }; const { out, text } = capture(); const calls = []; - await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(out, flips, calls), live: true, liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(out, flips, calls), live: true, ...instantClock(), liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n" }); assert.ok(!calls.some((c) => c.args.some((a) => String(a).includes("pi-dispatch-live"))), "no probe after the re-read"); assert.match(text(), /⚠ read back on podman: not run -- this shell's podman CLI is not observed to point at this host/); const refused = capture(); const refusedCalls = []; - await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(refused.out, { ...podmanLiveOk(), ...podmanPlan({ capabilities: "" }) }, refusedCalls), live: true, liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n" }); + await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(refused.out, { ...podmanLiveOk(), ...podmanPlan({ capabilities: "" }) }, refusedCalls), live: true, ...instantClock(), liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n" }); assert.match(refused.text(), /⚠ read back on podman: not run -- a podman job is refused as this uid \(any-uid-unsupported\), so a probe would read back a container no job gets\n {4}→ fix the podman job-user line above first/); assert.ok(!refusedCalls.some((c) => c.args.some((a) => String(a).includes("pi-dispatch-live")))); }); @@ -6439,6 +6556,9 @@ test("podman-only: a trigger-named image is asked of this account's Podman store const absent = await run({ "podman image inspect my-python:1.2.0": 1, ...podmanPlan() }); assert.equal(absent.code, 1, "a trigger that can never run fails, as on docker"); assert.match(absent.text, /✗ podman: trigger job image present in this account's Podman store \(my-python:1\.2\.0\)\n {4}→ pull, load or build it AS THE WORKER'S ACCOUNT[^\n]*podman pull my-python:1\.2\.0[^\n]*--pull=never/); + // The whole fix line (issue #453): `docker save ... | podman load` was offered as an alternative, and a podman-only + // host has no docker to save from. + assert.ok(absent.text.includes("\n → pull, load or build it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: podman pull my-python:1.2.0 -- a trigger names it in run.image, and jobs run with --pull=never, so the worker never fetches it at job time\n"), absent.text); assert.deepEqual(absent.calls.filter((c) => c.cmd === "docker"), [], "no docker image inspect"); assert.deepEqual(absent.calls.filter((c) => c.args.includes("my-python:1.2.0")).map((c) => [c.cmd, ...c.args]), [["podman", "image", "inspect", "my-python:1.2.0"]]); const wrapped = await run({ "podman image inspect --format={{json": { code: 0, output: '["/bin/sh"]\n' }, "podman image inspect my-python:1.2.0": 0, ...podmanPlan() }); @@ -6894,11 +7014,199 @@ test("the podman canary's argv cannot be built weaker than a job's: no job user, test("the podman bounds line is said only when the controllers are delegated (#354)", async () => { const { out, text } = capture(); - await runDoctor(podmanEnv(), podmanDeps(out, podmanPlan({ info: PODMAN_INFO({ cgroupControllers: ["cpu", "memory"] }) }), [])); + await runDoctor(podmanEnv(), podmanDeps(out, podmanPlan(), [], { observationFs: podmanFsWith("cpu memory\n") })); assert.doesNotMatch(text(), /podman: cgroup v2 controllers are delegated/); - assert.match(text(), /⚠ podman: isolation is ASSERTED by this account's Podman setup, not enforced: the pids cgroup controller is not delegated/); + assert.match(text(), /⚠ podman: isolation is ASSERTED by this account's Podman setup, not enforced: the pids cgroup controller is not delegated to this account's systemd user manager \(user@1234\.service\)/); +}); + +test("an account with no systemd user manager running, or one Podman cannot reach: doctor names the cause once, as --live's isolation failure does (#453)", async () => { + // Measured (round-446 M0-e E1, gate 456 L1-L3): linger off and a `sudo -iu` shell, or a manager Podman reaches no bus + // to, podman info still listing every controller. + const noManager = { observationFs: podmanFsWith(null) }; + const EVIDENCE = "no systemd user manager is running for this account (/sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/cgroup.controllers does not exist), so Podman leaves a job's pids, memory and cpu bounds unapplied, whichever cgroup manager it uses: with linger off the manager runs only while the account has a login session, and a `sudo -iu` shell starts none (measured). On a host without systemd managing cgroups under user.slice this path never exists, and the venue gives no credit there"; + const FIX = " → turn on linger for the worker's account: `sudo loginctl enable-linger ` starts its systemd user manager and keeps it running with no one logged in, and `pi-dispatch service install` runs the worker as a systemd user service inside it. A hand-written system unit with `User=` also needs `Wants=user@.service` and `After=user@.service` (docs/podman.md); `pi-dispatch doctor --live` reads pids.max and memory.max back off a real container\n"; + const ISOLATION = `⚠ podman: isolation is ASSERTED by this account's Podman setup, not enforced: ${EVIDENCE}\n${FIX}`; + // No floor: jobs still run, so one ⚠, the backend section's, carrying the cause and its fix; podman info's controller + // list no longer earns the ✓ line, and the podman section does not repeat the ⚠. + const plain = capture(); + assert.equal(await runDoctor(podmanEnv(), podmanDeps(plain.out, podmanPlan(), [], noManager)), 0); + assert.ok(plain.text().includes(ISOLATION), plain.text()); + assert.equal(plain.text().match(/no systemd user manager is running/g).length, 1, "said once"); + assert.doesNotMatch(plain.text(), /podman: cgroup v2 controllers are delegated/); + // A floor that needs the bounds: the same one line, and the floor line is the ✗ that fails doctor, saying the worker + // refuses to boot on this answer. + const floored = capture(); + assert.equal(await runDoctor(podmanEnv({ PI_BACKEND_FLOOR: "isolation=enforced" }), podmanDeps(floored.out, podmanPlan(), [], noManager)), 1); + assert.ok(floored.text().includes(ISOLATION), floored.text()); + assert.equal(floored.text().match(/no systemd user manager is running/g).length, 1, "said once"); + assert.match(floored.text(), /✗ PI_BACKEND_FLOOR asks for isolation=enforced \(podman provides it only while this worker's rootless Podman runs on cgroup v2 with a systemd user manager running for its account, the pids, memory and cpu controllers delegated to that manager, and Podman putting containers under it, and no containers\.conf it reads sets `cgroups`, and that is not observed\)\n {4}→ Turn on linger for the worker account[^\n]* The worker refuses to boot, and refuses each job, on the same answer\.\n/); + // A manager Podman cannot reach (gate L1: no user bus, so podman info reports cgroupfs) from a shell outside it: its + // own cause, and its own fix, which leads with running inside the manager. + const unreachable = capture(); + await runDoctor(podmanEnv(), podmanDeps(unreachable.out, podmanPlan({ info: PODMAN_INFO({ cgroupManager: "cgroupfs" }) }), [], { observationFs: podmanFsWith("cpuset cpu io memory pids\n", "0::/user.slice/user-501.slice/session-5.scope\n") })); + assert.ok(unreachable.text().includes("⚠ podman: isolation is ASSERTED by this account's Podman setup, not enforced: Podman is using the cgroupfs cgroup manager, not systemd, and this worker is not running inside the account's user manager (user@1234.service), so whether a job's pids, memory and cpu bounds are applied is not observed from here. Measured: where a configured systemd manager fell back to cgroupfs because Podman could not reach the user manager over D-Bus (no user bus socket, a DBUS_SESSION_BUS_ADDRESS pointing nowhere), the container landed in the worker's own cgroup unbounded; an explicit cgroupfs with the bus reachable had its bounds applied, and nothing read here tells the two apart\n → run the worker inside the account's user manager, as a systemd user service (`pi-dispatch service install`, with linger on), where a job's container lands under the manager whichever cgroup manager Podman uses; or let Podman reach the manager over the account's user bus, /run/user//bus (on Debian and Ubuntu the dbus-user-session package provides it), with no DBUS_SESSION_BUS_ADDRESS pointing elsewhere and no `cgroup_manager = \"cgroupfs\"` in its containers.conf; `pi-dispatch doctor --live` reads pids.max and memory.max back off a real container\n"), unreachable.text()); + // The same cgroupfs from inside the manager (measured E3a, E3b) is credited. + const inside = capture(); + await runDoctor(podmanEnv(), podmanDeps(inside.out, podmanPlan({ info: PODMAN_INFO({ cgroupManager: "cgroupfs" }) }), [], { observationFs: podmanFsWith("cpuset cpu io memory pids\n", "0::/user.slice/user-1234.slice/user@1234.service/app.slice/pi-dispatch-worker.service\n") })); + assert.match(inside.text(), /✓ podman: cgroup v2 controllers are delegated to this account/); + // Another cause keeps OBSERVATION_FIX: the cause fixes are tied to their causes, not to the observation. + const undelegated = capture(); + await runDoctor(podmanEnv(), podmanDeps(undelegated.out, podmanPlan(), [], { observationFs: podmanFsWith("cpu memory\n") })); + assert.match(undelegated.text(), /⚠ podman: isolation is ASSERTED by this account's Podman setup, not enforced: the pids cgroup controller is not delegated to this account's systemd user manager \(user@1234\.service\), so a job cannot have that bound: measured for cpu, Podman 5\.8\.1 refuses to start a container carrying --cpus \(exit 126, "controller `cpu` is not available"\)\n {4}→ Turn on linger for the worker account/); + // A running manager with the controllers: the ✓ line, listing the MANAGER's controllers, whatever podman info's + // caller-cgroup list says (measured E2a: `memory pids` inside a user unit, cpu applied anyway). + const session = capture(); + await runDoctor(podmanEnv(), podmanDeps(session.out, podmanPlan({ info: PODMAN_INFO({ cgroupControllers: ["memory", "pids"] }) }), [])); + assert.doesNotMatch(session.text(), /user manager/); + assert.match(session.text(), /✓ podman: cgroup v2 controllers are delegated to this account \(cpuset, cpu, io, memory, pids\), so a job's pid, memory and cpu bounds are applied\n/); + // --live: pids.max and memory.max read back `max` (measured E1), and the failure's fix names the cause, not docker's + // delegation, opening its own sentence with a capital. + const live = capture(); + const unbounded = LIVE_STATUS("1234").replace("pids.max:512", "pids.max:max").replace("memory.max:4294967296", "memory.max:max"); + const plan = { ...podmanLiveOk(), [`podman exec ${LIVE_ID} sh -c cat /proc/1/status`]: { code: 0, output: unbounded }, ...podmanPlan() }; + assert.equal(await runDoctor(liveEnv({ PI_BACKENDS: "podman" }), { ...podmanDeps(live.out, plan, [], noManager), live: true, liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n", ...instantClock() }), 1); + assert.match(live.text(), /✗ read back on podman: isolation does NOT hold -- declared enforced, observed: [^\n]*\n {4}→ the runtime did not apply a bound the worker passes: on rootless Podman that is the account's systemd user manager not running, Podman not reaching it, or a controller not delegated to it \(the "isolation is ASSERTED" line above names which, when the static read sees it\) -- jobs on this host do not have the boundary the table declares\. Turn on linger for the worker's account: `sudo loginctl enable-linger `/); + assert.doesNotMatch(live.text(), /on rootless podman, cgroup delegation/, "not docker's words with the name swapped"); +}); + +test("with linger off, doctor's ✓ for the bounds says it lasts only as long as a login session does (#453 gate)", async () => { + // Measured (gate 456, extra-login-session-sudo-doctor.txt): linger off, another login session of the account open, a + // `sudo -iu` doctor saw the manager running and gave ✓; the manager stops with the last session. + const run = async (linger) => { + const { out, text } = capture(); + const calls = []; + const plan = { ...podmanPlan(), ...(linger === null ? {} : { "loginctl show-user op -p Linger": { code: 0, output: `Linger=${linger}\n` } }) }; + await runDoctor(podmanEnv(), podmanDeps(out, plan, calls, { userName: () => "op" })); + return { text: text(), calls }; + }; + const off = await run("no"); + assert.match(off.text, /✓ podman: cgroup v2 controllers are delegated to this account \(cpuset, cpu, io, memory, pids\), so a job's pid, memory and cpu bounds are applied\n⚠ podman: linger is off for op, so the user manager above runs only while a login session of the account is open: when the last one ends, a worker run as a user service stops with it, and one run as a system unit with User= runs on with no bounds \(refused per job under an isolation floor\)\n {4}→ sudo loginctl enable-linger op -- it keeps the manager running with no one logged in, which is what a service needs\n/); + assert.ok(off.calls.some((c) => c.cmd === "loginctl" && c.args.join(" ") === "show-user op -p Linger"), "asked as up asks it"); + assert.doesNotMatch((await run("yes")).text, /linger is off/); + assert.doesNotMatch((await run(null)).text, /linger is off/, "an unanswered loginctl says nothing"); +}); + +test("the ladder's doctor step judges the service's .env: the provider key's presence (never its value) and VALKEY_URL, with the source said (#453 gate 2)", async () => { + // The exec's ladder-4.txt: step 2 put the key and the Valkey URL in .env, and doctor's shell has neither. + const cwd = scaffoldedCwd(); + writeFileSync(join(cwd, ".env"), "PI_BACKENDS=podman\nPI_EGRESS=0\nANTHROPIC_API_KEY=sk-ant-never-printed\nVALKEY_URL=redis://worker:hunter2@127.0.0.1:16456/1\n"); + const probed = []; + const { out, text } = capture(); + const { PI_BACKENDS: _b, PI_EGRESS: _e, ANTHROPIC_API_KEY: _k, ...shell } = podmanEnv(); + await runDoctor(shell, { ...podmanDeps(out, podmanPlan(), []), cwd, platform: "linux", probeValkey: async (url) => (probed.push(url), true), readEnvFile: (path) => readFileSync(path, "utf8") }); + const envPath = join(cwd, ".env").replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); + assert.match(text(), new RegExp(`✓ Provider key set \\(anthropic: ANTHROPIC_API_KEY\\) -- ANTHROPIC_API_KEY read from ${envPath}, as the service reads it; this shell does not set it\\n`)); + assert.match(text(), new RegExp(`✓ Valkey reachable \\(redis://127\\.0\\.0\\.1:16456/1\\) -- VALKEY_URL read from ${envPath}, as the service reads it; this shell does not set it\\n`)); + assert.deepEqual(probed, ["redis://worker:hunter2@127.0.0.1:16456/1"], "the service's URL is the one probed"); + for (const secret of ["sk-ant-never-printed", "hunter2"]) assert.ok(!text().includes(secret), `${secret} must never be printed`); + // This shell's own values win, and then nothing is said about the file. + const own = capture(); + await runDoctor({ ...shell, ANTHROPIC_API_KEY: "sk-shell", VALKEY_URL: "redis://127.0.0.1:6379" }, { ...podmanDeps(own.out, podmanPlan(), []), cwd, platform: "linux", probeValkey: async () => true, readEnvFile: (path) => readFileSync(path, "utf8") }); + assert.match(own.text(), /✓ Provider key set \(anthropic: ANTHROPIC_API_KEY\)\n/); + assert.match(own.text(), /✓ Valkey reachable \(redis:\/\/127\.0\.0\.1:6379\)\n/); +}); + +test("doctor: a PAUSED shipped proxy's fix is docker unpause, as up offers; a RUNNING stale one gets no ✓ line, and the canary through it blames the proxy, not the allowlist (#453 exec round 3)", async () => { + const paused = capture(); + await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(paused.out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "paused"), "gh auth status": { code: 0, output: ghStatusOutput } }))); + assert.match(paused.text(), /✗ Egress proxy is paused \(pi-dispatch-egress-proxy\)\n {4}→ docker unpause pi-dispatch-egress-proxy {2}-- as `pi-dispatch up` offers;/); + // A running proxy made from another image, the canary's provider probe failing through it (case6-docker-stale-imagerun.txt). + const stale = capture(); + await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(stale.out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "running", { image: "docker.io/library/alpine:3.20", entrypoint: [], cmd: ["sleep", "100000"] }), "docker run --rm --name pi-dispatch-egress-probe-provider": 3, "gh auth status": { code: 0, output: ghStatusOutput } }))); + assert.doesNotMatch(stale.text(), /✓ Egress proxy running/, "never healthy right before the ✗"); + assert.match(stale.text(), /✗ Egress proxy is running but is not this deployment's \(pi-dispatch-egress-proxy\): it was created from docker\.io\/library\/alpine:3\.20/); + assert.match(stale.text(), /⚠ Egress policy does NOT reach the provider \(api\.anthropic\.com\)\n {4}→ the canary ran through pi-dispatch-egress-proxy, which is not this deployment's proxy \(above\), so this says nothing about egress-allowlist\.conf: `pi-dispatch up` from the deployment folder offers to replace it/); + assert.doesNotMatch(stale.text(), /add api\.anthropic\.com to egress-allowlist\.conf/); + // A current proxy keeps its ✓ and the allowlist fix. + const current = capture(); + await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(current.out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "running"), "docker run --rm --name pi-dispatch-egress-probe-provider": 3, "gh auth status": { code: 0, output: ghStatusOutput } }))); + assert.match(current.text(), /✓ Egress proxy running \(pi-dispatch-egress-proxy\)/); + assert.match(current.text(), /add api\.anthropic\.com to egress-allowlist\.conf/); +}); + +test("doctor: a STOPPED shipped proxy that is not this deployment's gets up's replace offer as its fix, never compose (#453 gate 2)", async () => { + const folder = tempDir("pi-proxy-stopped-"); + mkdirSync(join(folder, "deploy")); + writeFileSync(join(folder, "deploy/egress-proxy.conf"), "http_port 3128\n"); + writeFileSync(join(folder, "egress-allowlist.conf"), "api.anthropic.com\n"); + const other = tempDir("pi-proxy-stopped-other-"); + writeFileSync(join(other, "egress-allowlist.conf"), "evil.example\n"); + const { out, text } = capture(); + await runDoctor(ghEnv({ PI_EGRESS: "1" }), { ...ghDeps(out, egressPlan({ [PROXY_KEY]: proxyAnswer("none", "exited", { cwd: folder, allowlist: join(other, "egress-allowlist.conf") }), "gh auth status": { code: 0, output: ghStatusOutput } })), cwd: folder, proxyFilesExist: () => true }); + assert.match(text(), /✗ Egress proxy is stopped \(pi-dispatch-egress-proxy\)\n {4}→ it is not this deployment's either \(its \/etc\/pi-dispatch\/allowlist\.conf is [^)]*\): `pi-dispatch up` from the deployment folder offers to replace it with the shipped one \(docker rm -f pi-dispatch-egress-proxy, then the shipped run\) -- /); + assert.doesNotMatch(text(), /✗ Egress proxy is stopped \(pi-dispatch-egress-proxy\)\n {4}→ docker compose/); +}); + +test("a URL doctor prints has its password blanked and its query dropped, or is not printed at all (#453 gate 3)", () => { + assert.equal(urlShown("redis://:p@ss@host"), "redis://host", "the userinfo is never printed, whatever @ it holds"); + assert.equal(urlShown("redis://127.0.0.1:6379?password=secret"), "redis://127.0.0.1:6379"); + assert.equal(urlShown("redis://:p@ss@host:6380/2?password=secret&token=t"), "redis://host:6380/2"); + assert.equal(urlShown("redis://worker:hunter2@127.0.0.1:16456/1"), "redis://127.0.0.1:16456/1", "a username can be a token too"); + assert.equal(urlShown("redis:///2"), ""); + assert.equal(urlShown("redis://127.0.0.1:6379"), "redis://127.0.0.1:6379", "nothing to hide, printed as written"); + assert.equal(urlShown("not a url with pass:word@"), ""); +}); + +test("doctor's .env reads pass ONE allowlist and ONE loader mapping (#453 gate 3)", async () => { + assert.deepEqual([...SERVICE_ENV_KEYS], ["PI_BACKENDS", "PI_EGRESS", "PI_EGRESS_PROXY", "VALKEY_URL", "PI_PROVIDER"]); + assert.deepEqual(serviceEnvKeys(["PI_JOB_IMAGE", "PI_ENV_SETUP", "VALKEY_URL", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"], ["ANTHROPIC_API_KEY"]), ["VALKEY_URL", "ANTHROPIC_API_KEY"]); + assert.deepEqual(["linux", "win32", "darwin", "freebsd"].map(serviceEnvLoader), ["systemd", "cmd", "shell", "shell"]); + // The venue read uses that mapping too: on freebsd a sourcing shell reads `export PI_BACKENDS=podman` as an assignment. + const cwd = scaffoldedCwd(); + writeFileSync(join(cwd, ".env"), "export PI_BACKENDS=podman\n"); + const { out, text } = capture(); + const { PI_BACKENDS: _b, ...shell } = podmanEnv(); + await runDoctor(shell, { ...podmanDeps(out, podmanPlan(), []), cwd, platform: "freebsd", readEnvFile: (path) => readFileSync(path, "utf8") }); + assert.match(text(), /✓ venue keys read from [^\n]* \(PI_BACKENDS="podman"\)/); }); +test("fleet host names and zones doctor prints from Valkey have their control bytes blanked (#453 gate 3)", async () => { + const ESC = String.fromCharCode(27); + const { out, text } = capture(); + await runDoctor(ghEnv({ PI_WORKER_NAME: "mini1" }), { ...ghDeps(out, green), readHosts: async () => ({ hosts: [{ name: "mini1" }, { name: `evil${ESC}[2Jhost`, tz: `Zone${ESC}]0;x` }] }) }); + assert.ok(!text().includes(ESC), "no escape byte reaches the terminal"); + assert.match(text(), /Fleet: 2 workers \(evil \[2Jhost, mini1\)/); + // C1 controls too (U+0080 to U+009F: U+009B is a one-byte CSI to some terminals). + const c1 = capture(); + await runDoctor(ghEnv({ PI_WORKER_NAME: "mini1" }), { ...ghDeps(c1.out, green), readHosts: async () => ({ hosts: [{ name: "mini1" }, { name: `evil${String.fromCharCode(0x9b)}2Jhost` }] }) }); + assert.ok(!c1.text().includes(String.fromCharCode(0x9b))); + assert.match(c1.text(), /Fleet: 2 workers \(evil 2Jhost, mini1\)/); +}); + +test("doctor reads the venue keys from the deployment's .env as up does, says so, reads the file once, and refuses a disagreement (#453 gate)", async () => { + // The exec's ladder step 4 (gate 456, case6-step4-doctor.txt): PI_BACKENDS=podman in .env, nothing in the shell, and + // doctor judged docker. + const cwd = scaffoldedCwd(); + writeFileSync(join(cwd, ".env"), "PI_BACKENDS=podman\nPI_EGRESS=0\n"); + const asked = []; + const { out, text } = capture(); + const { PI_BACKENDS: _b, PI_EGRESS: _e, ...shell } = podmanEnv(); + await runDoctor(shell, { ...podmanDeps(out, podmanPlan(), []), cwd, platform: "linux", readEnvFile: (path) => (asked.push(path), readFileSync(path, "utf8")) }); + assert.match(text(), new RegExp(`✓ venue keys read from ${join(cwd, ".env").replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")} \\(PI_BACKENDS="podman", PI_EGRESS="0"\\), as the service reads them: this shell does not set them\\n`)); + assert.match(text(), /✓ Jobs run on: podman/); + assert.doesNotMatch(text(), /Docker daemon reachable/); + assert.deepEqual(asked, [join(cwd, ".env")], "one file, read once"); + // This shell set differently: ✗, and doctor judges the shell's values. + const conflict = capture(); + assert.equal(await runDoctor({ ...shell, PI_BACKENDS: "local" }, { ...ghDeps(conflict.out, green), cwd, platform: "linux", readEnvFile: (path) => readFileSync(path, "utf8") }), 1); + assert.match(conflict.text(), new RegExp(`✗ which venue this deployment runs is unknown: PI_BACKENDS is "local" in this shell and "podman" in ${join(cwd, ".env").replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")}\\. doctor would judge the shell's venue while the service runs the file's\\.`)); + // A .env that is there and cannot be read is said, as up says it (gate round 2); an absent one is not. + const dirEnv = tempDir("pi-venue-dir-env-"); + mkdirSync(join(dirEnv, ".env")); + const unread = capture(); + await runDoctor(shell, { ...podmanDeps(unread.out, podmanPlan(), []), cwd: dirEnv, fileExists: existsSync, platform: "linux" }); + assert.match(unread.text(), new RegExp(`⚠ ${join(dirEnv, ".env").replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")} could not be read \\(not a regular file\\), so PI_BACKENDS, PI_EGRESS and PI_EGRESS_PROXY come from this shell alone\n`)); + const absent = capture(); + await runDoctor(shell, { ...podmanDeps(absent.out, podmanPlan(), []), cwd: tempDir("pi-venue-no-env-"), fileExists: existsSync, platform: "linux" }); + assert.doesNotMatch(absent.text(), /come from this shell alone/); + // A shell that agrees in meaning is no disagreement, and no source line is needed for keys the shell set. + const agree = capture(); + await runDoctor({ ...shell, PI_BACKENDS: " podman", PI_EGRESS: "0" }, { ...podmanDeps(agree.out, podmanPlan(), []), cwd, platform: "linux", readEnvFile: (path) => readFileSync(path, "utf8") }); + assert.doesNotMatch(agree.text(), /which venue this deployment runs is unknown|venue keys read from/); +}); + + + test("the pinning and ephemeral read-backs on podman name podman in what they could not read (#354)", async () => { const { out, text } = capture(); const plan = { ...podmanLiveOk(), "podman run --name=pi-dispatch-live-pin-": { code: 125, output: "Error: something new\n" }, "podman ps -a --no-trunc --filter id=": { code: 125, output: "" }, ...podmanPlan() }; @@ -6973,7 +7281,7 @@ const DOCKER_CANARY_PIN = { "docker rm -f pi-dispatch-egress-probe-unlisted-4242", "docker network disconnect -f pi-dispatch-egress-doctor-4242 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-4242", - "docker inspect --format={{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} pi-dispatch-egress-proxy", + 'docker inspect --format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}} pi-dispatch-egress-proxy', "docker network create --internal pi-dispatch-egress-doctor-7", "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", @@ -7040,7 +7348,7 @@ const DOCKER_CANARY_PIN = { "docker rm -f pi-dispatch-egress-probe-unlisted-4242", "docker network disconnect -f pi-dispatch-egress-doctor-4242 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-4242", - "docker inspect --format={{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} pi-dispatch-egress-proxy", + 'docker inspect --format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}} pi-dispatch-egress-proxy', "docker network create --internal pi-dispatch-egress-doctor-7", "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", @@ -7107,7 +7415,7 @@ const DOCKER_CANARY_PIN = { "docker rm -f pi-dispatch-egress-probe-unlisted-4242", "docker network disconnect -f pi-dispatch-egress-doctor-4242 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-4242", - "docker inspect --format={{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} pi-dispatch-egress-proxy", + 'docker inspect --format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}} pi-dispatch-egress-proxy', "docker network create --internal pi-dispatch-egress-doctor-7", "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", @@ -7181,7 +7489,7 @@ const DOCKER_CANARY_PIN = { "docker rm -f pi-dispatch-egress-probe-unlisted-4242", "docker network disconnect -f pi-dispatch-egress-doctor-4242 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-4242", - "docker inspect --format={{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} pi-dispatch-egress-proxy", + 'docker inspect --format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}} pi-dispatch-egress-proxy', "docker network create --internal pi-dispatch-egress-doctor-7", "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", @@ -7255,7 +7563,7 @@ const DOCKER_CANARY_PIN = { "docker rm -f pi-dispatch-egress-probe-unlisted-4242", "docker network disconnect -f pi-dispatch-egress-doctor-4242 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-4242", - "docker inspect --format={{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} pi-dispatch-egress-proxy", + 'docker inspect --format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}} pi-dispatch-egress-proxy', "docker network create --internal pi-dispatch-egress-doctor-7", "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", diff --git a/worker/test/egress-proxy-state.test.mjs b/worker/test/egress-proxy-state.test.mjs new file mode 100644 index 00000000..ff51f224 --- /dev/null +++ b/worker/test/egress-proxy-state.test.mjs @@ -0,0 +1,98 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { EGRESS_PROXY_CMD, EGRESS_PROXY_ENTRYPOINT, EGRESS_PROXY_IMAGE, jobNetworksOf, parseProxyState, shippedProxyDrift } from "../src/egress-proxy-state.mjs"; + +// Issue #453, gate round 2: `PROXY_STATE_FORMAT`'s answer for the shipped proxy on a real Docker Engine (29.8.1, +// compose 5.5.1, Ubuntu, rootful), copied byte for byte from round-446/pr3/m453/raw-m453-docker-engine.txt. C1 was made +// by `docker compose -f deploy/docker-compose.yml --profile egress up -d` from the real folder /var/tmp/m453/dep; U2 by +// `up`'s own `docker run` argv from /var/tmp/m453/link, a symlink to it, whose path docker kept as given. +const MEASURED_COMPOSE = String.raw`{"status":"running","health":"starting","image":"ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029","entrypoint":["entrypoint.sh"],"cmd":["-f","/etc/squid/squid.conf","-NYC"],"mounts":[{"Type":"bind","Source":"/var/tmp/m453/dep/egress-allowlist.conf","Destination":"/etc/pi-dispatch/allowlist.conf","Mode":"ro,z","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"/var/tmp/m453/dep/deploy/egress-proxy.conf","Destination":"/etc/squid/squid.conf","Mode":"ro,z","RW":false,"Propagation":"rprivate"},{"Type":"volume","Name":"eec1ec575581eb2532f1b5e0b9ab4aa6d6c82c3c0ce70319a7eaab9d130911b1","Source":"/var/lib/docker/volumes/eec1ec575581eb2532f1b5e0b9ab4aa6d6c82c3c0ce70319a7eaab9d130911b1/_data","Destination":"/var/log/squid","Driver":"local","Mode":"","RW":true,"Propagation":""},{"Type":"volume","Name":"d6c30b792e8cff6bf529e072f53986fa3da1478104257f4b8f26b54ed3356faf","Source":"/var/lib/docker/volumes/d6c30b792e8cff6bf529e072f53986fa3da1478104257f4b8f26b54ed3356faf/_data","Destination":"/var/spool/squid","Driver":"local","Mode":"","RW":true,"Propagation":""}],"networks":{"pi-dispatch-egress-out":{"IPAMConfig":null,"Links":null,"Aliases":["pi-dispatch-egress-proxy","egress-proxy"],"DriverOpts":null,"GwPriority":0,"NetworkID":"91ee40174e9c170ffa2b46a8c0b7dc08f92be3ff65f944da2df39d9b74356a25","EndpointID":"a635380754dfc55d65df63919622965ddc8a08c85078d3c05818a7936bf52421","Gateway":"172.18.0.1","IPAddress":"172.18.0.2","MacAddress":"fa:27:f9:dc:f3:17","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DNSNames":["pi-dispatch-egress-proxy","egress-proxy","613986d80a53"]}}}`; +const MEASURED_UP_VIA_SYMLINK = String.raw`{"status":"running","health":"none","image":"ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029","entrypoint":["entrypoint.sh"],"cmd":["-f","/etc/squid/squid.conf","-NYC"],"mounts":[{"Type":"bind","Source":"/var/tmp/m453/link/egress-allowlist.conf","Destination":"/etc/pi-dispatch/allowlist.conf","Mode":"ro,z","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"/var/tmp/m453/link/deploy/egress-proxy.conf","Destination":"/etc/squid/squid.conf","Mode":"ro,z","RW":false,"Propagation":"rprivate"},{"Type":"volume","Name":"000ea052364aab9a221eef6585c91a7b9e0449433c84ec3e1cc670fec0072ff9","Source":"/var/lib/docker/volumes/000ea052364aab9a221eef6585c91a7b9e0449433c84ec3e1cc670fec0072ff9/_data","Destination":"/var/log/squid","Driver":"local","Mode":"","RW":true,"Propagation":""},{"Type":"volume","Name":"da79a826deb0e79a01605e4b5d84af808770dfef684bcde0600a613b71c09cb7","Source":"/var/lib/docker/volumes/da79a826deb0e79a01605e4b5d84af808770dfef684bcde0600a613b71c09cb7/_data","Destination":"/var/spool/squid","Driver":"local","Mode":"","RW":true,"Propagation":""}],"networks":{"pi-dispatch-egress-out":{"IPAMConfig":null,"Links":null,"Aliases":null,"DriverOpts":null,"GwPriority":0,"NetworkID":"45c8be00149c32493ead92dd2d77b3d0964a12494765dee48c4387be61968468","EndpointID":"41e07ebd39e69be2a56a86ce74f2f787749c70e3197838a1c95a97cc9528e0d5","Gateway":"172.18.0.1","IPAddress":"172.18.0.2","MacAddress":"3a:ca:04:d5:d7:b2","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DNSNames":["pi-dispatch-egress-proxy","0e66893f026a"]}}}`; +// The VM's symlink, resolved as that host resolved it. +const realpath = (p) => p.replace(/^\/var\/tmp\/m453\/link(?=\/|$)/, "/var/tmp/m453/dep"); + +test("the measured Docker Engine shapes parse, and a compose-made or up-made proxy from the real or the symlinked folder is current (#453 gate 2)", () => { + for (const [label, raw] of [["compose", MEASURED_COMPOSE], ["up via the symlink", MEASURED_UP_VIA_SYMLINK]]) { + const state = parseProxyState(raw); + assert.equal(state.status, "running", label); + assert.equal(state.image, EGRESS_PROXY_IMAGE, `${label}: docker keeps the reference as written`); + assert.deepEqual(state.entrypoint, [...EGRESS_PROXY_ENTRYPOINT], `${label}: neither compose nor up sets one, so the image's`); + assert.deepEqual(state.cmd, [...EGRESS_PROXY_CMD], label); + assert.deepEqual(state.mounts.map((m) => [m.type, m.destination]).sort(), [["bind", "/etc/pi-dispatch/allowlist.conf"], ["bind", "/etc/squid/squid.conf"], ["volume", "/var/log/squid"], ["volume", "/var/spool/squid"]], label); + assert.deepEqual(state.networks, ["pi-dispatch-egress-out"], label); + assert.deepEqual(jobNetworksOf(state), [], label); + for (const cwd of ["/var/tmp/m453/dep", "/var/tmp/m453/link"]) assert.deepEqual(shippedProxyDrift(state, { cwd, realpath }), { drift: [], unknown: null }, `${label} judged from ${cwd}`); + } + // Compose's healthcheck is in the answer; up's run carries none. + assert.equal(parseProxyState(MEASURED_COMPOSE).health, "starting"); + assert.equal(parseProxyState(MEASURED_UP_VIA_SYMLINK).health, "none"); +}); + +// Gate round 3, read only on pd-fedora (round-446/pr3/r3/raw-r3-compat-inspect.txt): root's rootful Podman 5.8.1 and its +// compose-made proxy (whose folder had since been deleted), through its Docker API (the `local` venue's route, docker +// CLI 29.7.2) and natively, with `PROXY_STATE_FORMAT`. Both render the argv as lists and the health as "". +const MEASURED_PODMAN_COMPAT = String.raw`{"status":"exited","health":"","image":"docker.io/ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029","entrypoint":["entrypoint.sh"],"cmd":["-f","/etc/squid/squid.conf","-NYC"],"mounts":[{"Type":"volume","Name":"9c75e38082d0239e57e22b039ac7c9d62e983594c530e671f476274b328f270d","Source":"/var/lib/containers/storage/volumes/9c75e38082d0239e57e22b039ac7c9d62e983594c530e671f476274b328f270d/_data","Destination":"/var/log/squid","Driver":"local","Mode":"","RW":true,"Propagation":"rprivate"},{"Type":"volume","Name":"40c34d3773e8ad982609adaaf9db1d5b803e785ec13aeffaa7b74da22768372c","Source":"/var/lib/containers/storage/volumes/40c34d3773e8ad982609adaaf9db1d5b803e785ec13aeffaa7b74da22768372c/_data","Destination":"/var/spool/squid","Driver":"local","Mode":"","RW":true,"Propagation":"rprivate"},{"Type":"bind","Source":"/home/op/pi-dispatch/egress-allowlist.conf","Destination":"/etc/pi-dispatch/allowlist.conf","Mode":"","RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"/home/op/pi-dispatch/deploy/egress-proxy.conf","Destination":"/etc/squid/squid.conf","Mode":"","RW":false,"Propagation":"rprivate"}],"networks":{"pi-dispatch-egress-out":{"IPAMConfig":null,"Links":null,"Aliases":["pi-dispatch-egress-proxy","egress-proxy","3255b77c3ee3"],"DriverOpts":null,"GwPriority":0,"NetworkID":"65d36765319ca45a39cc062b1daf1fa193a8d263ad380f4a29195e6687ddab18","EndpointID":"","Gateway":"","IPAddress":"","MacAddress":"","IPPrefixLen":0,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DNSNames":null}}}`; +const MEASURED_PODMAN_NATIVE = String.raw`{"status":"exited","health":"","image":"docker.io/ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029","entrypoint":["entrypoint.sh"],"cmd":["-f","/etc/squid/squid.conf","-NYC"],"mounts":[{"Type":"volume","Name":"9c75e38082d0239e57e22b039ac7c9d62e983594c530e671f476274b328f270d","Source":"/var/lib/containers/storage/volumes/9c75e38082d0239e57e22b039ac7c9d62e983594c530e671f476274b328f270d/_data","Destination":"/var/log/squid","Driver":"local","Mode":"","Options":["nodev","exec","nosuid","rbind"],"RW":true,"Propagation":"rprivate"},{"Type":"volume","Name":"40c34d3773e8ad982609adaaf9db1d5b803e785ec13aeffaa7b74da22768372c","Source":"/var/lib/containers/storage/volumes/40c34d3773e8ad982609adaaf9db1d5b803e785ec13aeffaa7b74da22768372c/_data","Destination":"/var/spool/squid","Driver":"local","Mode":"","Options":["nodev","exec","nosuid","rbind"],"RW":true,"Propagation":"rprivate"},{"Type":"bind","Source":"/home/op/pi-dispatch/egress-allowlist.conf","Destination":"/etc/pi-dispatch/allowlist.conf","Driver":"","Mode":"","Options":["rbind"],"RW":false,"Propagation":"rprivate"},{"Type":"bind","Source":"/home/op/pi-dispatch/deploy/egress-proxy.conf","Destination":"/etc/squid/squid.conf","Driver":"","Mode":"","Options":["rbind"],"RW":false,"Propagation":"rprivate"}],"networks":{"pi-dispatch-egress-out":{"EndpointID":"","Gateway":"","IPAddress":"","IPPrefixLen":0,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"MacAddress":"","NetworkID":"65d36765319ca45a39cc062b1daf1fa193a8d263ad380f4a29195e6687ddab18","DriverOpts":null,"IPAMConfig":null,"Links":null,"Aliases":["pi-dispatch-egress-proxy","egress-proxy","3255b77c3ee3"]}}}`; +// Gate round 3, adversary (gate456-adv/r3/raw-r3-podman49.txt): Podman 4.9.3's NATIVE inspect renders +// `.Config.Entrypoint` as a string, "entrypoint.sh" for the squid image's and "" for none. +const PODMAN_49_ENTRYPOINT = '{"entrypoint":"entrypoint.sh","cmd":["-f","/etc/squid/squid.conf","-NYC"]}'; +const PODMAN_49_NO_ENTRYPOINT = '{"entrypoint":"","cmd":["/bin/sh"]}'; + +test("the argv is normalised whatever the runtime's spelling: Podman 4.9's string, its empty string, docker's null (#453 gate 3)", () => { + const shape = (over) => JSON.stringify({ ...JSON.parse(MEASURED_COMPOSE), ...JSON.parse(over) }); + const v49 = parseProxyState(shape(PODMAN_49_ENTRYPOINT)); + assert.deepEqual(v49.entrypoint, ["entrypoint.sh"]); + assert.deepEqual(shippedProxyDrift(v49, { cwd: "/var/tmp/m453/dep", realpath }), { drift: [], unknown: null }, "a correctly shipped proxy on Podman 4.9 is current"); + assert.deepEqual(parseProxyState(shape(PODMAN_49_NO_ENTRYPOINT)).entrypoint, []); + assert.deepEqual(parseProxyState(shape('{"entrypoint":null,"cmd":"squid -N"}')).entrypoint, []); + assert.deepEqual(parseProxyState(shape('{"entrypoint":null,"cmd":"squid -N"}')).cmd, ["squid -N"]); + assert.equal(parseProxyState(shape('{"entrypoint":7}')).entrypoint, null, "a shape no runtime renders is no argv, and stale"); +}); + +test("Podman's Docker API and native inspect, measured: the shipped proxy's shape, and its deleted folder is stale on Linux (#453 gate 3)", () => { + for (const [label, raw] of [["compat", MEASURED_PODMAN_COMPAT], ["native", MEASURED_PODMAN_NATIVE]]) { + const state = parseProxyState(raw); + assert.equal(state.status, "exited", label); + assert.equal(state.health, "none", `${label}: "" is no healthcheck`); + assert.deepEqual([state.entrypoint, state.cmd], [[...EGRESS_PROXY_ENTRYPOINT], [...EGRESS_PROXY_CMD]], label); + // Judged where the folder exists (its paths resolved as a host holding it would), it is current. + assert.deepEqual(shippedProxyDrift(state, { cwd: "/home/op/pi-dispatch", realpath: (p) => p }), { drift: [], unknown: null }, label); + // Judged on that Linux host after the folder was deleted: gone, so stale, never unknown. + const gone = shippedProxyDrift(state, { + cwd: "/home/op/pi-dispatch", + platform: "linux", + realpath: (p) => { + throw Object.assign(new Error(`ENOENT: ${p}`), { code: "ENOENT" }); + }, + }); + assert.equal(gone.unknown, null, label); + assert.ok(gone.drift.includes("its /etc/pi-dispatch/allowlist.conf is /home/op/pi-dispatch/egress-allowlist.conf, which does not exist on this host"), `${label}: ${gone.drift.join(" | ")}`); + } +}); + +test("the same measured proxy judged from another folder, and where a source cannot be resolved: unknown only for a VM path or a macOS/Windows host (#453 gates 2 and 3)", () => { + const state = parseProxyState(MEASURED_COMPOSE); + const elsewhere = shippedProxyDrift(state, { cwd: "/srv/other", realpath }); + assert.deepEqual(elsewhere.drift, ["its /etc/squid/squid.conf is /var/tmp/m453/dep/deploy/egress-proxy.conf, not /srv/other/deploy/egress-proxy.conf", "its /etc/pi-dispatch/allowlist.conf is /var/tmp/m453/dep/egress-allowlist.conf, not /srv/other/egress-allowlist.conf"]); + const nothing = (p) => { + throw Object.assign(new Error(`ENOENT: ${p}`), { code: "ENOENT" }); + }; + for (const platform of ["darwin", "win32"]) { + const off = shippedProxyDrift(state, { cwd: "/var/tmp/m453/dep", platform, realpath: nothing }); + assert.deepEqual(off.drift, [], `${platform}: unknown is never stale`); + assert.match(off.unknown, /^\/var\/tmp\/m453\/dep\/deploy\/egress-proxy\.conf, \/var\/tmp\/m453\/dep\/egress-allowlist\.conf are paths this host cannot resolve/, platform); + } + const desktop = { ...state, mounts: state.mounts.map((m) => (m.type === "bind" ? { ...m, source: `/host_mnt${m.source}` } : m)) }; + assert.match(shippedProxyDrift(desktop, { cwd: "/var/tmp/m453/dep", platform: "linux", realpath: nothing }).unknown, /^\/host_mnt\/var\/tmp\/m453\/dep\/deploy\/egress-proxy\.conf, \/host_mnt\/var\/tmp\/m453\/dep\/egress-allowlist\.conf are paths /, "a Desktop VM prefix is unknown even judged as Linux"); + const wsl = { ...state, mounts: state.mounts.map((m) => (m.type === "bind" ? { ...m, source: `/run/desktop/mnt/host/c${m.source}` } : m)) }; + assert.ok(shippedProxyDrift(wsl, { cwd: "/var/tmp/m453/dep", platform: "linux", realpath: nothing }).unknown); + // Each bind on its own (round-cap re-review): one under a VM prefix is unknown, the other gone bind and an extra mount + // are still drift. + const mixed = { ...state, mounts: [...state.mounts.map((m) => (m.destination === "/etc/squid/squid.conf" ? { ...m, source: `/host_mnt${m.source}` } : m)), { type: "bind", source: "/host_mnt/tmp/open.conf", destination: "/tmp/open.conf" }] }; + const judgedMixed = shippedProxyDrift(mixed, { cwd: "/var/tmp/m453/dep", platform: "linux", realpath: nothing }); + assert.match(judgedMixed.unknown, /^\/host_mnt\/var\/tmp\/m453\/dep\/deploy\/egress-proxy\.conf is a path this host cannot resolve/); + assert.deepEqual(judgedMixed.drift, ["its /etc/pi-dispatch/allowlist.conf is /var/tmp/m453/dep/egress-allowlist.conf, which does not exist on this host", "it has a bind at /tmp/open.conf (from /host_mnt/tmp/open.conf) that the shipped proxy does not"]); + const linuxGone = shippedProxyDrift(state, { cwd: "/var/tmp/m453/dep", platform: "linux", realpath: nothing }); + assert.equal(linuxGone.unknown, null, "on Linux a source that does not resolve is gone"); + assert.equal(linuxGone.drift.length, 2); +}); diff --git a/worker/test/egress.test.mjs b/worker/test/egress.test.mjs index 513f61fc..ea4e0b47 100644 --- a/worker/test/egress.test.mjs +++ b/worker/test/egress.test.mjs @@ -73,19 +73,41 @@ test("an UNARMED preflight admits everything and spawns NOTHING", async () => { test("a running proxy is admitted, and costs exactly ONE spawn", async () => { const calls = []; - const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn(calls, { inspect: 0, info: 0 }, "true\n") }); + const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn(calls, { inspect: 0, info: 0 }, "running\n") }); assert.deepEqual(await preflight({}), { ok: true, proxy: DEFAULT_EGRESS_PROXY }); // The `docker info` disambiguation runs ONLY on the failure path: every job pays this gate, so the // happy path must not pay for the diagnosis of a case it is not in. assert.equal(calls.length, 1, "the happy path does not probe the daemon a second time"); - assert.deepEqual(calls[0].args, ["inspect", "--format={{.State.Running}}", DEFAULT_EGRESS_PROXY]); + assert.deepEqual(calls[0].args, ["inspect", "--format={{.State.Status}}", DEFAULT_EGRESS_PROXY]); }); test("a proxy the daemon knows but has STOPPED is policy, not infra", async () => { - const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn([], { inspect: 0, info: 0 }, "false\n") }); + const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn([], { inspect: 0, info: 0 }, "exited\n") }); assert.deepEqual(await preflight({}), { proxyStopped: DEFAULT_EGRESS_PROXY }); }); +test("a PAUSED, exited or dead proxy refuses the job: Status decides, never the Running boolean, which docker reports true for paused (#453)", async () => { + // `created` among them: a container created and never started stays so (measured, Docker Engine 29.8.1, ExitCode 127). + for (const status of ["paused", "exited", "dead", "created"]) { + const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn([], { inspect: 0, info: 0 }, `${status}\n`) }); + assert.deepEqual(await preflight({}), { proxyStopped: DEFAULT_EGRESS_PROXY }, status); + } +}); + +test("EVERY other state is a retry naming it, never a refusal: restarting, created, stopping, removing, podman's stopped, a word no runtime prints yet (#453 gate 3)", async () => { + for (const status of ["restarting", "stopping", "removing", "initialized", "stopped", "configured", "somethingnew"]) { + const calls = []; + const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn(calls, { inspect: 0, info: 0 }, `${status}\n`) }); + assert.deepEqual(await preflight({}), { unavailable: DEFAULT_EGRESS_PROXY, state: status }, status); + assert.equal(calls.length, 1, `${status}: no second probe, the status alone decides`); + } + // An empty answer, or one carrying a control byte, is named "unreported", never echoed. + for (const junk of ["", `run${String.fromCharCode(27)}[2Jning`]) { + const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn([], { inspect: 0, info: 0 }, `${junk}\n`) }); + assert.deepEqual(await preflight({}), { unavailable: DEFAULT_EGRESS_PROXY, state: "unreported" }, JSON.stringify(junk)); + } +}); + test("a proxy the daemon does not have is POLICY -- disambiguated positively, never by stderr text", async () => { const calls = []; const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn(calls, { inspect: 1, info: 0 }) }); @@ -103,9 +125,11 @@ test("a daemon that does not answer is INFRA, so a blip never becomes a permanen assert.deepEqual(await gone({}), { unavailable: DEFAULT_EGRESS_PROXY }); }); -test("a proxy whose inspect returns junk is treated as NOT running, which is the safe direction", async () => { +test("a proxy whose inspect returns junk is NOT running, and retried rather than refused: a word nothing reads is not a stopped proxy (#453 gate 3)", async () => { + // The simple rule: only running admits, only paused, exited and dead refuse. Junk admits nothing (the safe + // direction for the money gate) and is retried, pre-spend, with the state named as unreported. const preflight = makeEgressPreflight({ armed: true, spawnFn: fakeSpawn([], { inspect: 0, info: 0 }, "\n") }); - assert.deepEqual(await preflight({}), { proxyStopped: DEFAULT_EGRESS_PROXY }); + assert.deepEqual(await preflight({}), { unavailable: DEFAULT_EGRESS_PROXY, state: "unreported" }); }); test("createJobNetwork passes --internal at CREATE time, so there is no window with a route out", async () => { diff --git a/worker/test/init.test.mjs b/worker/test/init.test.mjs index 5b1e3c3b..bc3daaa4 100644 --- a/worker/test/init.test.mjs +++ b/worker/test/init.test.mjs @@ -91,3 +91,103 @@ test("init never overwrites an edited allowlist", () => { runInit(dir, { out: () => {} }); assert.equal(readFileSync(join(dir, "egress-allowlist.conf"), "utf8"), "example.com\n", "create-only, like every other scaffold here"); }); + +// --- issue #453: the next steps follow the venue ------------------------------------------------------------------------ + +// Byte for byte what init printed before #453, and still prints for every venue set that includes `local`. +const DOCKER_NEXT = ` +Next: + 1. docker pull ghcr.io/edgehero/pi-job:latest && docker tag ghcr.io/edgehero/pi-job:latest pi-job:latest + # the prebuilt job image (or build image/Dockerfile) + 2. docker compose -f deploy/docker-compose.yml up -d # the durable queue (Valkey) + 3. edit .env # set ANTHROPIC_API_KEY (or your provider's key) + 4. pi-dispatch doctor # verify Docker, Valkey, image, and key + 5. pi-dispatch worker # drain the queue + +Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch + (or let the panel do all of the above: /dispatch setup walks these steps with a consent per action) +`; + +// The podman ladder, docs/podman.md "Setup" in the order a fresh folder needs it, linger named up front (gate 456). +const PODMAN_NEXT = ` +Next (the podman venue; run these as the worker's own account. First set the account up as docs/podman.md "Setup" +steps 1-4 say, linger included: sudo loginctl enable-linger , without which a job gets no bounds): + 1. podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest + # the prebuilt job image, in this account's own store + 2. edit .env # PI_BACKENDS=podman, and ANTHROPIC_API_KEY (or your provider's key) + 3. pi-dispatch up # Valkey and the egress proxy as Quadlet units in your user manager + 4. pi-dispatch doctor # verify Podman, Valkey, image, and key + 5. pi-dispatch service install # the worker as a user service, after those units (also installs them) + 6. pi-dispatch doctor --live # read the bounds, egress and job user back off real containers + +Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch + (or let the panel do all of the above: /dispatch setup walks these steps with a consent per action) +`; + +// Off Linux the podman venue refuses the host, so there is no ladder to print. +const PODMAN_OFF_LINUX = ` +Next: PI_BACKENDS lists only the podman venue, which runs on Linux alone: on this host the worker refuses it +(podman-platform), so there are no podman steps to run here. Run this deployment on a Linux host, or add \`local\` to +PI_BACKENDS to run jobs on Docker here (then run \`pi-dispatch init\` again for those steps). +`; + +function nextFor({ envFile, env, venues, platform = "linux" } = {}) { + const dir = tmp(); + writeFileSync(join(dir, ".env.example"), "ANTHROPIC_API_KEY=\n"); + if (envFile !== undefined) writeFileSync(join(dir, ".env"), envFile); + const { out, text } = capture(); + assert.equal(runInit(dir, { out, env, venues, platform }), 0); + return { text: text(), dir }; +} + +test("init's next steps: the docker text byte for byte wherever `local` is a venue (#453)", () => { + for (const [label, opts] of [ + ["no env, fresh .env", {}], + ["shell PI_BACKENDS unset, .env without it", { envFile: "ANTHROPIC_API_KEY=\n" }], + ["local,podman in .env", { envFile: "PI_BACKENDS=local,podman\n" }], + ["podman,local in the shell", { env: { PI_BACKENDS: "podman,local" } }], + ["'local, podman' in the shell", { env: { PI_BACKENDS: "local, podman" } }], + ["up's venues", { venues: { localUsed: true, podmanUsed: true, podmanDefault: false }, env: { PI_BACKENDS: "podman" } }], + ["off Linux with local listed", { env: { PI_BACKENDS: "local,podman" }, platform: "darwin" }], + ]) { + const { text } = nextFor(opts); + assert.ok(text.endsWith(DOCKER_NEXT), `${label}:\n${text}`); + assert.doesNotMatch(text, /podman pull/, label); + } +}); + +test("init's next steps: the podman ladder when podman is the only venue, from the shell, the .env or up (#453)", () => { + for (const [label, opts] of [ + ["the shell", { env: { PI_BACKENDS: "podman" } }], + ["the shell, spaced", { env: { PI_BACKENDS: " podman " } }], + ["the deployment .env", { envFile: "ANTHROPIC_API_KEY=\nPI_BACKENDS=podman\n" }], + ["the .env with CRLF", { envFile: "PI_BACKENDS=podman\r\n" }], + ["the .env, quoted, which systemd unquotes", { envFile: 'PI_BACKENDS="podman"\n' }], + ["up's venues, whatever the shell says", { venues: { localUsed: false, podmanUsed: true, podmanDefault: true }, env: { PI_BACKENDS: "local" } }], + ]) { + const { text } = nextFor(opts); + assert.ok(text.endsWith(PODMAN_NEXT), `${label}:\n${text}`); + assert.doesNotMatch(text, /docker/, label); + } +}); + +test("init decides the venue exactly as up does: a disagreement, a line the loaders read differently, an unknown backend or a non-Linux host get no ladder (#453 gate)", () => { + // Gate 456's adversary harness: each of these printed the docker ladder silently. + const conflict = nextFor({ env: { PI_BACKENDS: "local" }, envFile: "PI_BACKENDS=podman\n" }); + assert.match(conflict.text, new RegExp(`\\nNext: which venue this deployment runs is unknown, so no steps are shown: PI_BACKENDS is "local" in this shell and "podman" in ${join(conflict.dir, ".env").replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")}\\. init's next steps would be for the shell's venue while the service runs the file's\\.`)); + // An empty shell value is a value: it means `local`, which the file's podman contradicts, as up reads it. + assert.match(nextFor({ env: { PI_BACKENDS: "" }, envFile: "PI_BACKENDS=podman\n" }).text, /\nNext: which venue this deployment runs is unknown, so no steps are shown: PI_BACKENDS is "" in this shell and "podman" in /); + for (const envFile of ["export PI_BACKENDS=podman\n", "PI_BACKENDS = podman\n", "PI_BACKENDS=podman # venue\n", "PI_BACKENDS=podman, local\n", "PI_BACKENDS=pod\\\nman\n"]) { + const { text } = nextFor({ envFile }); + assert.match(text, /\nNext: which venue this deployment runs is unknown, so no steps are shown: /, JSON.stringify(envFile)); + assert.doesNotMatch(text, /docker pull|podman pull/, JSON.stringify(envFile)); + } + for (const env of [{ PI_BACKENDS: "Podman" }, { PI_BACKENDS: '"podman"' }]) { + const { text } = nextFor({ env }); + assert.match(text, /\nNext: which venue this deployment runs is unknown, so no steps are shown: PI_BACKENDS names an unknown backend/, JSON.stringify(env)); + } + // Off Linux, podman alone: the venue refuses the host, so no Quadlet steps. The .env is read with that platform's + // loader, and a line it reads differently is a note, as up says it. + assert.ok(nextFor({ env: { PI_BACKENDS: "podman" }, platform: "darwin" }).text.endsWith(PODMAN_OFF_LINUX)); + assert.ok(nextFor({ envFile: "PI_BACKENDS=podman\n", platform: "darwin" }).text.endsWith(PODMAN_OFF_LINUX)); +}); diff --git a/worker/test/processor.test.mjs b/worker/test/processor.test.mjs index 3cc9d231..e860e032 100644 --- a/worker/test/processor.test.mjs +++ b/worker/test/processor.test.mjs @@ -919,6 +919,20 @@ test("a podman job's floor refusal names podman's own observations in fixed word assert.doesNotMatch(texts[0], /docker|mounts\.conf|\/home\/|unix:/, "no docker word, and none of the evidence, reaches a forge comment"); }); +test("the podman bounds comment is true for either cause, controllers or no user manager, and names neither (#453)", () => { + // Pinned whole: the observation misses when a controller is not delegated AND when no systemd user manager runs for + // the account while podman info lists every controller, so a sentence blaming the controllers was false for the + // second. The cause is evidence, and evidence stays in the operator's log. + assert.equal(OBSERVATION_COMMENT[PODMAN_BOUNDS_DELEGATED], "the worker's rootless Podman is not observed applying a container's pid, memory and cpu bounds"); + assert.doesNotMatch(OBSERVATION_COMMENT[PODMAN_BOUNDS_DELEGATED], /controller|cgroupfs|systemd|manager|session/); +}); + +test("a proxy on its way somewhere is a retry that names the proxy and its state, never the runtime, and spends nothing (#453 gate 3)", async () => { + const { deps: d, calls } = deps({ redis: fakeRedis(), egressPreflight: async () => ({ unavailable: "pi-dispatch-egress-proxy", state: "restarting" }) }); + await assert.rejects(runJob(ghJob, d), (err) => err instanceof InfraRetry && err.message === 'egress proxy "pi-dispatch-egress-proxy" is restarting, not running; the job is retried once, then failed'); + assert.ok(!calls.some((c) => c[0] === "incr"), "no budget reserved"); +}); + test("the image and egress retries name the venue's runtime: docker's words on local byte-identical, podman's on podman (#354)", async () => { // The message is the job_failed log line and BullMQ's failedReason. "docker unavailable" on a host with no docker // sends the operator after the wrong daemon. diff --git a/worker/test/sandbox.test.mjs b/worker/test/sandbox.test.mjs index 7264924e..038e366c 100644 --- a/worker/test/sandbox.test.mjs +++ b/worker/test/sandbox.test.mjs @@ -1375,7 +1375,10 @@ describe("decideSandboxJobUser on the podman venue (#429)", () => { const USER_MOUNTS = `${HOME}/.config/containers/mounts.conf`; const USER_CONF = `${HOME}/.config/containers/containers.conf`; /** A host fs: `files` maps a path to its text, everything else is ENOENT. */ - const hostFs = (files = { [USER_MOUNTS]: "" }) => { + // user@1234.service's controllers: the account's user manager running with them delegated (issue #453). + const USER_MANAGER = "/sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/cgroup.controllers"; + const hostFs = (given = { [USER_MOUNTS]: "" }) => { + const files = { [USER_MANAGER]: "cpuset cpu io memory pids\n", ...given }; const miss = (path) => { throw Object.assign(new Error(`ENOENT: ${path}`), { code: "ENOENT" }); }; @@ -1385,7 +1388,7 @@ describe("decideSandboxJobUser on the podman venue (#429)", () => { readdirSync: (path) => miss(path), }; }; - const INFO = { rootless: true, serviceIsRemote: false, selinux: true, cgroupVersion: "v2", controllers: ["cpuset", "cpu", "io", "memory", "pids"], version: "5.8.1" }; + const INFO = { rootless: true, serviceIsRemote: false, selinux: true, cgroupVersion: "v2", cgroupManager: "systemd", controllers: ["cpuset", "cpu", "io", "memory", "pids"], version: "5.8.1" }; const answered = (over = {}) => async () => ({ answered: true, info: { ...INFO, ...over } }); const base = { venue: "podman", platform: "linux", euid: 1234, egid: 1234, home: HOME, env: {}, fs: hostFs(), readInfo: answered(), imageCapabilities: async () => ({ ok: true, capabilities: ["anyUid"] }) }; const stamped = { image: "pi-job:x", jobUser: { user: "1234:1234", home: "/home/pi" } }; @@ -1425,11 +1428,11 @@ describe("decideSandboxJobUser on the podman venue (#429)", () => { test("the observations are judged against the floor exactly as a job's are", async () => { const floor = { isolation: "enforced" }; - const undelegated = await decideSandboxJobUser({ ...base, backendFloor: floor, readInfo: answered({ controllers: ["cpu"] }), manifest: stamped }); + const undelegated = await decideSandboxJobUser({ ...base, backendFloor: floor, fs: hostFs({ [USER_MOUNTS]: "", [USER_MANAGER]: "cpu\n" }), manifest: stamped }); assert.equal(undelegated.refused, "backend-floor"); assert.match(undelegated.message, /PI_BACKEND_FLOOR asks for something this host is not observed to provide/); assert.ok((await decideSandboxJobUser({ ...base, backendFloor: floor, manifest: stamped })).user, "a delegated host passes the same floor"); - assert.ok((await decideSandboxJobUser({ ...base, readInfo: answered({ controllers: ["cpu"] }), manifest: stamped })).user, "no floor, nothing observed refuses"); + assert.ok((await decideSandboxJobUser({ ...base, fs: hostFs({ [USER_MOUNTS]: "", [USER_MANAGER]: "cpu\n" }), manifest: stamped })).user, "no floor, nothing observed refuses"); // A miss that rests only on a read that did not answer is its own token, not the floor's: the fix is to make // podman answer, not to change the host. const unread = await decideSandboxJobUser({ ...base, backendFloor: floor, readInfo: async () => ({ answered: false, reason: "timeout", transient: true }), manifest: stamped }); diff --git a/worker/test/start-wiring.test.mjs b/worker/test/start-wiring.test.mjs index 11eeadc4..bf2630a2 100644 --- a/worker/test/start-wiring.test.mjs +++ b/worker/test/start-wiring.test.mjs @@ -88,7 +88,7 @@ const NO_HOST_FILES = { statSync: (p) => { throw enoent(p); }, readFileSync: (p) function PODMAN_INFO(over = {}, { answer, calls } = {}) { return async () => { calls?.push("podman info"); - return answer ?? { answered: true, info: { rootless: true, serviceIsRemote: false, selinux: false, cgroupVersion: "v2", controllers: ["cpuset", "cpu", "io", "memory", "pids"], version: "5.8.1", ...over } }; + return answer ?? { answered: true, info: { rootless: true, serviceIsRemote: false, selinux: false, cgroupVersion: "v2", cgroupManager: "systemd", controllers: ["cpuset", "cpu", "io", "memory", "pids"], version: "5.8.1", ...over } }; }; } @@ -2733,7 +2733,9 @@ test("with local blessed, the preflights are local's own members and a job namin const PODMAN_ID = { platform: "linux", release: "6.8.0-test", euid: 1234, egid: 1234, home: "/home/pdjob" }; /** The one file that earns `podmanAddsNoMounts`: this account's own mounts.conf override, empty. */ -const PODMAN_FILES = hostFiles({ "/home/pdjob/.config/containers/mounts.conf": "" }); +// The account's user manager running with the controllers delegated (issue #453): user@1234.service's cgroup. +const PODMAN_USER_MANAGER = "/sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/cgroup.controllers"; +const PODMAN_FILES = hostFiles({ "/home/pdjob/.config/containers/mounts.conf": "", [PODMAN_USER_MANAGER]: "cpuset cpu io memory pids\n" }); const PODMAN_KEYS = ["podmanVersion", "podmanRootless", "podmanJobUser", "podmanBoundsDelegated", "podmanAddsNoMounts", "podmanServiceLocal"]; test("podman unblessed: no podman info, no podman reaper, no podman bundle, and the boot line's podman keys are null (#354)", { skip }, async () => { @@ -2966,8 +2968,15 @@ test("a floor naming a podman observation refuses to BOOT on a missed one, tagge const base = { jobUserIdentity: PODMAN_ID, observationFs: PODMAN_FILES, makeAuth: async () => ({ mintToken: async () => "tok", selfId: 1, source: "gh" }), makeHost: () => fakeHost() }; const order = []; await assert.rejects( - () => runStart({ ...base, env: { PI_BACKENDS: "podman", PI_BACKEND_FLOOR: "isolation=enforced" }, readPodmanInfo: PODMAN_INFO({ controllers: ["memory", "pids"] }), order, makePodmanReaper: () => (order.push("makePodmanReaper"), async () => ({ reaped: true })) }), - (err) => err.piDispatchConfig === true && /podman: isolation=enforced/.test(err.message) && /cpu cgroup controller is not delegated/.test(err.message), + () => runStart({ ...base, env: { PI_BACKENDS: "podman", PI_BACKEND_FLOOR: "isolation=enforced" }, readPodmanInfo: PODMAN_INFO(), observationFs: hostFiles({ "/home/pdjob/.config/containers/mounts.conf": "", [PODMAN_USER_MANAGER]: "memory pids\n" }), order, makePodmanReaper: () => (order.push("makePodmanReaper"), async () => ({ reaped: true })) }), + (err) => err.piDispatchConfig === true && /podman: isolation=enforced/.test(err.message) && /cpu cgroup controller is not delegated to this account's systemd user manager/.test(err.message), + ); + assert.deepEqual(order, [], "before the reaper and the worker"); + // Issue #453: an account with no systemd user manager running, podman info still listing every controller (measured + // E1), is refused at boot under an isolation floor, tagged (exit 2), with the missing manager named. + await assert.rejects( + () => runStart({ ...base, env: { PI_BACKENDS: "podman", PI_BACKEND_FLOOR: "isolation=enforced" }, readPodmanInfo: PODMAN_INFO(), observationFs: hostFiles({ "/home/pdjob/.config/containers/mounts.conf": "" }), order }), + (err) => err.piDispatchConfig === true && /podman: isolation=enforced/.test(err.message) && /no systemd user manager is running for this account \(\/sys\/fs\/cgroup\/user\.slice\/user-1234\.slice\/user@1234\.service\/cgroup\.controllers does not exist\)/.test(err.message), ); assert.deepEqual(order, [], "before the reaper and the worker"); // Blessed but NOT the default, it is still judged: the floor is over every blessed venue. @@ -3001,18 +3010,19 @@ test("each venue's boot observations are judged for that venue alone, so local a test("the boot line carries each podman observation under its own key (#354)", { skip }, async () => { // Three different answers, so a key reading another observation's value cannot pass: bounds withheld (no cpu - // controller), mounts credited (the empty override), service local. No floor, so nothing refuses. + // controller delegated to the user manager, issue #453), mounts credited (the empty override), service local. No + // floor, so nothing refuses. const { logs } = await runStart({ env: { PI_BACKENDS: "local,podman" }, jobUserIdentity: PODMAN_ID, - observationFs: PODMAN_FILES, - readPodmanInfo: PODMAN_INFO({ controllers: ["memory", "pids"] }), + observationFs: hostFiles({ "/home/pdjob/.config/containers/mounts.conf": "", [PODMAN_USER_MANAGER]: "memory pids\n" }), + readPodmanInfo: PODMAN_INFO(), makeAuth: async () => ({ mintToken: async () => "tok", selfId: 1, source: "gh" }), makeHost: () => fakeHost(), }); const started = logs.find((l) => l.event === "worker_started"); assert.deepEqual([started.podmanBoundsDelegated, started.podmanAddsNoMounts, started.podmanServiceLocal], [false, true, true]); - const noOverride = await runStart({ env: { PI_BACKENDS: "local,podman" }, jobUserIdentity: PODMAN_ID, makeAuth: async () => ({ mintToken: async () => "tok", selfId: 1, source: "gh" }), makeHost: () => fakeHost() }); + const noOverride = await runStart({ env: { PI_BACKENDS: "local,podman" }, jobUserIdentity: PODMAN_ID, observationFs: hostFiles({ [PODMAN_USER_MANAGER]: "cpuset cpu io memory pids\n" }), makeAuth: async () => ({ mintToken: async () => "tok", selfId: 1, source: "gh" }), makeHost: () => fakeHost() }); const s2 = noOverride.logs.find((l) => l.event === "worker_started"); assert.deepEqual([s2.podmanBoundsDelegated, s2.podmanAddsNoMounts, s2.podmanServiceLocal], [true, false, true]); }); diff --git a/worker/test/up.test.mjs b/worker/test/up.test.mjs index a4d4e23f..bbef90fe 100644 --- a/worker/test/up.test.mjs +++ b/worker/test/up.test.mjs @@ -62,6 +62,7 @@ function harness({ plan = {}, listening = true, answers = [], files = {}, doctor const calls = []; const promptCalls = []; const initCalls = []; + const initOpts = []; const doctorCalls = []; const doctorOpts = []; const buf = []; @@ -100,8 +101,9 @@ function harness({ plan = {}, listening = true, answers = [], files = {}, doctor // `up` can persist a path it must never persist. logsDirPathFn: logsDirPathFn ?? ((e) => e.PI_LOGS_DIR || "/home/op/.pi-dispatch/logs"), settingsFilePathFn: settingsFilePathFn ?? ((e) => e.PI_SETTINGS_FILE || "/home/op/.pi-dispatch/settings.json"), - runInitFn: (cwd) => { + runInitFn: (cwd, opts) => { initCalls.push(cwd); + initOpts.push(opts); return 0; }, runDoctorFn: (env, opts) => { @@ -111,7 +113,7 @@ function harness({ plan = {}, listening = true, answers = [], files = {}, doctor }, ...extra, }; - return { run: () => runUp(argv, deps), deps, calls, promptCalls, initCalls, doctorCalls, doctorOpts, store, text: () => buf.join("") }; + return { run: () => runUp(argv, deps), deps, calls, promptCalls, initCalls, initOpts, doctorCalls, doctorOpts, store, text: () => buf.join("") }; } const green = { "docker version": 0, "docker image inspect": 0, "docker ps": { code: 0, output: "pi-dispatch-valkey\n" } }; @@ -152,7 +154,7 @@ test("up: declined prompts run NOTHING and the summary says skipped", async () = assert.deepEqual(h.calls.map((c) => c.args), [ ["version"], ["image", "inspect", "pi-job:latest"], - ["inspect", "--format={{.State.Running}}", "pi-dispatch-egress-proxy"], + ["inspect", PROXY_INSPECT.slice("docker inspect ".length, -" pi-dispatch-egress-proxy".length), "pi-dispatch-egress-proxy"], ]); assert.match(h.text(), /job image\s+skipped \(declined\)/); assert.match(h.text(), /valkey\s+skipped \(declined\)/); @@ -598,6 +600,32 @@ test("defaultPrompt: non-TTY stdin declines immediately without readline (the ev // --- REQ-EGRESS-ALLOWLIST: up offers the proxy, and only to a deployment that armed the policy -------- const EGRESS_ENV = { PI_PROVIDER: "anthropic", PI_EGRESS: "1" }; +// Issue #453: the shipped proxy's state is one inspect, status, image and mounts (`egress-proxy-state.mjs`). A container +// made by `up` or compose in THIS folder carries the pinned image and the two files below. +const PROXY_INSPECT = 'docker inspect --format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}} pi-dispatch-egress-proxy'; +const PINNED_SQUID = "ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029"; +// The pinned image's own entrypoint and command and its two anonymous volumes (its registry config, 2026-09-27). +const SQUID_ENTRYPOINT = ["entrypoint.sh"]; +const SQUID_CMD = ["-f", "/etc/squid/squid.conf", "-NYC"]; +const proxyState = (status, { image = PINNED_SQUID, conf = "/deploy/deploy/egress-proxy.conf", allowlist = "/deploy/egress-allowlist.conf", entrypoint = SQUID_ENTRYPOINT, cmd = SQUID_CMD, extra = [], networks = ["pi-dispatch-egress-out"], stderr } = {}) => ({ + code: 0, + output: `${JSON.stringify({ + status, + health: "none", + image, + entrypoint, + cmd, + mounts: [ + ...(conf ? [{ Type: "bind", Source: conf, Destination: "/etc/squid/squid.conf" }] : []), + ...(allowlist ? [{ Type: "bind", Source: allowlist, Destination: "/etc/pi-dispatch/allowlist.conf" }] : []), + { Type: "volume", Source: "/var/lib/docker/volumes/a1/_data", Destination: "/var/log/squid" }, + { Type: "volume", Source: "/var/lib/docker/volumes/b2/_data", Destination: "/var/spool/squid" }, + ...extra, + ], + networks: Object.fromEntries(networks.map((n) => [n, {}])), + })}\n`, + ...(stderr ? { stderr } : {}), +}); const EGRESS_NET = ["network", "create", "pi-dispatch-egress-out"]; const EGRESS_RUN = [ "run", "-d", "--name", "pi-dispatch-egress-proxy", "--restart", "unless-stopped", @@ -620,7 +648,7 @@ test("up: a deployment that turned the policy OFF is never asked about a proxy", test("up: an armed policy with the proxy already up prompts for nothing", async () => { const h = harness({ env: EGRESS_ENV, - plan: { ...green, "docker inspect --format={{.State.Running}} pi-dispatch-egress-proxy": 0 }, + plan: { ...green, [PROXY_INSPECT]: proxyState("running") }, listening: true, }); assert.equal(await h.run(), 0); @@ -631,9 +659,9 @@ test("up: an armed policy with the proxy already up prompts for nothing", async test("up: --yes starts the proxy with exactly the argv it showed", async () => { const h = harness({ env: EGRESS_ENV, - plan: { ...green, "docker inspect --format={{.State.Running}} pi-dispatch-egress-proxy": 1, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, + plan: { ...green, [PROXY_INSPECT]: 1, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, - files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n" }, + files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, argv: ["--yes"], }); await h.run(); @@ -648,7 +676,7 @@ test("up: an armed policy with NO allowlist file declines itself rather than sta // belonged, and a squid that fails confusingly. Naming the file is the fix; starting it is not. const h = harness({ env: EGRESS_ENV, - plan: { ...green, "docker inspect --format={{.State.Running}} pi-dispatch-egress-proxy": 1 }, + plan: { ...green, [PROXY_INSPECT]: 1 }, listening: true, argv: ["--yes"], }); @@ -660,9 +688,9 @@ test("up: an armed policy with NO allowlist file declines itself rather than sta test("up: a declined proxy prompt runs nothing, and the summary says what that costs", async () => { const h = harness({ env: EGRESS_ENV, - plan: { ...green, "docker inspect --format={{.State.Running}} pi-dispatch-egress-proxy": 1 }, + plan: { ...green, [PROXY_INSPECT]: 1 }, listening: true, - files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n" }, + files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, answers: ["n"], }); await h.run(); @@ -686,7 +714,7 @@ test("up on podman: --yes runs exactly the lines it showed, installs the Quadlet plan: { "podman image exists": 1, "podman pull": 0, "podman tag": 0, "podman inspect": 1, systemctl: 0, "loginctl show-user": { code: 0, output: "Linger=no\n" } }, listening: false, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n" }, + files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra(), }); assert.equal(await h.run(), 0); @@ -757,10 +785,10 @@ test("up on podman: a listener on 6379 is left alone, as on docker", async () => test("up on local,podman: the docker steps are unchanged, then podman's image and proxy; Valkey stays docker's", async () => { const h = harness({ env: { PI_BACKENDS: "local,podman" }, - plan: { ...green, "docker inspect": 0, "podman image exists": 0, "podman inspect": 1, systemctl: 0, "loginctl show-user": { code: 0, output: "Linger=yes\n" } }, + plan: { ...green, [PROXY_INSPECT]: proxyState("running"), "podman image exists": 0, "podman inspect": 1, systemctl: 0, "loginctl show-user": { code: 0, output: "Linger=yes\n" } }, listening: true, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra(), }); assert.equal(await h.run(), 0); @@ -774,15 +802,232 @@ test("up on local,podman: the docker steps are unchanged, then podman's image an assert.ok(h.calls.some((c) => c.cmd === "systemctl" && c.args.join(" ") === "--user start pi-dispatch-egress-proxy.service")); }); +// Issue #453: the docker proxy is RUNNING only when `docker inspect` prints `true` on stdout, as the podman path has read it +// since #430. An exited container also exits 0, so the exit code alone called a stopped proxy present and offered nothing. +const EGRESS_START = ["start", "pi-dispatch-egress-proxy"]; + +test("up on docker: a stopped shipped proxy is started as it is, never recreated, with exactly the argv shown (#453)", async () => { + const h = harness({ + env: EGRESS_ENV, + plan: { ...green, [PROXY_INSPECT]: proxyState("exited"), "docker start pi-dispatch-egress-proxy": 0 }, + listening: true, + files: { "/deploy/egress-allowlist.conf": "api.anthropic.com\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, + argv: ["--yes"], + }); + assert.equal(await h.run(), 0); + assert.doesNotMatch(h.text(), /Egress proxy already present/); + assert.match(h.text(), /the allowlist proxy exists but is exited\. up would start that same container \(as deploy\/docker-compose\.yml --profile egress up -d does\):\n\s+docker start pi-dispatch-egress-proxy\n/); + const mutations = h.calls.filter((c) => c.cmd === "docker" && ["start", "run", "network", "rm"].includes(c.args[0])).map((c) => c.args); + assert.deepEqual(mutations, [EGRESS_START], "one start of the existing container: no rm, no run on a taken name, no network create"); + assert.match(h.text(), /egress\s+started the exited pi-dispatch-egress-proxy/); +}); + +test("up on docker: a stopped shipped proxy, declined or failing to start, is said, and nothing else runs (#453)", async () => { + const declined = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("exited") }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, answers: ["n"] }); + assert.equal(await declined.run(), 0); + assert.ok(!declined.calls.some((c) => ["start", "run", "network"].includes(c.args[0])), "declined runs nothing"); + assert.match(declined.text(), /skipped: start it later with `docker start pi-dispatch-egress-proxy`/); + assert.match(declined.text(), /egress\s+skipped \(declined\): every job is refused pre-spend until the proxy is up/); + const failing = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("exited"), "docker start": 1 }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, argv: ["--yes"] }); + assert.equal(await failing.run(), 0, "doctor is the judge of what remains"); + assert.match(failing.text(), /✗ could not start the egress proxy; `docker rm -f pi-dispatch-egress-proxy` and re-run `pi-dispatch up` recreates it/); + assert.ok(!failing.calls.some((c) => c.args[0] === "run" || c.args[0] === "rm"), "never removes or recreates it by itself"); + // No allowlist here: declined before any offer, as for an absent proxy. + const bare = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("exited") }, listening: true, argv: ["--yes"] }); + await bare.run(); + assert.match(bare.text(), /egress-allowlist\.conf is not here/); + assert.ok(!bare.calls.some((c) => c.args[0] === "start")); +}); + +test("up on docker: a running proxy is read off stdout alone, whatever docker prints on stderr (#453)", async () => { + const h = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("running", { stderr: "WARNING: something the CLI says\n" }) }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, argv: ["--yes"] }); + assert.equal(await h.run(), 0); + assert.match(h.text(), /✓ Egress proxy already present \(pi-dispatch-egress-proxy\)/); + assert.ok(!h.calls.some((c) => ["start", "run", "network"].includes(c.args[0]))); +}); + +test("up on docker: a PI_EGRESS_PROXY container that exists but is stopped is reported, never started (#453)", async () => { + const stopped = harness({ env: { PI_EGRESS_PROXY: "my-squid" }, plan: { ...green, "docker inspect": { code: 0, output: "exited\n" } }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, argv: ["--yes"] }); + assert.equal(await stopped.run(), 0); + assert.match(stopped.text(), /✗ PI_EGRESS_PROXY names my-squid, and that container exists but is not running \(exited\)\. up starts only the shipped pi-dispatch-egress-proxy, so start my-squid yourself\n/); + assert.match(stopped.text(), /egress\s+my-squid \(PI_EGRESS_PROXY\) exists but is not running; every job is refused pre-spend until it runs/); + assert.doesNotMatch(stopped.text(), /Egress proxy present/); + assert.ok(!stopped.calls.some((c) => ["start", "run", "network"].includes(c.args[0])), "the operator's container is never started"); + const running = harness({ env: { PI_EGRESS_PROXY: "my-squid" }, plan: { ...green, "docker inspect": { code: 0, output: "running\n" } }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" } }); + assert.equal(await running.run(), 0); + assert.match(running.text(), /✓ Egress proxy present \(my-squid, named by PI_EGRESS_PROXY\)/); +}); + +const EGRESS_RM = ["rm", "-f", "pi-dispatch-egress-proxy"]; +const PROXY_FILES = { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }; +// Mutations only: the quiet `network inspect` before a create is a read (exec round 3). +const dockerMutations = (h) => h.calls.filter((c) => c.cmd === "docker" && ["start", "unpause", "run", "network", "rm"].includes(c.args[0]) && c.args[1] !== "inspect").map((c) => c.args); + +test("up on docker: a shipped proxy from another squid or another folder is never started as it is, running or not, and is offered a recreate (#453 gate)", async () => { + for (const [label, state, said] of [ + ["stale digest, exited", proxyState("exited", { image: "ubuntu/squid@sha256:" + "0".repeat(64) }), /it was created from ubuntu\/squid@sha256:0{64}, not the pinned ubuntu\/squid@sha256:6a097f68/], + ["another folder's allowlist, exited", proxyState("exited", { allowlist: "/srv/other/egress-allowlist.conf" }), /its \/etc\/pi-dispatch\/allowlist\.conf is \/srv\/other\/egress-allowlist\.conf, not \/deploy\/egress-allowlist\.conf/], + ["another folder's squid.conf, RUNNING", proxyState("running", { conf: "/srv/other/deploy/egress-proxy.conf" }), /its \/etc\/squid\/squid\.conf is \/srv\/other\/deploy\/egress-proxy\.conf, not \/deploy\/deploy\/egress-proxy\.conf/], + ["no mounts at all, running", proxyState("running", { conf: null, allowlist: null }), /nothing is mounted at \/etc\/squid\/squid\.conf, where \/deploy\/deploy\/egress-proxy\.conf belongs/], + ]) { + const yes = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: state, "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + assert.equal(await yes.run(), 0, label); + assert.match(yes.text(), /✗ pi-dispatch-egress-proxy exists \((exited|running)\) but is not this deployment's proxy: /, label); + assert.match(yes.text(), said, label); + assert.doesNotMatch(yes.text(), /Egress proxy already present/, label); + assert.deepEqual(dockerMutations(yes), [EGRESS_RM, EGRESS_NET, EGRESS_RUN], `${label}: removed and run from the shown argv, never started as it is`); + assert.match(yes.text(), /egress\s+replaced the stale pi-dispatch-egress-proxy with the shipped one/, label); + const no = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: state }, listening: true, files: PROXY_FILES, answers: ["n"] }); + await no.run(); + assert.deepEqual(dockerMutations(no), [], `${label}: declined runs nothing`); + assert.match(no.text(), /egress\s+stale proxy left as it is \(declined\): its policy is not this deployment's/, label); + } + // A symlinked deployment folder is one folder: the realpath of both sides decides. + const linked = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("running", { conf: "/real/deploy/egress-proxy.conf", allowlist: "/real/egress-allowlist.conf" }) }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + linked.deps.fs.realpathSync = (p) => p.replace(/^\/deploy\//, "/real/"); + assert.equal(await linked.run(), 0); + assert.match(linked.text(), /✓ Egress proxy already present/); + // Stale, and this folder cannot recreate it: said, nothing run. + const bare = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("exited", { image: "squid:old" }) }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n" }, argv: ["--yes"] }); + await bare.run(); + assert.match(bare.text(), /✗ deploy\/egress-proxy\.conf is not here, so up cannot recreate it from this folder/); + assert.deepEqual(dockerMutations(bare), []); +}); + +test("up on docker: running means Status running: a paused proxy is unpaused, a crash-looping one is reported, and a missing egress-proxy.conf is never mounted (#453 gate)", async () => { + const paused = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("paused"), "docker unpause": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + assert.equal(await paused.run(), 0); + assert.doesNotMatch(paused.text(), /Egress proxy already present/); + assert.deepEqual(dockerMutations(paused), [["unpause", "pi-dispatch-egress-proxy"]]); + assert.match(paused.text(), /egress\s+unpaused the paused pi-dispatch-egress-proxy/); + const looping = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("restarting") }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + assert.equal(await looping.run(), 0); + assert.match(looping.text(), /✗ pi-dispatch-egress-proxy is restarting: its squid keeps exiting and its restart policy keeps bringing it back\. `docker logs pi-dispatch-egress-proxy` says why; meanwhile each job is retried once, then failed\n/); + assert.deepEqual(dockerMutations(looping), []); + // Absent proxy, allowlist here, the shipped squid.conf not: the runtime would create a directory there (gate 456). + const noConf = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: 1 }, listening: true, files: { "/deploy/egress-allowlist.conf": "x\n" }, argv: ["--yes"] }); + await noConf.run(); + assert.match(noConf.text(), /✗ the egress policy is on but deploy\/egress-proxy\.conf is not here, so no proxy is started without it/); + assert.deepEqual(dockerMutations(noConf), []); +}); + +test("up on docker: another entrypoint or command, or a bind the shipped proxy does not have, is stale; its anonymous volumes are not (#453 gate 2)", async () => { + for (const [label, state, said] of [ + ["a command pointing squid at another config (adversary Y7)", proxyState("running", { entrypoint: ["squid"], cmd: ["-N", "-f", "/tmp/open.conf"] }), /its entrypoint is \["squid"\], not the image's \["entrypoint\.sh"\]; its command is \["-N","-f","\/tmp\/open\.conf"\], not the image's \["-f","\/etc\/squid\/squid\.conf","-NYC"\]/], + ["an extra bind", proxyState("running", { extra: [{ Type: "bind", Source: "/deploy/open.conf", Destination: "/tmp/open.conf" }] }), /it has a bind at \/tmp\/open\.conf \(from \/deploy\/open\.conf\) that the shipped proxy does not/], + ["a volume somewhere else", proxyState("exited", { extra: [{ Type: "volume", Source: "/var/lib/docker/volumes/c3/_data", Destination: "/etc/squid/conf.d" }] }), /it has a volume at \/etc\/squid\/conf\.d/], + ]) { + const h = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: state, "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await h.run(); + assert.match(h.text(), said, label); + assert.deepEqual(dockerMutations(h), [EGRESS_RM, EGRESS_NET, EGRESS_RUN], label); + } + // The shipped shape itself, with its two anonymous volumes, is current. + const current = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("running") }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await current.run(); + assert.match(current.text(), /✓ Egress proxy already present/); +}); + +test("up on docker: bind sources that do not resolve on this host make the mounts UNKNOWN, never stale, and never a removal (#453 gate 2)", async () => { + // The shape Docker Desktop is expected to report (its VM's own paths); not measured here, so never acted on. + const desktop = { conf: "/host_mnt/Users/op/deploy/deploy/egress-proxy.conf", allowlist: "/host_mnt/Users/op/deploy/egress-allowlist.conf" }; + const noResolve = (p) => { + if (p.startsWith("/host_mnt/")) throw Object.assign(new Error(`ENOENT: ${p}`), { code: "ENOENT" }); + return p; + }; + for (const status of ["running", "exited"]) { + const h = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState(status, desktop), "docker start": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + h.deps.fs.realpathSync = noResolve; + await h.run(); + assert.match(h.text(), /⚠ could not compare pi-dispatch-egress-proxy's mounts on this host: \/host_mnt\/Users\/op\/deploy\/deploy\/egress-proxy\.conf, \/host_mnt\/Users\/op\/deploy\/egress-allowlist\.conf are paths this host cannot resolve \(the runtime's own VM's, as Docker Desktop reports its sources\), so those mounts cannot be compared from here\. Everything else about it is still judged, and up replaces nothing while one of its own two mounts is unknown\n/, status); + assert.doesNotMatch(h.text(), /is not this deployment's proxy/, status); + assert.ok(!dockerMutations(h).some((a) => a[0] === "rm" || a[0] === "run"), `${status}: never removed or recreated on an unknown`); + } + // Stale by image while an expected mount is unknown: the finding is said, and nothing is removed until it can be judged. + const staleImage = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("running", { ...desktop, image: "squid:old" }), "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + staleImage.deps.fs.realpathSync = noResolve; + await staleImage.run(); + assert.match(staleImage.text(), /is not this deployment's proxy: it was created from squid:old, not the pinned ubuntu\/squid@sha256:6a097f68[0-9a-f]{56}\n/, "the image drift alone, the unknown mounts not called stale"); + assert.match(staleImage.text(), /not replaced: one of its own mounts could not be compared on this host \(above\)/); + assert.deepEqual(dockerMutations(staleImage), []); + // One bind unknown, the other and an extra mount still judged (round-cap re-review): each on its own. + const partial = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("running", { conf: "/host_mnt/Users/op/deploy/deploy/egress-proxy.conf", allowlist: "/srv/other/egress-allowlist.conf", extra: [{ Type: "bind", Source: "/host_mnt/tmp/open.conf", Destination: "/tmp/open.conf" }] }) }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + partial.deps.fs.realpathSync = (p) => (p.startsWith("/host_mnt/") ? noResolve(p) : p); + await partial.run(); + assert.match(partial.text(), /⚠ could not compare pi-dispatch-egress-proxy's mounts on this host: \/host_mnt\/Users\/op\/deploy\/deploy\/egress-proxy\.conf is a path this host cannot resolve/); + assert.match(partial.text(), /is not this deployment's proxy: its \/etc\/pi-dispatch\/allowlist\.conf is \/srv\/other\/egress-allowlist\.conf, not \/deploy\/egress-allowlist\.conf; it has a bind at \/tmp\/open\.conf/); + assert.deepEqual(dockerMutations(partial), [], "an expected mount is unknown, so nothing is removed"); +}); + +test("up on docker: replacing a stale proxy that carries job networks says so and says to stop the worker first; an unreadable state is left alone (#453 gate 2)", async () => { + const state = proxyState("running", { image: "squid:old", networks: ["pi-dispatch-egress-out", "pi-job-local-a1-net", "pi-sandbox-b2-net"] }); + const h = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: state }, listening: true, files: PROXY_FILES, answers: ["n"] }); + await h.run(); + assert.match(h.promptCalls.join("\n") + h.text(), /It is attached to pi-job-local-a1-net, pi-sandbox-b2-net: removing it cuts those jobs off from their egress mid-run, so stop the worker first \(and let running jobs finish\):/); + // --yes does not cover it (gate round 3): the lines are printed, a person is asked, and without a yes nothing runs. + const unattended = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: state, "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await unattended.run(); + assert.match(unattended.text(), /--yes does not cover replacing a proxy that jobs are using: answer below, or stop the worker and re-run\n/); + assert.equal(unattended.promptCalls.length, 1, "asked, even under --yes"); + assert.deepEqual(dockerMutations(unattended), [], "an unanswered prompt removes nothing"); + const answered = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: state, "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"], answers: ["y"] }); + await answered.run(); + assert.deepEqual(dockerMutations(answered), [EGRESS_RM, EGRESS_NET, EGRESS_RUN], "a person's yes does"); + // A STOPPED stale proxy carries no traffic, so --yes still covers replacing it. + const stopped = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("exited", { image: "squid:old", networks: ["pi-job-local-a1-net"] }), "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await stopped.run(); + assert.deepEqual(dockerMutations(stopped), [EGRESS_RM, EGRESS_NET, EGRESS_RUN]); + const quiet = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: proxyState("running", { image: "squid:old" }) }, listening: true, files: PROXY_FILES, answers: ["n"] }); + await quiet.run(); + assert.doesNotMatch(quiet.promptCalls.join("\n") + quiet.text(), /It is attached to/); + // Exists, and its inspect answer reads as nothing: no start, no run on a taken name. + const junk = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: { code: 0, output: "not json\n" } }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await junk.run(); + assert.match(junk.text(), /✗ pi-dispatch-egress-proxy exists, but its state could not be read from `docker inspect`, so up leaves it as it is/); + assert.deepEqual(dockerMutations(junk), []); +}); + +test("up's recreate creates the proxy's network only where it does not exist, so the daemon's 'already used' never prints (#453 exec round 3)", async () => { + const plan = (exists) => ({ ...green, [PROXY_INSPECT]: proxyState("exited", { image: "squid:old" }), "docker network inspect pi-dispatch-egress-out": exists ? 0 : { code: 1, stderr: "Error: no such network\n" }, "docker rm": 0, "docker network create": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }); + const there = harness({ env: EGRESS_ENV, plan: plan(true), listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await there.run(); + assert.deepEqual(dockerMutations(there), [EGRESS_RM, EGRESS_RUN], "no create for a network that exists"); + assert.match(there.text(), /docker network create pi-dispatch-egress-out {3}\(only if it does not exist yet\)\n/, "the shown line says it is conditional"); + const absent = harness({ env: EGRESS_ENV, plan: plan(false), listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await absent.run(); + assert.deepEqual(dockerMutations(absent), [EGRESS_RM, EGRESS_NET, EGRESS_RUN]); + // The first start of an absent proxy follows the same rule. + const fresh = harness({ env: EGRESS_ENV, plan: { ...green, [PROXY_INSPECT]: 1, "docker network inspect pi-dispatch-egress-out": 0, "docker run -d --name pi-dispatch-egress-proxy": 0 }, listening: true, files: PROXY_FILES, argv: ["--yes"] }); + await fresh.run(); + assert.deepEqual(dockerMutations(fresh), [EGRESS_RUN]); +}); + +test("up's proxy image is the compose file's and the Quadlet unit's, one pinned digest (#453 gate)", () => { + const compose = readFileSync(new URL("../../deploy/docker-compose.yml", import.meta.url), "utf8"); + const quadlet = readFileSync(new URL("../../deploy/pi-dispatch-egress-proxy.container", import.meta.url), "utf8"); + assert.match(compose, new RegExp(`^\\s*image: ${PINNED_SQUID.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")}\\s*$`, "m")); + assert.ok(quadlet.includes(`Image=docker.io/${PINNED_SQUID}\n`)); + assert.equal(EGRESS_RUN.at(-1), PINNED_SQUID); +}); + +test("up hands init the venues it decided, so init's next steps match the pass (#453)", async () => { + const docker = harness({ plan: green, listening: true }); + await docker.run(); + assert.deepEqual(docker.initOpts.map((o) => o.venues), [{ localUsed: true, podmanUsed: false, podmanDefault: false }]); + const podman = harness({ env: { PI_BACKENDS: "podman" }, plan: podmanPlan(), listening: true, extra: podmanExtra() }); + await podman.run(); + assert.deepEqual(podman.initOpts.map((o) => o.venues), [{ localUsed: false, podmanUsed: true, podmanDefault: true }]); +}); + test("up on docker: PI_EGRESS_PROXY is the name up looks for, and it never starts the shipped proxy in its place", async () => { const h = harness({ env: { PI_EGRESS_PROXY: "my-squid" }, plan: { ...green, "docker inspect": 1 }, listening: true, - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, }); assert.equal(await h.run(), 0); - assert.ok(h.calls.some((c) => c.args.join(" ") === "inspect --format={{.State.Running}} my-squid")); + assert.ok(h.calls.some((c) => c.args.join(" ") === "inspect --format={{.State.Status}} my-squid")); assert.ok(!h.calls.some((c) => c.args[0] === "run" || c.args[0] === "network"), "nothing started"); assert.match(h.text(), /PI_EGRESS_PROXY names my-squid, and docker has no container of that name/); }); @@ -830,10 +1075,10 @@ test("D2/D6: a .env line the loaders read differently stops up before anything r test("D4: an EXITED proxy prints false with exit 0, and is offered the unit rather than called present", async () => { const h = harness({ env: { PI_BACKENDS: "podman" }, - plan: podmanPlan({ "podman inspect": { code: 0, output: "false\n" } }), + plan: podmanPlan({ "podman inspect": { code: 0, output: "exited\n" } }), listening: true, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra(), }); assert.equal(await h.run(), 0); @@ -843,10 +1088,10 @@ test("D4: an EXITED proxy prints false with exit 0, and is offered the unit rath test("M7: a RUNNING proxy (true) is left alone and nothing is planned for it", async () => { const h = harness({ env: { PI_BACKENDS: "podman" }, - plan: podmanPlan({ "podman inspect": { code: 0, output: "true\n" } }), + plan: podmanPlan({ "podman inspect": { code: 0, output: "running\n" } }), listening: true, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra(), }); assert.equal(await h.run(), 0); @@ -858,10 +1103,10 @@ test("M7: a RUNNING proxy (true) is left alone and nothing is planned for it", a test("D3: up installs nothing over a container of the unit's name that the unit does not own", async () => { const h = harness({ env: { PI_BACKENDS: "podman" }, - plan: podmanPlan({ "podman inspect": { code: 0, output: "false\n" }, "podman container inspect": { code: 0, output: "\n" } }), + plan: podmanPlan({ "podman inspect": { code: 0, output: "exited\n" }, "podman container inspect": { code: 0, output: "\n" } }), listening: true, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra(), }); assert.equal(await h.run(), 0); @@ -914,10 +1159,10 @@ test("M32: the gate judges the worker's own ids: as root, up refuses exactly as test("M17/M28: on local,podman a refused podman gate leaves the docker pass whole and installs no podman stack", async () => { const h = harness({ env: { PI_BACKENDS: "local,podman" }, - plan: { ...green, "docker inspect": 0 }, + plan: { ...green, [PROXY_INSPECT]: proxyState("running") }, listening: true, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra({ readPodmanInfo: async () => ({ answered: true, info: { rootless: false, serviceIsRemote: false } }) }), }); assert.equal(await h.run(), 0); @@ -952,10 +1197,10 @@ const PODMAN_WARN = "time=\"2026-09-27T10:00:00Z\" level=warning msg=\"The cgrou test("E2: a running proxy is read off stdout, so podman's stderr warnings cannot make `true` unequal", async () => { const h = harness({ env: { PI_BACKENDS: "podman" }, - plan: podmanPlan({ "podman inspect": { code: 0, output: "true\n", stderr: PODMAN_WARN } }), + plan: podmanPlan({ "podman inspect": { code: 0, output: "running\n", stderr: PODMAN_WARN } }), listening: true, argv: ["--yes"], - files: { "/deploy/egress-allowlist.conf": "x\n" }, + files: { "/deploy/egress-allowlist.conf": "x\n", "/deploy/deploy/egress-proxy.conf": "conf\n" }, extra: podmanExtra(), }); assert.equal(await h.run(), 0);