diff --git a/.github/scripts/podman-conformance.mjs b/.github/scripts/podman-conformance.mjs index 66cbcf5e..10a3e29b 100644 --- a/.github/scripts/podman-conformance.mjs +++ b/.github/scripts/podman-conformance.mjs @@ -226,8 +226,9 @@ const withBrokenEnumeration = () => makeReaper({ log: () => {}, bin: `pi-dispatc /** * The egress readings `egressVerdict` takes, from doctor's own canary (`runEgressCanary`, issue #431) under this - * account's Podman: a job-shaped `--internal` network with the proxy attached, and two probe containers built by the - * podman builder as a job's are, as the job user, running the runner's route to the network. This script carried a copy + * account's Podman: a job-shaped `--internal` network with the proxy attached, and three probe containers built by the + * podman builder as a job's are, as the job user: two running the runner's route to the network, and one sending plain + * HTTP to a listed host off port 80, which the proxy must refuse (issue #508). This script carried a copy * of its own until then, with a plain `fetch`, which proved the network and the allowlist and not the runner's route; * the one canary now serves `doctor --live` and this run alike. Every line it prints that is not a pass is repeated * here, since the harness only sees the readings. @@ -387,6 +388,8 @@ async function staleCanarySweep() { const [emptyPid, fullPid] = dead; const emptyNet = egressCanaryNetwork(emptyPid); const fullNet = egressCanaryNetwork(fullPid); + // The first two slugs only: two leftover shapes (one running, one stopped) are what the sweep is tested on, and the + // third slug (issue #508) is matched by the same pattern, so a third leftover would add nothing. const [runningProbe, stoppedProbe] = CANARY_PROBE_SLUGS.map((slug) => egressCanaryProbe(slug, fullPid)); const exists = async (kind, name) => (await podman([kind, "exists", name])).code === 0; try { diff --git a/.github/scripts/podman-host-check.mjs b/.github/scripts/podman-host-check.mjs index 92b98f8a..84b5c48d 100644 --- a/.github/scripts/podman-host-check.mjs +++ b/.github/scripts/podman-host-check.mjs @@ -287,6 +287,8 @@ const fromDoctor = (id, pattern, what) => { fromDoctor("D-health", /^Egress proxy health: healthy/, "the proxy's health"); fromDoctor("EG1", /^Egress policy reaches the provider/, "the provider"); fromDoctor("EG2", /^Egress policy denies an unlisted host/, "an unlisted host"); +// Issue #508: the proxy's port rule for plain (untunnelled) requests, read by doctor's third canary probe. +fromDoctor("EG8", /^Egress policy refuses plain HTTP/, "plain HTTP to a listed host off port 80"); fromDoctor("D-egress", /^read back on local: egress holds/, "egress"); fromDoctor("EG6", /^read back on local: jobToJobIsolation holds/, "jobToJobIsolation"); fromDoctor("D-localFolders", /^read back on local: localFolders holds/, "localFolders"); @@ -363,7 +365,9 @@ const ROWS = [ { row: "SELinux: an operator's local folder, unlabelled", result: "refused: job-inputs-unreadable", needs: /^(S-\w+:(none|ro)$|R-local$|R-control$)/ }, { row: "SELinux: the global overlay, unlabelled", result: "refused: job-inputs-unreadable", needs: /^(R-overlay|R-control)$/ }, { row: "SELinux: SecurityOptions carries name=selinux", result: "measured", needs: /^SECOPT$/ }, - { row: "nftables: egress reaches the provider and denies an unlisted host", result: "measured", needs: /^(EG[1-57]|EG-net|D-egress)$/ }, + // EG8 (issue #508) rides this row rather than a row of its own: it is the same canary on the same network, and a new + // row would need a measured date on this page's SELinux host before it could be printed. + { row: "nftables: egress reaches the provider and denies an unlisted host", result: "measured", needs: /^(EG[1-578]|EG-net|D-egress)$/ }, { row: "nftables: jobToJobIsolation", result: "measured", needs: /^EG6$/ }, { row: "Health checks under systemd", result: "measured", needs: /^(H-|D-health$)/ }, { row: "SELinux: the compose file's config mounts", result: "argv: :ro,z", needs: /^(C-compose|H-proxy)$/ }, diff --git a/deploy/egress-proxy.conf b/deploy/egress-proxy.conf index e46ddcc0..1bcb72d9 100644 --- a/deploy/egress-proxy.conf +++ b/deploy/egress-proxy.conf @@ -6,9 +6,11 @@ # misordered one silently allows everything, so the file an operator edits contains no ordering at all. # # What this policy is, and is not: -# - Hostname filtering on CONNECT, to port 443 only. The provider, the forge and the registry are -# ordinary entries in the allowlist, and nothing is special. The one address-based rule is a DENY: -# a listed name that resolves to a fixed loopback or link-local address of this host is refused. +# - Hostname filtering, and exactly two shapes pass (issue #508): a CONNECT tunnel to a listed host +# on port 443, or a plain (untunnelled) request to a listed host on port 80. Nothing else. The +# provider, the forge and the registry are ordinary entries in the allowlist, and nothing is +# special. The one address-based rule is a DENY: a listed name that resolves to a fixed loopback +# or link-local address of this host is refused. # - TLS is NEVER terminated. This proxy sees the name a client asks for and no byte inside the tunnel, # so it cannot read a credential and cannot account for a token. A proxy that decrypts provider # traffic is OQ-011's mechanism, a materially larger change, and it is not this. @@ -29,6 +31,15 @@ http_port 3128 acl allowed dstdomain -n "/etc/pi-dispatch/allowlist.conf" acl SSL_ports port 443 +# Stock squid's name, narrowed to one port (issue #508). Without it `allow allowed` below let any +# tool that forwards plain HTTP (npm undici's EnvHttpProxyAgent from 8.7 on without proxyTunnel, a +# package manager, a curl -x a job brings) reach ANY port of a listed host, in clear text. What every +# job has never takes this path: pi and the runner tunnel every origin, Node 22's built-in fetch sends a +# CONNECT even for http://, and the job image's git ignores HTTP_PROXY for an http:// remote. A side +# effect: a plain forward `GET https://host/`, where this proxy would open the TLS itself, is refused +# too, since it names port 443 without a CONNECT. Measured on Docker and rootless Podman 4.9.3 and 5.8.1 +# (docs/egress.md): port 8080 went from 200 to 403, port 80 stayed 200. +acl Safe_ports port 80 acl CONNECT method CONNECT # The host's own services, by address (issue #428). On rootless Podman an account's containers.conf @@ -55,9 +66,14 @@ acl to_host_local dst 127.0.0.0/8 0.0.0.0/32 169.254.0.0/16 10.0.2.2/32 ::1 ::/1 # debug_options 78,3: a CONNECT to an unlisted name sent a DNS query, where without the rule it sent # none), which is a DNS channel out of an egress-armed job. With `allowed` first, only a listed name # is ever resolved here, and it is the only kind the allows below could let through anyway. +# The port rule for plain requests (issue #508) sits right before the allow it narrows: first match +# wins, so after it it would never apply. It names only port and method ACLs, which resolve nothing, +# so no unlisted name is resolved by it. It never matches a CONNECT, so the tunnel rules (including a +# CONNECT allow later placed between the first two rules, see #503) are untouched by it. http_access deny allowed to_host_local http_access deny CONNECT !SSL_ports http_access allow CONNECT allowed +http_access deny !Safe_ports !CONNECT http_access allow allowed http_access deny all diff --git a/docs/egress.md b/docs/egress.md index 9dc2695d..6a577e56 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -82,7 +82,7 @@ than everything else combined. | | | |---|---| | One `--internal` network **per job** | `pi-job--net`, created at job start and removed at job end. Holds exactly two endpoints: the container and the proxy. If the worker dies before it can remove one, the next boot removes it, detaching whatever is still on it first, and says so in the log if it cannot. | -| One long-lived proxy | `pi-dispatch-egress-proxy`, squid, hostname filtering on `CONNECT` to port 443, and a listed name that resolves to one of this host's fixed loopback or link-local addresses (or slirp4netns's `10.0.2.2`) is refused (issue #428). The allowlist is matched as written (`dstdomain -n`): an IP-literal request is refused unless that literal is itself listed, and the proxy makes no reverse lookup of it. A host mapped to another address (pasta's `--map-host-loopback
` or `--map-gw`, slirp4netns's `cidr=`) is not covered here; on the `podman` venue the worker refuses the containers.conf that would do that, and the account's running rootless network while it still does after the key is gone (issue #450). Publishes no port. | +| One long-lived proxy | `pi-dispatch-egress-proxy`, squid, filtering by hostname. Exactly two shapes pass, both to a listed host: a `CONNECT` tunnel to port 443, or a plain (untunnelled) request to port 80. Nothing else passes (issue #508). "A plain request to port 80" also covers squid's own gateways for other schemes, but only to port 80 (`ftp://host:80/`, `https://host:80/`); every other scheme and port is refused. A plain forward request for an `https://` URL on 443, where the proxy would open the TLS itself, is refused too. A listed name that resolves to one of this host's fixed loopback or link-local addresses (or slirp4netns's `10.0.2.2`) is refused (issue #428). The allowlist is matched as written (`dstdomain -n`): an IP-literal request is refused unless that literal is itself listed, and the proxy makes no reverse lookup of it. A host mapped to another address (pasta's `--map-host-loopback
` or `--map-gw`, slirp4netns's `cidr=`) is not covered here; on the `podman` venue the worker refuses the containers.conf that would do that, and the account's running rootless network while it still does after the key is gone (issue #450). Publishes no port. | | One upstream network | `pi-dispatch-egress-out`. Only the proxy is on it. | **Per job, not one shared network**, and that is the part worth understanding. A shared network is a shared @@ -129,6 +129,7 @@ deployment. `doctor` does it once, when you ask. ✓ Egress proxy health: healthy ✓ Egress policy reaches the provider (api.anthropic.com answered, so the whole path works and no key was spent) ✓ Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose) +✓ Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443) ``` ### On the native `podman` venue @@ -142,7 +143,8 @@ mount), so a pass is about what a job gets. Its lines carry the section's prefix ``` ✓ podman: Egress policy reaches the provider (api.anthropic.com answered, so the whole path works and no key was spent) ✓ podman: Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose) -✓ read back on podman: egress holds (the provider was reached and an unlisted host was not) +✓ podman: Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443) +✓ read back on podman: egress holds (the provider was reached, an unlisted host was not, and plain HTTP off port 80 was refused) ``` A plain `pi-dispatch doctor` does not run it there: the first keep-id start of an image copies its layers, which @@ -158,7 +160,7 @@ them. ### Lines about leftovers You may also see a line about a leftover. Those are the canary's own objects, `pi-dispatch-egress-doctor-` -and its two probe containers. The rule is simple enough to rely on: **every one of them is removed at the end +and its three probe containers. The rule is simple enough to rely on: **every one of them is removed at the end of the run that made it, or reported in that same run** -- so what a later run finds is what a run that was KILLED left behind, and doctor does not have to guess which. What it knows is that the process in the name is no longer alive. The next run sweeps whatever belongs to a process that is no longer alive @@ -186,7 +188,7 @@ generator. -The two policy lines each run a throwaway container on a throwaway network, using **your job image's own node and +The provider and unlisted-host lines each run a throwaway container on a throwaway network, using **your job image's own node and its runner's own module** (pi loaded, then the runner's proxy restore, issue #427), so they prove the route your jobs' provider calls take once the runner has called that module; that it does, before its first request, is checked in CI. (Before #427 they used a plain `fetch`, which never loads pi and stayed green while every job failed.) They cost nothing: `api.anthropic.com` answers `401` to an @@ -195,6 +197,32 @@ without spending a token. The deny probe asks for `example.com`, a host that res that lets everything out is caught; it is only contacted if your proxy lets the request out, which is the finding. A probe container that does not run at all is reported as not run, never as a deny. +The third line (issue #508) runs a third container the same way, but not the runner's route: that route tunnels +every request, so it would only ever send a `CONNECT`. This one sends a plain forward request with raw `node:http`, +`GET http://api.anthropic.com:443/`, the way a tool that forwards plain HTTP would (npm undici's `EnvHttpProxyAgent` +without `proxyTunnel`, a package manager, or a `curl -x` a job brings). It asks for port +443 because that host certainly listens there, so a proxy that lets the request through gets an answer (a `400` for +clear text on a TLS port) instead of a timeout, and nothing extra has to be listed. It counts as refused only when +squid itself refuses it: a `403` with an `X-Squid-Error` header starting `ERR_ACCESS_DENIED`. Any other answer means +the proxy let it through. A proxy that cannot be reached or never answers is no reading. The check is best effort: +an upstream on port 443 that answered clear text with a `403` and a forged `X-Squid-Error` header would read as +refused, which needs control of the provider's own answer. + +**Right after an upgrade this line can fail.** squid reads its rules only at start, so until the proxy restarts on +the refreshed `egress-proxy.conf` it still runs the old rules: the line reads `Egress policy lets plain HTTP through +to api.anthropic.com on port 443, ...` and `--live`'s egress verdict fails. `pi-dispatch up` refreshes the file and +restarts the proxy; on the `podman` venue `pi-dispatch service install --force` does. A proxy you started by hand +needs the updated file in its mount and a restart (`docker restart pi-dispatch-egress-proxy`, or `podman restart`). + +**With the policy on, a forge must be served over `https://` on port 443.** A job gets the proxy variables in +uppercase only, and git ignores `HTTP_PROXY` for an `http://` remote, so it never sends one through the proxy: from +behind the job's `--internal` network an `http://` forge fails on any port, 80 included (measured in the job image, +git 2.39.5; this was so before issue #508 too). An `https://` remote is a `CONNECT`, which the proxy refuses to any +port but 443. The clone runs on the host before the container exists, so with an `http://` forge it is a job's +git push and fetch that fail; on a port other than 80 its API calls fail too (glab and tea do use `HTTP_PROXY`, and +port 80 is allowed). `pi-dispatch doctor` warns when `GITLAB_URL` or `FORGEJO_URL` is anything but `https://` on 443. The ways +out are `https://` on 443, or `PI_EGRESS=0`. + An absent proxy is a **hard failure** in doctor, because every job is refused while it is down. Everything that needs the network to answer is a **warning**, because a custom provider base URL or a transient blip would each make a red there a false alarm. So is any leftover the canary could not clear: a network nobody is @@ -250,9 +278,11 @@ env-proxy dispatcher itself, from pi's own `undici`, right after pi is loaded, a Before, it was a plain `fetch` and stayed green through all of this. On a job image built before the fix, the canary says the image cannot find that module, rather than reporting a policy result. That the runner's entrypoint calls the module, before its first request, is checked in CI, where the job image's contract job runs the real -entrypoint against a stub proxy. The runner's restored dispatcher tunnels every origin (`proxyTunnel: true`, -as pi's own CLI does since undici 8.7 stopped tunnelling `http://`), so an `http://` provider is a CONNECT -to port 80, which the shipped policy refuses, as before. +entrypoint against a stub proxy. pi and the runner always tunnel (`proxyTunnel: true`, as pi's own CLI does +since undici 8.7 stopped tunnelling `http://`). So a provider configured with an `http://` URL is a CONNECT to +that URL's port, and the shipped policy refuses it unless the port is 443. A tool that forwards plain HTTP +instead, such as npm undici's `EnvHttpProxyAgent` without `proxyTunnel` or a package manager, gets port 80 of a +listed host and nothing else (issue #508). ## What this does not buy you @@ -265,6 +295,9 @@ It also accounts for nothing. A staged package that spawns a `pi` subprocess spe host, which is on the allowlist by necessity, and a proxy that does not decrypt cannot count tokens (`OQ-011`). +Plain HTTP to port 80 of a listed host is allowed, because some package mirrors and redirects still use it +(issue #508). Whatever a job sends that way crosses the network in clear text. + And it does not hide **this host** from the job. `--internal` stops the network routing anywhere beyond itself, but its gateway is still the host, so a service listening on `0.0.0.0` there answers a job container that dials the gateway address. Measured on Docker 27.5.1 and on rootful Podman 5.8.2 alike: a listener on @@ -282,6 +315,15 @@ All of it was run. The method costs nothing and is worth repeating on your own h - **The whole path, end to end**, with the shipped compose profile and a real per-job network: the provider answered `401` in 228 ms, an unlisted host was denied in 18 ms, and `api.github.com` answered in 205 ms through the `.github.com` rule. +- **The port rule for plain HTTP** (issue #508), measured on 2026-09-30 with the pinned squid (6.13) on Docker + 29.1.3, rootless Podman 4.9.3 and rootless Podman 5.8.1 (SELinux enforcing), all three alike. Before the rule, a + plain `GET` to a listed host on port 8080 answered `200`; after it, `403`. Port 80 answered `200` before and after. + A `CONNECT` to port 8080 was `403` both times. A plain `GET http://api.anthropic.com:443/`, the canary's third + probe, got Cloudflare's `400` before and squid's `403` with `X-Squid-Error: ERR_ACCESS_DENIED` after. A plain + forward `GET https://api.anthropic.com/` made squid open the TLS itself before (`503`), and is `403` after. An + unlisted name was never resolved, before or after. npm undici 8.10.0's `EnvHttpProxyAgent` without + `proxyTunnel` went from `200` to `403` on port 8080 and stayed `200` on 80. Node 22's built-in `fetch` with + `NODE_USE_ENV_PROXY=1` sends a `CONNECT` even for `http://`, so it was already refused off port 443. - **The pre-spend refusal**, in both directions: with the proxy stopped the preflight returns `proxyStopped`, with it removed `proxyMissing`, and with `PI_EGRESS` unset it returns admit **without spawning docker at all**. diff --git a/docs/forgejo.md b/docs/forgejo.md index 4dc23346..62ab9ced 100644 --- a/docs/forgejo.md +++ b/docs/forgejo.md @@ -32,6 +32,12 @@ FORGEJO_BOT_ID=42 # the harness account's numeric id, from its profile FORGEJO_WEBHOOK_SECRET=... # a long random string ``` +With the egress policy on (the default), Forgejo must be served over `https://` on port 443. git in a job never +sends an `http://` remote through the proxy, and the proxy refuses a `CONNECT` to any other port (issue #508, +`docs/egress.md`). With Forgejo's default `http://forgejo.example:3000` a job's git push and fetch fail, and its +API calls too, since the port is not 80. Plain `http://` works only with `PI_EGRESS=0`. `pi-dispatch doctor` warns when `FORGEJO_URL` is anything +but `https://` on 443. + There is no source to choose here either. `FORGEJO_AUTH_SOURCE` accepts only `pat` and defaults to it, for the same reason GitLab's does: Forgejo has no App or installation token, so the variable exists to refuse a wrong assumption of symmetry with `GITHUB_AUTH_SOURCE` rather than to offer an alternative, and it diff --git a/docs/gitlab.md b/docs/gitlab.md index db69a234..ab63c76f 100644 --- a/docs/gitlab.md +++ b/docs/gitlab.md @@ -116,8 +116,9 @@ for `RECEIVER_IDENTITY_RETRY_SECONDS`, default 10 minutes); when a window ends t the supervisor starts another, so an instance restart measured in minutes is ridden out rather than outlasting the recovery. -The **job container** is a separate trust store: `git clone` and `glab` run inside it, so the CA has to be -in the image. Add it to your own image (`docs/job-image.md`) — a `COPY` of the cert into +The **job container** is a separate trust store: the job's `git push` and `fetch` and `glab` run inside it +(the clone runs on the host first), so the CA has to be in the image. Add it to your own image +(`docs/job-image.md`): a `COPY` of the cert into `/usr/local/share/ca-certificates/` plus `update-ca-certificates`. The stock image trusts only the public roots. @@ -125,9 +126,13 @@ roots. HTTPS, calls fail rather than silently following — a redirect is somewhere to send a credential, and following one is how a token ends up at a host you did not name. Set `GITLAB_URL` to the final URL. -Plain `http://` does work if that is genuinely what you run. It sends the token in the clear on every API -call and every clone, so it is only reasonable on a trusted network, and it is not what the isolation -model assumes elsewhere. +Plain `http://` works only with the egress policy off (`PI_EGRESS=0`). It sends the token in the clear on +every API call, push and fetch (and the clone on the host), so it is only reasonable on a trusted network, and it is not what the +isolation model assumes elsewhere. With the policy on (the default), GitLab must be served over `https://` on +port 443: git in a job never sends an `http://` remote through the proxy, and the proxy refuses a `CONNECT` to +any other port (issue #508, `docs/egress.md`). Otherwise a job's git push and fetch fail, and on a port other +than 80 its API calls fail too. +`pi-dispatch doctor` warns when `GITLAB_URL` is anything but `https://` on 443. ## Minimum GitLab version: 17.4 diff --git a/docs/podman.md b/docs/podman.md index 4f119d2b..7a1c05a9 100644 --- a/docs/podman.md +++ b/docs/podman.md @@ -1155,10 +1155,11 @@ an unprivileged account (uid 1234), on 2026-09-25. Run everything below as the w that the job image is in this account's store, and, with the egress policy armed, that the proxy is running under this Podman and that its rootless network keeper holds (not holding is ✗ on Podman 4.x only, step 6). `--live` reads the declarations back off real containers and says it read them back on podman, - `egress` included: with the policy armed it first runs doctor's egress canary under this account's Podman, two + `egress` included: with the policy armed it first runs doctor's egress canary under this account's Podman, three containers built like a podman job (your uid as `--user`, `--userns=keep-id`, the venue's pinned flags, a job's proxy variables) on a job-shaped `--internal` network with the proxy attached, one that must reach the provider - through the runner's own route and one that must not reach an unlisted host. Its lines start `podman: Egress`, + through the runner's own route, one that must not reach an unlisted host, and one whose plain HTTP to a listed + host off port 80 must be refused (issue #508). Its lines start `podman: Egress`, before the read-back's. A plain `pi-dispatch doctor` does not run it on this venue, and says so in a ⚠ line pointing at `--live`. A canary network a killed `--live` left behind is removed by the next `--live`, on Podman 4.9 and 5.x alike: the sweeps read what is on a network with `podman ps -a --filter network=`, diff --git a/image/runner/src/env-proxy.mjs b/image/runner/src/env-proxy.mjs index bfb126a2..e0718a2f 100644 --- a/image/runner/src/env-proxy.mjs +++ b/image/runner/src/env-proxy.mjs @@ -32,8 +32,9 @@ import { createRequire } from "node:module"; * exactly as it did at 0.80.7 and as it does under pi's own CLI. That matters for an operator-configured `http://` * provider (a local gateway, an overlay models.json baseUrl): tunnelled, it arrives as `CONNECT host:80`, which * deploy/egress-proxy.conf refuses (`deny CONNECT !SSL_ports`), the verdict it has always had; untunnelled it would - * arrive as a plain forward request, which the same file's `allow allowed` admits for a listed host, a silent - * widening of the policy by a dependency bump. Measured at the 0.99.1 pin: without the option the env-proxy test's + * arrive as a plain forward request, which the same file's `allow allowed` admits for a listed host on port 80 only + * (issue #508: `deny !Safe_ports !CONNECT` refuses every other port), a silent widening of the policy by a dependency + * bump all the same. Measured at the 0.99.1 pin: without the option the env-proxy test's * CONNECT-only proxy saw no tunnel and answered the plain request 405. * * Only when `NODE_USE_ENV_PROXY` is "1", which is exactly what the worker emits with egress armed. Without it nothing is diff --git a/specs/design.md b/specs/design.md index fdd90e7e..d74d078c 100644 --- a/specs/design.md +++ b/specs/design.md @@ -3422,7 +3422,8 @@ money with no upstream turn limit (`REQ-RUNNER-TURN-BUDGET`). - **Decision**: Ship the egress control as **one `--internal` Docker network per job, named in the worker's own `docker run` argv**, plus **one long-lived allowlist proxy** attached to each of those networks for the - life of that run, filtering `CONNECT` by hostname and never terminating TLS. The provider is an ordinary + life of that run, filtering by hostname and never terminating TLS: a `CONNECT` to port 443 or a plain request + to port 80, both to a listed host, and nothing else (issue #508). The provider is an ordinary entry on the allowlist. A **pre-spend check** on the proxy refuses a job the policy cannot serve. Off unless `PI_EGRESS=0`, and on every job with no per-trigger opt-out (`REQ-EGRESS-ALLOWLIST`, `INT-EGRESS-POLICY-CONTRACT`). @@ -3432,6 +3433,17 @@ money with no upstream turn limit (`REQ-RUNNER-TURN-BUDGET`). `doctor`, refusing before money is spent -- needs the policy to be an object the worker names. That is what makes this an argv change rather than a documentation change, and it is why the network and not the proxy is the load-bearing half. +- **Plain HTTP on port 80, not HTTPS only** (issue #508, the owner's decision). The rules had no port rule for a + request that is not a `CONNECT`, so any tool that forwards plain HTTP (npm undici's `EnvHttpProxyAgent` + without `proxyTunnel`, a package manager) reached every port of a listed host. The job image's git is not one: + it ignores `HTTP_PROXY` for an `http://` remote, so a forge must be `https://` on 443 with the policy on. + They now carry stock squid's `Safe_ports`, narrowed to port 80, as `http_access deny !Safe_ports !CONNECT` + right before `http_access allow allowed`. Refusing plain HTTP entirely was the alternative, and it is rejected + for now because some package mirrors and redirects still use port 80; pi itself does not need it, since pi + and the runner tunnel every origin. What port 80 carries crosses the network in clear text, and + `docs/egress.md` says so. Also rejected: a port rule after the allow (first match wins, so it would never + apply), and a `dst` rule ahead of `allowed` (it would resolve every requested name, a DNS channel out). `doctor` + reads the rule back with a third canary probe, a raw forward request to `api.anthropic.com:443`. - **What cleans up, and why never `-f`** (issues #357, #350). The job path removes its own network in a `finally`; what that cannot cover is a worker that DIED, and the boot reaper's sweep for those was a bare `network rm` inside a `try {} catch {}` whose comment said an in-use network "belongs to something else". @@ -6391,3 +6403,4 @@ a tunnel. | 2026-09-29 | Issue #483 (insights page: labels that overlap wires, loops and each other), with PR #487's review round 1 and its final review folded in. **`DES-ADMIN-VIA-PI-EXTENSION` AMENDED**: a new passage after the label-fit gate's results, "Where the topology draws its labels and wires", stating what is guaranteed (the under-chip stack below every under-row route; one greedy label placement on points of each wire's own path, clear of every drawn thing and every label before it and read as that wire's (its own wire within 16 units, every other pair's at least 1.5 farther), a label with no such spot moved to its wire's tooltip and counted in the legend; column gaps crossed by labelled wires widened to hold their widest label; back edges out of one node on runs of their own; a label line above each folder's first row; a lane, gap line and gutter per wire that leaves its group, wires into one port on separate lines, square crossings of the lanes; skipped columns passed between their boxes; skill-group titles full width unless a path enters their band, else between the chip's ports; the cumulative end label above its last point and off the gridlines; the viewBox at drawn bounds plus 12 and at most 1.35 page pixels per unit), what is not (two wires that stay in one group may still run along each other; a label box is an estimate the gate alone measures; a dense scene puts labels in tooltips; a chip label cut to its chip's edge may end up to 3 units under its own port, per #422's measured decision), and the gate's collision probe (text against text, paths, painted rect borders, lines and svg edges, and each wire label against the reading rule, on seven pages, with a shrink that scales with the rendered scale). A CORRECTION: the corpus overflow count with the fit stripped was 31 on the tree before this change, not the 29 the passage recorded, and is 30 after it (the builder now cuts the corpus's long skill-group title). Causes, one per defect of the issue: the spend badge baseline (chip top + 53) and the re-arm loop run (chip top + 55) were two literals in two files; a pair's second label was stepped 10.5px from the first at its own curve's midpoint, less than a line; a trigger-to-tier wire was a bezier across every group stacked between; the cumulative end label was start-anchored 6px right of a point on the plot's right edge; the viewBox took a fixed 80px margin. Found on the way: a skill group's title began left of the chip's input port (a wire from its own row or above ran through it). Review round 1 refuted three first cuts, and the final review a fourth (a clear spot up to six lines out read as another edge's label; replaced by the reading rule, with the gap widening that gives it room), each replaced by the rule above rather than patched: the per-pair label stack (a fan-out's counts on the same pixels, lifts under a chip and into a folder title), the shared lane and gap line (one trunk for three wires, and an intra-group curve along the lane), and the between-ports title cut applied always (near-identical names read the same); it also found the skip-column crossing, on main too. **Why rules and not nudges**: each is a property of every scene, which is what the unit pins assert over their scenes (mutation-checked: reverting any one rule turns a pin red) and the browser probe measures on its pages. **`REQ-INSIGHTS-HTML-EXPORT` UNCHANGED, checked**: its clauses (a) to (h) are encodings, not geometry. **`REQ-TOPOLOGY-GRAPH` UNCHANGED, checked**: (c)'s labels keep their words; a label in a tooltip keeps them too, and the legend states the count. ASCII scenes change bytes: under-row routes drop 7px, the band under them grows 4px, the first row sits 14px lower, a group with lanes widens the pitch after their column, wire labels move and a hidden one becomes a title element, a gap crossed by a labelled wire widens, a second back edge out of one node runs a step lower, skill-group titles move 22px right and gain an unpainted rect, the viewBox shrinks, the pane gains a max-width, and the cumulative plot starts 6px lower. **Code evidence**: admin/src/graph-html.mjs; admin/src/insights-html.mjs; admin/test/graph-html.test.mjs and admin/test/insights-html.test.mjs (the issue #483 pins); .github/scripts/label-fit-check.mjs (the collision probe and its pages). | | 2026-09-29 | Issue #492, with PR #493's review folded in. **`DES-COST-FOLD-BY-SCAN` AMENDED**: the aggregation sentence gains its one exception, a bucket every run of which ONE declared plan covers, with no run a floor, keeps `plan:` instead of demoting to `estimated` (the review found the first cut dropped a floor's `≥` and joined two ids with a character an id may hold), so the fold agrees with `REQ-COST-ANALYTICS` (b) and with its own per-model bucket; mixed buckets still demote with coverage, and zero-rated and unknown buckets are unchanged (the issue is about plan coverage; widening the exception to them is a separate decision). The daily chart draws a plan day with the same 1px dashed sliver as before; only its tooltip now reads `plan:`. **`DES-ADMIN-VIA-PI-EXTENSION` UNCHANGED, checked**: the topology's layout rules and the label-fit gate are untouched (the gate exits 0, with a new page of an armed and a spent one-shot and a long plan badge; the documented fixture's page probes clean). docs/costs.md gains the sentence saying what a wholly plan-covered bucket reads and what `dispatch_costs` returns for it. **Code evidence**: admin/src/costs.mjs; admin/test/costs.test.mjs. | | 2026-09-30 | Issue #509, the pi 0.80.7 -> 0.99.1 bump, one row for the runner, worker and admin halves. **`DES-USAGE-METER-VIA-API-PROVIDER-REGISTRY` AMENDED in place** (ID kept, it is an address): at pi 0.99.1 the api-provider registry is off the session path, so the process-wide meter moved to ModelRuntime.prototype, with the registry kept as a second half for legacy extension calls and an AsyncLocalStorage deduplicating the composed-provider path where the two meet; a Rejected entry records why patching the prototype is not the rejected `dist/` patch; the Must-handle list is marked as the registry half's; compaction and branch summaries now land in `otherTotal`. The observe-don't-consume design, the fallback rule and the brake are UNCHANGED. **`DES-ADMIN-VIA-PI-EXTENSION` AMENDED**, the supported version (the pin, `0.99.1`), the evidence line numbers re-read at 0.99.1, and the column-width bullet: pi-tui 0.99.1 gives 472 spacing marks a column and walks a cluster from after its base, so the #417 doubling is gone (its pinned counts, 13,545 and 3,273 doubled forms and 6,446 breaking predecessors, are now zero, re-pinned rather than dropped) and the matched-leader rule is re-derived for the opposite correction: a spacing mark that LEADS a cluster draws nothing, so `panel.mjs` bundles the leading part of a run with its base as one step as wide as the base alone, and a cluster made only of non-printing code points as one zero step. The segmenter probe, the memo and the context stand-in are kept; the stand-in's safe direction flipped (a wrong join is now an under-count) and the predecessor sweep pins the new count (1,111,984 predecessors lead, fifteen join). New pinned arms: 472 spacing marks against 242 tails (1,888 and 124 narrower forms), and against the five emoji modifiers (exact at a string's start, 155 exact after a letter, the rest the declared skin-tone over-count). Residuals restated: the U+FFA0 under-count is gone, a joined piece now draws one column wider and is clipped by its frame, a Devanagari conjunct no longer over-counts. **`DES-SUBSCRIPTIONS-ARE-COUNTERFACTUAL-ONLY` AMENDED**, the first Why bullet: kimi-coding, zai and zai-coding-cn now carry implied API-equivalent rates, so prepaid runs record a positive cost rather than 0, and a zero rate is no longer a subscription signal; the declaration's role is unchanged. **The `run.excludeTools` decision AMENDED**: the known set and the runner's derivation are the eight built-ins of pi 0.99.1 (`powershell` added). The package-discovery Rejected entry re-checked: `collectPackageResources` still falls through to the convention dirs at 0.99.1. PR #510's review found two cuts the new rule left measuring text where it is not drawn, both fixed at the rule: a cut that strips a dangling joiner re-measures and re-cuts (a spacing mark and a joiner draw nothing together, the mark alone a column; thirty-one marks, swept), and every cutter takes an `afterSpace` context, used by both frame titles and the divider's label and meta (an 8-column box drew its top line at 9; the divider missed its width in 13,380 swept forms at the previous commit). **UNCHANGED, checked**: `OQ-035` (the substitution class is untouched; `interpreted` keeps every mark as composing), `REQ-TOPOLOGY-GRAPH`, `REQ-INSIGHTS-HTML-EXPORT` (the chip estimate is still one-sided; its exact-count pin moved from 185,319 to 185,583 for the 264 BMP spacing marks), `DES-COST-FOLD-BY-SCAN` (classification order unchanged: an owned plan first, so a declared plan reads `plan:` across the rate change), counterfactual-only subscriptions. **Code evidence**: image/runner/src/usage-meter.mjs, model-runtime.mjs; admin/src/panel.mjs -> widthSteps, leaderSteps, planRun, contextOf, NON_PRINTING, SPACING; admin/src/style.mjs -> visibleLen (comments); admin/test/width-leaders.test.mjs, width-predecessors.test.mjs, width.test.mjs, graph-html.test.mjs, control-bytes.test.mjs, helpers/renderer.mjs (pin 0.99.1). | +| 2026-09-30 | Issue #508 (plain HTTP reached every port of a listed host). **`DES-EGRESS-DENY-ON-A-DEDICATED-NETWORK` AMENDED**, the Decision and one new bullet: the proxy passes a `CONNECT` to 443 or a plain request to 80, both by hostname to a listed host, by `http_access deny !Safe_ports !CONNECT` right before `http_access allow allowed`; the owner chose port 80 over HTTPS only because some package mirrors and redirects still use plain HTTP; a rule after the allow and a `dst` rule ahead of `allowed` are rejected; `doctor`'s canary gains a third probe that reads the rule back; the job image's git ignores `HTTP_PROXY` for an `http://` remote, so with the policy on a forge must be `https://` on 443. | diff --git a/specs/interfaces.md b/specs/interfaces.md index bfaa7729..de288d4d 100644 --- a/specs/interfaces.md +++ b/specs/interfaces.md @@ -1674,7 +1674,7 @@ contract governs the argv of one container, this governs the estate that argv jo | live-probe peer networks | `pi-dispatch-live-peer---net` | the same, built by the same `createJobNetworkWith`, for `doctor --live`'s two peers only while it runs (`INT-LIVE-PROBE-CONTRACT`) | | upstream network | `pi-dispatch-egress-out` | an ordinary bridge; only the proxy is on it | | doctor canary network | `pi-dispatch-egress-doctor-` | `--internal`, one per doctor PROCESS, built and removed inside one `runEgressCanary` run (`egressCanaryNetwork`), on the runtime of the venue it reads: docker's on every `doctor` of a deployment with `local`, this account's Podman on `doctor --live` for the `podman` venue (issue #431) | - | doctor canary probes | `pi-dispatch-egress-probe--` | `--rm`, one per direction on that network (`egressCanaryProbe`). On docker the canary's own short argv; on podman a JOB's, from the podman builder (`egressCanaryProbeArgs`): the job user, `--userns=keep-id`, `PODMAN_PINNED_FLAGS`, a job's egress variables and HOME, and no mount | + | doctor canary probes | `pi-dispatch-egress-probe--` | `--rm`, one per probe on that network (`egressCanaryProbe`), the slugs `provider`, `unlisted` and `plainhttp` (issue #508), in that order. On docker the canary's own short argv; on podman a JOB's, from the podman builder (`egressCanaryProbeArgs`): the job user, `--userns=keep-id`, `PODMAN_PINNED_FLAGS`, a job's egress variables and HOME, and no mount | | proxy component | `pi-dispatch-egress-proxy` | squid, `http_port 3128`, **no published port**. Overridable by `PI_EGRESS_PROXY`, and then the operator's own: `up` and `service install` look for the overriding name and start only the shipped one (issue #430). On the native `podman` venue it is the Quadlet unit `pi-dispatch-egress-proxy.service`, on the same-named upstream network. | | rootless network keeper (`podman` venue only) | `pi-dispatch-netns-keeper`, on its own network of that name | one idle container (`sleep` in the proxy's image and digest), on an `--internal`, DNS-disabled network no job and no proxy is on; `--read-only`, `--cap-drop=all`, `no-new-privileges`, uid 65534, no port, no mount. It exists so this account's rootless network helper always has a running bridge member: on Podman 4.9 the per-job network's teardown otherwise cuts the proxy's route out (issue #458, measured). It counts only RUNNING, in bridge mode, ON that network: one on `--network none`, slirp4netns, pasta or host holds nothing open (measured). The Quadlet unit `pi-dispatch-netns-keeper.service`, installed while the policy is armed whatever `PI_EGRESS_PROXY` names. Outside every prefix below that anything removes, and never removed by pi-dispatch; `doctor` reads it (✗ on Podman 4.x when it does not hold), and so does the worker's egress preflight on 4.x, which retries the job pre-spend while it does not hold. | @@ -1885,6 +1885,21 @@ contract governs the argv of one container, this governs the estate that argv jo closure is the podman venue's refusal of such a conf, which since issue #450 also reads the live network: the account's running rootless network helper, from `/proc`, keeps a removed key's option until every bridge container stops, the proxy's among them, and is refused as `podman-conf-widens-job` while it does. + **Exactly two shapes pass, both to a listed host** (issue #508): a `CONNECT` tunnel to port 443 + (`deny CONNECT !SSL_ports`, then `allow CONNECT allowed`), or a plain (untunnelled) request to port 80 + (`acl Safe_ports port 80`, and `http_access deny !Safe_ports !CONNECT` directly before `http_access allow allowed`). + Before #508 that last allow had no port rule, so any tool that forwards plain HTTP (npm undici's + `EnvHttpProxyAgent` from 8.7 without `proxyTunnel`, a package manager, a `curl -x` a job brings) reached every port + of a listed host in clear text. The port rule sits right before the allow it narrows, since first match wins; it + names only a port and a method ACL, which resolve nothing, so no unlisted name is resolved by it; and it never + matches a `CONNECT`, so the tunnel rules, a `CONNECT` allow later placed between the first two rules included + (#503), are untouched. pi and the runner tunnel every origin, so they never take this path. A plain forward + `GET https:///` (the proxy opening the TLS) names port 443 without a `CONNECT` and is refused. With the + policy armed a job reaches a forge only over `https://` on 443: a job's proxy variables are uppercase only and + git ignores `HTTP_PROXY` for an `http://` remote (measured, git 2.39.5 in the job image), so an `http://` forge + fails on any port, and a `CONNECT` to any port but 443 is refused. The clone runs on the host, so what fails is a + job's git push and fetch, and its API calls too off port 80 (glab and tea use `HTTP_PROXY`). `doctor` warns (⚠) when the policy is armed and the `GITLAB_URL` or + `FORGEJO_URL` of a forge the triggers name is anything but `https://` on 443. - **The proxy image is digest-pinned**, and the `valkey/valkey:8` precedent one service over deliberately does not transfer: a floating tag on a queue breaks loudly and spends nothing, while this container **is the allowlist**, so a floating tag would let an upstream rebuild change what every job may reach with no @@ -1918,7 +1933,8 @@ contract governs the argv of one container, this governs the estate that argv jo proxy variable is emitted and the preflight spawns nothing. Given any other accepted value and a running proxy, each job runs on a network whose only other member is that proxy, reaches an allowlisted host, and is refused - an unlisted one by the proxy rather than by the agent. Given a proxy that is absent or stopped, the job + an unlisted one by the proxy rather than by the agent; a plain request to a listed host on any port but 80 is + refused by the proxy (issue #508). Given a proxy that is absent or stopped, the job returns `outcome: "policy"` with `budgetReserved: false`, `docker run` is never spawned, and the queue does not retry it. Given a daemon that does not answer, the job throws and IS retried. Given the `podman` venue on Podman 4.x with the proxy up and the keeper not running on its own bridge network (or started more than 15 s after @@ -2595,10 +2611,14 @@ is its only entry point (`worker/src/live-probes.mjs`, driven from `doctor.mjs`) attached, with a note, and removes only memberless ones. - `egress`: **the one reading this module does not make.** It needs the proxy's allowlist and a provider to reach, so it is folded in from `doctor`'s egress canary (`runEgressCanary`) for the venue being read back, - whose two probe checks carry a non-rendered `readBack`: docker's from plain `doctor`'s egress lines, the - `podman` venue's from the canary its own read-back runs under this account's Podman (issue #431). Both readings must be present and must be answers: the deny probe asks for `example.com`, a host + whose three probe checks carry a non-rendered `readBack` (`{ property, want, reached, probe }`, `probe` the + slug): docker's from plain `doctor`'s egress lines, the + `podman` venue's from the canary its own read-back runs under this account's Podman (issue #431). Each of the three probes must be read exactly once, by its `probe` name, and each reading must be an answer: the deny probe asks for `example.com`, a host that resolves (a reserved `.example` name no proxy can reach read as denied behind an allow-everything - proxy), and a probe container that did not run (any exit but the script's 0 or 3) is no reading. The policy + proxy), and a probe container that did not run (any exit but the script's 0 or 3) is no reading. The third + probe (issue #508) sends plain HTTP to a listed host off port 80 (`GET http://api.anthropic.com:443/`) with raw + `node:http`, not the runner's route, which tunnels; it reads as refused only on squid's own 403 with an + `X-Squid-Error` of `ERR_ACCESS_DENIED`. A wrong reading is worded by its probe. The policy off, the proxy down or a skipped canary is not read back. - **No budget, no credential, no fix.** Nothing here reserves budget (`reserveBudget` is called only from the processor), mints a token, or puts a key in a container, and no check it renders carries a `fixAction`. @@ -2671,7 +2691,8 @@ is its only entry point (`worker/src/live-probes.mjs`, driven from `doctor.mjs`) whose `serviceIsRemote` is anything but `false`, no podman command runs and no fixture is created. On that venue with the policy armed (issue #431), the canary's probe argv is the podman builder's with `--user=` the decided job user, `--userns=keep-id` and every member of `PODMAN_PINNED_FLAGS`, and no `-v`; no docker command runs; a - provider reached and an unlisted host denied is `egress` held, an unlisted host reached fails it; given a podman + provider reached, an unlisted host denied and plain HTTP off port 80 refused is `egress` held, an unlisted host + reached or plain HTTP off port 80 let through fails it; given a podman venue refusal, a service re-read as remote, or the policy off, no canary object is made; and a dead run's canary network in this account's Podman is swept through `podman` under the same name filter. @@ -5541,3 +5562,4 @@ onFailureTimeoutMs; worker/test/on-failure.test.mjs; worker/test/start-wiring.te | 2026-09-29 | Issue #480. **`INT-EGRESS-POLICY-CONTRACT` AMENDED**, the rules bullet: `pi-dispatch init` now copies the package's `deploy/egress-proxy.conf` into a deployment folder that lacks one, create-only, so the docker proxy's `./deploy/egress-proxy.conf` mount exists in a folder made without a clone. The rules themselves, the allowlist, the proxy's argv and its mounts are UNCHANGED, checked. Being create-only, a copy init made is not refreshed when the package is upgraded, the same as a clone's file between pulls; `up`'s proxy judgement compares mount paths, not file contents, so a stale copy is not reported by it. **UNCHANGED, checked**: `INT-CONTAINER-RUNTIME-CONTRACT` (no job flag or mount moves). | | 2026-09-29 | Issue #484. **`INT-EGRESS-POLICY-CONTRACT` AMENDED**, the rules bullet: a copy of the package's `deploy/egress-proxy.conf` outside the package (the deployment folder's, which `init` writes create-only since issue #480, and the podman venue's account-owned copy `service install` writes) is compared by its bytes with the installed package's file, because an upgrade rewrites neither and the mount-path comparison `up` and `doctor` already made could not see old rules at the right path. `doctor` warns (⚠) for the shipped proxy only, silent for an identical or absent copy; `up` offers the folder copy's refresh, asked even under `--yes`, with a backup, a same-directory temp file renamed over the copy, and no symlink followed, then offers `docker restart` of a running current proxy, and `docker unpause` then `docker restart` of a paused one (PR #491's review), since a renamed file is not the one a running or paused container's bind mount holds. **UNCHANGED, checked**: the rules' content, the allowlist and its scaffold, the proxy's argv and its two mounts, the mount-path drift rule (`egress-proxy-state.mjs`), the podman Quadlet plan (it already compared and replaced the account copy under `--force`), and the job-side contract (no job flag, mount or environment variable moves). | | 2026-09-30 | Issue #509, the pi 0.80.7 -> 0.99.1 bump, one row for the runner, worker and admin halves. **`INT-SDK-SESSION-OPTIONS` AMENDED**: the model/auth wiring is one `ModelRuntime` (OQ-005's migration shipped): created after enforceOfflineMode and enforceTelemetryOff with a discarding models store and `allowModelNetwork: false`, `getModel`, `hasConfiguredAuth(provider)`, handed to createAgentSession as `modelRuntime`; the preamble and the pinned-artifact evidence are re-verified at 0.99.1, the 0.80.7 evidence kept as the record; the option set is re-pinned (modelRuntime replaces authStorage and modelRegistry) and the tool factories are eight (`powershell`). Trap (g)'s invariant now rests on the injected ModelRuntime class plus an IDENTITY check of the compat copy against pi's VIRTUAL_MODULES (the ModelRegistry mutation probe cannot work: the registry no longer writes to pi-ai's registry). Trap (h) REWRITTEN: the process-wide choke point moved to ModelRuntime.prototype, the api-provider registry is kept only for legacy extension calls, and the two halves meet on composed providers, deduplicated by an AsyncLocalStorage (measured). New trap (m): cacheWarming off, maxAgentDelayMs pinned, install telemetry off with PI_TELEMETRY forced to "0", a discarding models store, no builtin: extension loaded. Trap (f) re-worded for `SettingsManager.inMemory`, which now accepts an options argument the runner does not pass (trust default unchanged); trap (l) re-checked at 0.99.1 (open still throws on an unparseable file, measured; the session-dir variable is read only by the CLI's main.js). Traps (a) to (e), (i) to (k) UNCHANGED, checked against 0.99.1 (loader tests green). **`INT-RUNNER-EXIT-CODE-PROTOCOL` AMENDED**: three new rows, all under existing codes: pi's unresumable retry after a tool ran (prompt() rejects while a retry is in flight) is `2` / `retry-unresumable` instead of an unclassified `1` that the queue would re-run on a fresh budget (measured by the loopback test); `stopReason: "deferred"` is `2` / `deferred`; `"pending"` is `1` / `pending`. StopReason is seven values at the pin and the runner's list is pinned to the union. The Bedrock refusal reads the exception NAME `UnrecognizedClientException: ` (pi-ai now carries AWS's message and no status); AccessDeniedException and ExpiredTokenException stay residual (no real-AWS measurement of the transient forms). The KnownApi union is the same ten families. Every other row UNCHANGED, checked by the loopback table at 0.99.1. **`INT-RUN-HISTORY-FILE-CONTRACT` AMENDED in wording only**: a non-zero `otherTotal` is spend under a session id other than the root's, since a compaction or branch summary now lands there; the `tokens` keys, the `usage` ledger (its `piAi` now stamps 0.99.1), `context` and `session` UNCHANGED, checked. **`INT-CONTAINER-RUNTIME-CONTRACT` AMENDED**: anthropic's key list is now `[ANTHROPIC_AUTH_TOKEN, ANTHROPIC_OAUTH_TOKEN, ANTHROPIC_API_KEY]` and pi sends the first as `Authorization: Bearer`; `provider-key.mjs` gains the `/_AUTH_TOKEN$/` suffix beside `/_OAUTH_TOKEN$/`, so an auth.json API key is always written under `ANTHROPIC_API_KEY`, and a host bearer token is forwarded with a doctor warning exactly as a host OAuth token is; radius's variable is `RADIUS_API_KEY` and the catalog is now a superset of the key table, so the candidates-first order is defence in depth; the getProviderEnvValue order is re-checked at 0.99.1. **`INT-TRIGGERS-FILE-CONTRACT` AMENDED**: the excludable set is the eight built-ins; the steering set is re-derived (twenty-one names in, three out), `ANTHROPIC_AUTH_TOKEN` is retained by name, the OS directory variables are subtracted as runtime properties, the custom-header variables are pinned as credential substitution, and the no-baseUrl control is re-derived. **`INT-SUBSCRIPTIONS-FILE-CONTRACT` and `INT-PRICING-EXPORT-CONTRACT` AMENDED**: kimi-coding, zai and zai-coding-cn carry implied prices (all three were all-zero at 0.80.7), `isZeroRated` is no longer a subscription signal, and the pricing pins moved to openai/gpt-5.4 and the qwen-token-plan tables. UNCHANGED, checked: the auth.json refusals (OAuth, command/variable-reference, non-string, companion env), the closed container env map. Doctor also names the auth.json key a host token displaces (the worker reads the environment first, pi on the host reads the stored key first), and warns on an `sk-ant-oat` value in the API-key variable. | +| 2026-09-30 | Issue #508 (plain HTTP reached every port of a listed host). **`INT-EGRESS-POLICY-CONTRACT` AMENDED**, the rules bullet, the object table and the Acceptance: exactly two shapes pass, both to a listed host, a `CONNECT` to 443 or a plain request to 80, by `acl Safe_ports port 80` and `http_access deny !Safe_ports !CONNECT` directly before `http_access allow allowed`, which resolves nothing and never matches a `CONNECT`; a plain forward `GET https://` is now refused; the canary probe slugs are `provider`, `unlisted` and `plainhttp`; a job reaches a forge only over `https://` on 443 (git ignores `HTTP_PROXY` for `http://`), and `doctor` warns on an armed policy with a triggered forge's `GITLAB_URL` or `FORGEJO_URL` that is anything else; the Acceptance gains a plain request to a listed host on any port but 80 refused by the proxy. **`INT-LIVE-PROBE-CONTRACT` AMENDED**, the `egress` bullet and the Acceptance: three probes, each `readBack` carrying `probe` and each probe required exactly once by that name, the third a raw `node:http` forward request counted as refused only on squid's 403 with `X-Squid-Error` `ERR_ACCESS_DENIED`. | diff --git a/specs/requirements.md b/specs/requirements.md index 9a996ebb..1130e799 100644 --- a/specs/requirements.md +++ b/specs/requirements.md @@ -1555,7 +1555,8 @@ and nothing about the box itself (`INT-CONTAINER-RUNTIME-CONTRACT`). - **Statement**: What a job container reaches on the network shall be bounded **by default**, with a control pi-dispatch itself applies. Every job runs on its own `--internal` Docker network whose only other - member is an allowlist proxy, reaching listed hosts by name and nothing else beyond this host; and a job + member is an allowlist proxy, reaching listed hosts by name and nothing else beyond this host, and a listed + host only through a `CONNECT` tunnel to port 443 or a plain request to port 80 (issue #508); and a job whose policy cannot serve it shall be **refused before it spends**, with a reason naming what is wrong and no budget slot consumed. `PI_EGRESS=0` is the opt-out and takes a deployment back to the prior behaviour exactly: no `--network`, no proxy variable, no preflight spawn, and a docker argv byte-identical to one built @@ -1627,16 +1628,18 @@ and nothing about the box itself (`INT-CONTAINER-RUNTIME-CONTRACT`). a running proxy, every job's argv carries `--network=pi-job--net` and its env carries all four proxy variables; a job reaching a listed host succeeds and one reaching an unlisted host is refused by the proxy rather than by the agent, as is one reaching a listed name that resolves to one of this host's fixed loopback - or link-local addresses (issue #428), while a request the proxy refuses as unlisted costs no DNS query at all, + or link-local addresses (issue #428), and a plain (untunnelled) request to a listed host on any port but 80 + (issue #508), while a request the proxy refuses as unlisted costs no DNS query at all, forward for a name or reverse for an IP literal; and the network is removed when the container exits, or, where a worker died before it could, by the next boot's reaper, which detaches what is still attached and otherwise names the network it could not remove in the log rather than leaving it silently forever (issue #357). Given a proxy that is absent or stopped, the job returns `outcome: "policy"` with `budgetReserved: false` and reason `egress-proxy-missing` or `egress-proxy-stopped`, `docker run` is never spawned, and the queue does not retry it. Given a daemon that does not answer, the job throws and IS retried. Given a configured - deployment, `pi-dispatch doctor` reports the proxy's state and proves both directions of the policy - without spending a token, and no check it emits carries a `fixAction`; on the `podman` venue `doctor --live` - proves both directions under this account's Podman and spawns no `docker` command (issue #431). + deployment, `pi-dispatch doctor` reports the proxy's state and proves both directions of the policy, and that + plain HTTP to a listed host off port 80 is refused (issue #508), without spending a token, and no check it emits + carries a `fixAction`; on the `podman` venue `doctor --live` proves the same three under this account's Podman and + spawns no `docker` command (issue #431). ## REQ-RESUMABLE-SESSION @@ -2606,6 +2609,7 @@ instead of drifting. | Date | Change | |---|---| +| 2026-09-30 | Issue #508 (plain HTTP reached every port of a listed host). **`REQ-EGRESS-ALLOWLIST` AMENDED**, the Statement and the Acceptance: a listed host is reached only through a `CONNECT` tunnel to port 443 or a plain request to port 80, and a plain request to a listed host on any other port is refused by the proxy; `doctor` proves that refusal beside both directions, on the `podman` venue under `--live`. Port 80 stays open, not HTTPS only, because some package mirrors and redirects still use it (the owner's decision). A forge a job pushes to must still be `https://` on 443, which `doctor` warns about, because git ignores the job's `HTTP_PROXY` for `http://`. | | 2026-09-30 | Issue #509, the pi 0.80.7 -> 0.99.1 bump, one row for the runner and worker halves. **`REQ-TOKEN-ACCOUNTING-AND-CAPS` AMENDED** (mechanism, not contract): the process-wide meter's choke point moved from pi-ai's api-provider registry (no longer on a session's path) to ModelRuntime.prototype, with the registry kept for legacy extension calls; the brake covers classify/generateImages too; compaction now lands in otherTotal, and cache warming is off in every job. **`REQ-RUNNER-TURN-BUDGET` AMENDED**, evidence only: three loopback premises moved at 0.99.1 and are re-pinned (the throw after a tool ran now ends `2` / `retry-unresumable`, a queued follow-up starts no second run, the token budget's abort suppresses pi's retry); the bound and every count are UNCHANGED. **`REQ-TRIGGER-SECRETS` AMENDED**, the load-time clause: the steering set is re-derived against 0.99.1 and keeps `ANTHROPIC_AUTH_TOKEN` by name after pi made it anthropic's first key variable, so a version bump did not widen what a trigger may bind. UNCHANGED, checked: the exit-line `tokens` shape, the `usage` ledger shape and every exit code (INT-RUN-HISTORY-FILE-CONTRACT); the pre-spend per-provider reservation (it now reserves `ANTHROPIC_AUTH_TOKEN` for anthropic jobs through pi's own list); `REQ-DEPLOYMENT-BOOTSTRAP` (a host bearer token is a warning, like an OAuth token); `REQ-COST-ANALYTICS` (b) (an uncovered zero-rate run still renders `$0 (unrated)`; which providers are zero-rate changed, not the rule). | | 2026-09-29 | Issue #492, with PR #493's review folded in. **`REQ-TOPOLOGY-GRAPH` AMENDED**, clause (h): the plan-covered spend badge now holds, because the fold keeps `plan:` for a bucket every run of which ONE declared plan covers with no run a floor (it had returned only metered or estimated, so the badge read `~$0 est.` beside a by-model table saying `plan:kimi`); the badge fits its chip, cut with the whole text in a tooltip (a long plan id ran onto the wires); a narrowed close rule's label leads with its `#` (the chip cut `action[closed]…` and dropped `#40`); a one-shot trigger carries `[once]` or `[spent]` on its chip. **`REQ-COST-ANALYTICS` UNCHANGED, checked**: its (b) is the rule this makes true; its (c) still demotes any bucket mixing plan and billed runs to estimated with coverage; its (d) holds because a plan bucket holding a floor stays estimated with its `≥` (the review found the first cut dropped it: a fallback-metered run cannot see subagent spend, and `plan:` has no place for `≥`). A bucket two plans cover stays estimated too: the subscriptions validator accepts any non-empty id, so no separator could join two unambiguously (pinned against the parser). **`REQ-INSIGHTS-HTML-EXPORT` UNCHANGED, checked**: its (b) (a plan bucket draws a chip, no dollar bar) now applies to the trigger, flow and repo tables whenever a plan covers the whole bucket, which it always required. **Code evidence**: admin/src/costs.mjs (combineContributions); admin/src/graph-model.mjs (triggerMatchLabel); admin/src/graph-html.mjs (the status line); admin/test/costs.test.mjs, graph-html.test.mjs, insights-html.test.mjs (the issue #492 pins). | | 2026-09-29 | Issue #484 (the folder's `egress-proxy.conf` went stale after an upgrade, and nothing said so). **`REQ-DEPLOYMENT-BOOTSTRAP` AMENDED**, one clause after the #480 scaffold: `doctor` warns (⚠) when the folder's `deploy/egress-proxy.conf`, or the podman venue's account-owned copy, differs from the installed package's file, for the shipped proxy with the policy armed, and is silent for an identical or absent copy; `up` offers the folder copy's refresh before its proxy step, asked even under `--yes` (the one action `--yes` does not accept, since the difference may be the operator's own edit), keeping a timestamped backup, writing through a same-directory temp file and a rename, refusing a symlinked copy or `deploy/`, and then offers `docker restart` of a running current proxy, and `docker unpause` then `docker restart` of a paused one (PR #491's review: an unpause alone resumed squid on the old file while `up` had said the rules were replaced), which `--yes` accepts unless job networks are attached. A copy that differs only in its line endings is said to. A declined refresh changes nothing. **UNCHANGED, checked**: `init` (still create-only; it never refreshes), `REQ-EGRESS-ALLOWLIST` (the rules' content and the policy), the consent contract for every other `up` action, and `service install`, which already compared the account copy and replaced it only under `--force`. | diff --git a/worker/deploy/egress-proxy.conf b/worker/deploy/egress-proxy.conf index e46ddcc0..1bcb72d9 100644 --- a/worker/deploy/egress-proxy.conf +++ b/worker/deploy/egress-proxy.conf @@ -6,9 +6,11 @@ # misordered one silently allows everything, so the file an operator edits contains no ordering at all. # # What this policy is, and is not: -# - Hostname filtering on CONNECT, to port 443 only. The provider, the forge and the registry are -# ordinary entries in the allowlist, and nothing is special. The one address-based rule is a DENY: -# a listed name that resolves to a fixed loopback or link-local address of this host is refused. +# - Hostname filtering, and exactly two shapes pass (issue #508): a CONNECT tunnel to a listed host +# on port 443, or a plain (untunnelled) request to a listed host on port 80. Nothing else. The +# provider, the forge and the registry are ordinary entries in the allowlist, and nothing is +# special. The one address-based rule is a DENY: a listed name that resolves to a fixed loopback +# or link-local address of this host is refused. # - TLS is NEVER terminated. This proxy sees the name a client asks for and no byte inside the tunnel, # so it cannot read a credential and cannot account for a token. A proxy that decrypts provider # traffic is OQ-011's mechanism, a materially larger change, and it is not this. @@ -29,6 +31,15 @@ http_port 3128 acl allowed dstdomain -n "/etc/pi-dispatch/allowlist.conf" acl SSL_ports port 443 +# Stock squid's name, narrowed to one port (issue #508). Without it `allow allowed` below let any +# tool that forwards plain HTTP (npm undici's EnvHttpProxyAgent from 8.7 on without proxyTunnel, a +# package manager, a curl -x a job brings) reach ANY port of a listed host, in clear text. What every +# job has never takes this path: pi and the runner tunnel every origin, Node 22's built-in fetch sends a +# CONNECT even for http://, and the job image's git ignores HTTP_PROXY for an http:// remote. A side +# effect: a plain forward `GET https://host/`, where this proxy would open the TLS itself, is refused +# too, since it names port 443 without a CONNECT. Measured on Docker and rootless Podman 4.9.3 and 5.8.1 +# (docs/egress.md): port 8080 went from 200 to 403, port 80 stayed 200. +acl Safe_ports port 80 acl CONNECT method CONNECT # The host's own services, by address (issue #428). On rootless Podman an account's containers.conf @@ -55,9 +66,14 @@ acl to_host_local dst 127.0.0.0/8 0.0.0.0/32 169.254.0.0/16 10.0.2.2/32 ::1 ::/1 # debug_options 78,3: a CONNECT to an unlisted name sent a DNS query, where without the rule it sent # none), which is a DNS channel out of an egress-armed job. With `allowed` first, only a listed name # is ever resolved here, and it is the only kind the allows below could let through anyway. +# The port rule for plain requests (issue #508) sits right before the allow it narrows: first match +# wins, so after it it would never apply. It names only port and method ACLs, which resolve nothing, +# so no unlisted name is resolved by it. It never matches a CONNECT, so the tunnel rules (including a +# CONNECT allow later placed between the first two rules, see #503) are untouched by it. http_access deny allowed to_host_local http_access deny CONNECT !SSL_ports http_access allow CONNECT allowed +http_access deny !Safe_ports !CONNECT http_access allow allowed http_access deny all diff --git a/worker/src/doctor.mjs b/worker/src/doctor.mjs index f5f1617e..d51471e0 100644 --- a/worker/src/doctor.mjs +++ b/worker/src/doctor.mjs @@ -80,7 +80,7 @@ import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_FIRST_START_TIMEOUT_MS, PODMAN_INFO import { PODMAN_PINNED_FLAGS, buildPodmanRunArgs, containerSpec, podmanArgsFromSpec } from "./docker-run.mjs"; import { PODMAN_SERVICE_TIMEOUT_MS, PODMAN_SERVICE_UNIT, makePodmanServiceReader, observeHost, observeRootfulConf, readRootfulService, rootfulConfFix, rootfulConfRetries, rootfulConfResidual, rootfulUnreadList } from "./runtime-observations.mjs"; import { endpointShown, makeDockerEndpointResolver, quotedShown } from "./backend-local.mjs"; -import { DEFAULT_EGRESS_PROXY, STOPPED_PROXY_STATES, EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEnv, egressProxyName, networkEndpoints, removeNetworkOrSay } from "./egress.mjs"; +import { DEFAULT_EGRESS_PROXY, STOPPED_PROXY_STATES, EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEnv, egressProxyName, egressProxyUrl, networkEndpoints, removeNetworkOrSay } from "./egress.mjs"; import { detachBlockedSentence, makeDetachGate, runtimeFromFacts } from "./netns-keeper.mjs"; import { runLiveProbes } from "./live-probes.mjs"; import { VALKEY_PASSWORD_KEY, VALKEY_PASSWORD_HOWTO, VALKEY_PORT_KEY, isLoopbackHost, valkeyPasswordProblem, valkeyPortConflict } from "./valkey-auth.mjs"; @@ -1733,6 +1733,9 @@ export async function collectChecks(shellVars, seams) { } } + // Issue #508: with the egress policy on, a job reaches a forge only over https on port 443. + checks.push(...forgeUrlEgressChecks(env, forges)); + // Azure DevOps, when the triggers file names it (issue #80) -- same shape, mirrored from // receiver/src/config.mjs loadAzureConfig. AZURE_WEBHOOK_MODE gets its own line because it is // required-UNDEFAULTED: Azure offers no HMAC at all, so both modes are shared-secret compares, and @@ -4190,7 +4193,7 @@ export async function sweepStaleCanaryNetworks({ run, pid, isAlive, endpoint, bi // gets added to the producer and not to the reaper. // // ASYMMETRY ON PURPOSE, recorded because it looks like an oversight three characters apart: the OWNER is - // escaped and the slugs are interpolated raw. `CANARY_PROBE_SLUGS` is a frozen literal of two plain words + // escaped and the slugs are interpolated raw. `CANARY_PROBE_SLUGS` is a frozen literal of three plain words // three lines below, so today there is nothing to escape and escaping it would say the set is untrusted // when it is this file's own. It is here so that whoever adds a slug with a `.` or a `-` in it sees the // obligation: a metacharacter there widens what this `rm -f` matches (issue #360, item 6). @@ -4306,8 +4309,55 @@ export async function sweepStaleCanaryNetworks({ run, pid, isAlive, endpoint, bi return checks; } -/** The canary's two probe directions. ONE list: the loop names its containers from it and the sweep matches on it. */ -export const CANARY_PROBE_SLUGS = Object.freeze(["provider", "unlisted"]); +/** + * Issue #508: with the egress policy on, a forge must be served over https on port 443, the one shape a job's git reaches + * it by. git never uses the job's proxy for an `http://` remote: the job gets the proxy variables in UPPERCASE only, and + * libcurl ignores `HTTP_PROXY` for `http://` (measured in the job image, git 2.39.5: `git ls-remote http://...` went to + * DNS, and the proxy saw nothing), so an http:// forge fails behind the `--internal` network on ANY port, 80 included. + * An `https://` remote is a CONNECT, which the proxy refuses to any port but 443 (a rule older than #508). The clone + * itself runs on the host before the container exists, so it is the job's git push and fetch that fail, and its API + * calls too unless the forge is on port 80: glab and tea (Go) do use `HTTP_PROXY` for `http://`, and the proxy passes + * plain HTTP to a listed host on port 80 (round 3 of #508's review). Any other scheme reaches nothing. One ⚠ + * per such URL, for a forge the triggers file names; nothing with the policy off or unreadable (the .env check reports + * a malformed PI_EGRESS). + */ +export function forgeUrlEgressChecks(env, forges = []) { + let armed; + try { + armed = egressArmed(env); + } catch { + return []; + } + if (armed !== true) return []; + const checks = []; + for (const [forge, key] of [["gitlab", "GITLAB_URL"], ["forgejo", "FORGEJO_URL"]]) { + if (!forges.includes(forge) || typeof env[key] !== "string") continue; + let url; + try { + url = new URL(env[key].trim()); + } catch { + continue; + } + // The WHATWG parser drops a default port, so an empty port on https: is 443. + if (url.protocol === "https:" && url.port === "") continue; + const why = + url.protocol === "https:" + ? `https:// on port ${url.port}, and the egress proxy refuses a CONNECT to any port but 443, so with the egress policy on every job's push, fetch and API call to it fails` + : url.protocol === "http:" + ? `http://, so with the egress policy on a job's git push and fetch fail (git never sends http:// through the proxy)${url.port === "" ? "" : `, and on port ${url.port}, not 80, its API calls fail too`}` + : "not an https:// URL, so with the egress policy on a job cannot reach it"; + checks.push({ + ok: false, + warn: true, + label: `${key} (${urlShown(env[key])}) is ${why} (issue #508)`, + fix: `serve the forge over https:// on port 443 and point ${key} there, or set PI_EGRESS=0 if you accept jobs without the policy (docs/egress.md)`, + }); + } + return checks; +} + +/** The canary's three probes. ONE list: the loop names its containers from it and the sweep matches on it. */ +export const CANARY_PROBE_SLUGS = Object.freeze(["provider", "unlisted", "plainhttp"]); /** Where the job image's runner keeps the module its own provider call goes through (issue #427). */ export const EGRESS_CANARY_RUNNER_MODULE = "/app/image/runner/src/env-proxy.mjs"; @@ -4331,6 +4381,21 @@ export function egressCanaryScript(url) { return `import(${JSON.stringify(EGRESS_CANARY_RUNNER_MODULE)}).then(m=>m.loadPiThenRestore(),e=>{console.log("error",e.code??e.message);process.exit(e.code==="ERR_MODULE_NOT_FOUND"?${EGRESS_CANARY_STALE_RUNNER}:5)}).then(()=>fetch(${JSON.stringify(url)},{method:"POST"}).then(r=>{console.log("reached",r.status);process.exit(0)},e=>{console.log("blocked",e.cause?.code??e.message);process.exit(3)}),e=>{console.log("error",e.message);process.exit(5)})`; } +/** + * The third probe's in-container script (issue #508): ONE plain forward request, `GET `, written straight + * to `proxyUrl`, which the caller passes as `egressProxyUrl(proxy)`: the value both venues' probe argv put in + * HTTPS_PROXY. Passed rather than read from the container's environment because this file's text is pinned to read no + * environment by name outside the resolver (issue #471), and a script string is text. Raw `node:http` with `agent: false`, and NOT the + * runner's route (`loadPiThenRestore`, `fetch`): those tunnel every origin, so they would send a CONNECT, which is + * the path the other two probes read, never the plain one a curl or a package manager takes. Exit 3 only for squid's + * own refusal (403 with an `X-Squid-Error` of ERR_ACCESS_DENIED), 0 for any other answer (the request went on: a 400 + * from a TLS port spoken to in clear, or squid's 502 when the far end hung up, both mean the proxy let it through), + * and 5 for a proxy that could not be spoken to or never answered, which is no reading at all. + */ +export function egressCanaryPlainScript(url, { proxyUrl, timeoutMs = 15_000 } = {}) { + return `import("node:http").then(h=>{let p,u;try{p=new URL(${JSON.stringify(String(proxyUrl))});u=new URL(${JSON.stringify(url)})}catch(e){console.log("error",e.message);process.exit(5)}const q=h.request({host:p.hostname,port:p.port||80,method:"GET",path:u.href,headers:{Host:u.host},agent:false,timeout:${Number(timeoutMs)}},r=>{const x=String(r.headers["x-squid-error"]??"");r.resume();if(r.statusCode===403&&x.startsWith("ERR_ACCESS_DENIED")){console.log("blocked",r.statusCode,x);process.exit(3)}console.log("reached",r.statusCode,x);process.exit(0)});q.on("timeout",()=>{console.log("error","timeout");process.exit(5)});q.on("error",e=>{console.log("error",e.code??e.message);process.exit(5)});q.end()},e=>{console.log("error",e.message);process.exit(5)})`; +} + /** * The bound on one canary docker step. Shorter than the 30 s the PROBES get, because these are `network` * calls that either answer at once or are wedged, and the teardown must not be the slow part of a doctor run. @@ -4592,7 +4657,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo } // The end-to-end probe, and the only place in this codebase that proves the policy rather than - // inspecting it. Two containers, on a throwaway network built exactly like a job's, gated on the image + // inspecting it. Three containers, on a throwaway network built exactly like a job's, gated on the image // being present because it uses the job image's own node and runner module -- which is the point: it proves the // operator's OWN image routes a request through the proxy the way its runner does (pi loaded, then the runner's // restore), the property a stale image would silently lack and the one whose absence turns the whole policy into an @@ -4619,6 +4684,9 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo * docker's is what it always was: a plain `docker run` on the canary network with the two proxy variables, which is * pinned byte for byte and not widened here, because moving it is its own change with its own review. * + * `script` is what the container runs, the runner's route (`egressCanaryScript`) unless a probe says otherwise: the + * plain HTTP probe (issue #508) passes `egressCanaryPlainScript`, and its argv differs from the others in nothing else. + * * podman's is a JOB's, built by the podman venue's own builder (`podmanArgsFromSpec` over `containerSpec`, the path * `buildPodmanRunArgs` takes), never a hand-rolled argv: `ISOLATION_FLAGS`, the job's memory and cpu bounds, * the job user this host decides as `--user`, `--userns=keep-id` and `PODMAN_PINNED_FLAGS`, and a job's own egress @@ -4635,7 +4703,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo * then replaced by none; `CANARY_NO_WORKSPACE` is a path nothing creates, so if a later edit ever kept the mount, the * run would fail on a missing source rather than bind a real directory. */ -export function egressCanaryProbeArgs({ bin = "docker", slug, pid, network, proxy, image, url, user = null }) { +export function egressCanaryProbeArgs({ bin = "docker", slug, pid, network, proxy, image, url, user = null, script = egressCanaryScript(url) }) { const name = egressCanaryProbe(slug, pid); if (bin !== "podman") { return [ @@ -4657,14 +4725,14 @@ export function egressCanaryProbeArgs({ bin = "docker", slug, pid, network, prox "node", image, "-e", - egressCanaryScript(url), + script, ]; } // HOME as a podman job gets it (`resolvePodmanImageUser` always answers CONTAINER_HOME): under keep-id the job user's // passwd entry otherwise names this host's home path, which does not exist in the image, and the canary loads pi as // that user. No credential rides along: the canary proves the route, and a 401 from the provider is its success. const { mounts: _placeholder, ...spec } = containerSpec({ image, name, env: { HOME: CONTAINER_HOME, ...egressEnv({ proxy, armed: true }) }, workspace: CANARY_NO_WORKSPACE, network, user, userns: "keep-id", extraFlags: ["--entrypoint", "node"] }); - return [...podmanArgsFromSpec({ ...spec, mounts: [] }), "-e", egressCanaryScript(url)]; + return [...podmanArgsFromSpec({ ...spec, mounts: [] }), "-e", script]; } /** The workspace `containerSpec` requires and the canary never mounts (see `egressCanaryProbeArgs`). */ @@ -4672,10 +4740,11 @@ const CANARY_NO_WORKSPACE = "/nonexistent/pi-dispatch-egress-canary-mounts-nothi /** * The egress canary (REQ-EGRESS-ALLOWLIST, issue #431): a throwaway `--internal` network built like a job's, the proxy - * attached, two probe containers running `egressCanaryScript` (the runner's own route to the network, #427), one that - * must reach the provider and one that must not reach an unlisted host, and a teardown that removes everything it made - * or names what it could not. Returns `{ checks, results }`: the lines to print, and the `[{ property, want, reached }]` - * readings `egressVerdict` folds into `--live`. + * attached, three probe containers, and a teardown that removes everything it made or names what it could not. Two run + * `egressCanaryScript` (the runner's own route to the network, #427): one must reach the provider and one must not reach + * an unlisted host. The third runs `egressCanaryPlainScript` and must not get plain HTTP through to a listed host on a + * port other than 80 (issue #508). Returns `{ checks, results }`: the lines to print, and the + * `[{ property, want, reached, probe }]` readings `egressVerdict` folds into `--live`, `probe` being the slug. * * ONE canary for every caller: docker's `doctor` (`egressChecks`), the podman venue's `doctor --live` * (`podmanLiveChecks`) and `.github/scripts/podman-conformance.mjs`. The conformance script used to carry a canary of @@ -4756,11 +4825,17 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = // which no proxy can reach, so a proxy allowing every host still read as denying this one (measured: an // allow-all squid answered 503 for it and let `example.com` through). `example.com` is reserved for documentation // (RFC 2606), answers everywhere, and is contacted only when the proxy lets the request out, which is the finding. - for (const [slug, host, url, want] of [ + // + // The THIRD probe (issue #508) asks the proxy for plain HTTP to a LISTED host on a port other than 80, as a client + // that forwards rather than tunnels would (npm undici's EnvHttpProxyAgent without proxyTunnel, a package manager). Port 443 of the provider because it is a + // port that host certainly listens on, so a proxy that lets it through gets an answer (a 400 for clear text on a + // TLS port) rather than a timeout, and no second host has to be listed for the canary. + for (const [slug, host, url, want, script] of [ [CANARY_PROBE_SLUGS[0], "the provider", "https://api.anthropic.com/v1/messages", true], [CANARY_PROBE_SLUGS[1], "an unlisted host", "https://example.com/", false], + [CANARY_PROBE_SLUGS[2], "plain HTTP to a listed host off port 80", "http://api.anthropic.com:443/", false, egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: egressProxyUrl(proxy) })], ]) { - const answer = await probe(egressCanaryProbeArgs({ bin, slug, pid, network: net, proxy, image, url, user })); + const answer = await probe(egressCanaryProbeArgs({ bin, slug, pid, network: net, proxy, image, url, user, ...(script ? { script } : {}) })); // The script exits 0 (reached) or 3 (blocked). Anything else is the container not running it -- a name clash, // the image, the daemon -- which is no reading at all, and must not pass for a deny. // `code === null` is the ONE case where a container may still be RUNNING under a name we chose: the @@ -4772,7 +4847,7 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = // nothing of ours to remove. Harmless either way today (`docker rm -f ` exits 0, measured), // and written out because the conflation it removes is the one this item is about. if (answer.code === null && answer.ended !== "error") unfinished.push(egressCanaryProbe(slug, pid)); - // Said ONCE and the second probe not run: the unlisted host's "blocked" would come from the internal network + // Said ONCE and the later probes not run: the unlisted host's "blocked" would come from the internal network // refusing a runner that never tried the proxy, which reads exactly like a policy that denies it. if (answer.code === EGRESS_CANARY_STALE_RUNNER) { checks.push({ @@ -4780,7 +4855,7 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = warn: true, label: `${venue.prefix}Egress policy: not proved, because the job image could not find ${EGRESS_CANARY_RUNNER_MODULE} (or an import of it): its runner predates issue #427, whose provider call goes around the proxy so that with egress armed every job fails at its first turn, or the image is not built from this project's`, fix: `use a job image built after issue #427 (ghcr.io/edgehero/pi-job:latest, or rebuild yours FROM it), or set PI_EGRESS=0 until you can`, - readBack: { property: "egress", want, reached: null }, + readBack: { property: "egress", want, reached: null, probe: slug }, }); break; } @@ -4790,17 +4865,31 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = warn: true, label: `${venue.prefix}Egress policy probe for ${host} did not run (${answer.code === null ? `${bin} run did not finish` : `${bin} run exited ${answer.code}`})`, fix: "re-run doctor; if it persists, run the job image by hand to see why a container on this network will not start", - readBack: { property: "egress", want, reached: null }, + readBack: { property: "egress", want, reached: null, probe: slug }, }); continue; } const reached = answer.code === 0; + if (slug === CANARY_PROBE_SLUGS[2]) { + checks.push({ + ok: reached === want, + warn: true, + readBack: { property: "egress", want, reached, probe: slug }, + label: `${venue.prefix}${ + reached === want + ? "Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)" + : "Egress policy lets plain HTTP through to api.anthropic.com on port 443, so a client that does not tunnel reaches every port of a listed host" + }`, + fix: `the proxy runs rules from before issue #508: restart it on the current egress-proxy.conf (\`pi-dispatch up\`; on the podman venue \`pi-dispatch service install --force\`; for a proxy you started by hand, update the egress-proxy.conf it mounts, then \`${bin} restart ${proxy}\`); a copy you keep by hand must add "http_access deny !Safe_ports !CONNECT" right before "http_access allow allowed"`, + }); + continue; + } checks.push({ ok: reached === want, warn: true, // NOT rendered: what `doctor --live` folds into its egress read-back (live-probes.mjs), so the canary is // run once and read twice rather than a second canary built beside it. - readBack: { property: "egress", want, reached }, + readBack: { property: "egress", want, reached, probe: slug }, label: `${venue.prefix}${ reached === want ? want @@ -5952,7 +6041,7 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" }) // an image copies its layers (27 to 32 s measured), and `--live` is where this venue already starts containers // built like a job's. if (proxyRunning && seams.live !== true) { - checks.push({ ok: false, warn: true, label: "podman: the egress allowlist is read back by `pi-dispatch doctor --live` on this venue, not by this run", fix: "run `pi-dispatch doctor --live`: its egress canary runs two containers built like a podman job's (the job user, --userns=keep-id, the venue's pinned flags) on a job-shaped --internal network under this account's Podman, one that must reach the provider through the proxy and one that must not reach an unlisted host" }); + checks.push({ ok: false, warn: true, label: "podman: the egress allowlist is read back by `pi-dispatch doctor --live` on this venue, not by this run", fix: "run `pi-dispatch doctor --live`: its egress canary runs three containers built like a podman job's (the job user, --userns=keep-id, the venue's pinned flags) on a job-shaped --internal network under this account's Podman, one that must reach the provider through the proxy, one that must not reach an unlisted host, and one that must not get plain HTTP through to a listed host off port 80" }); } // Issue #484: the account-owned copy of the rules the shipped unit mounts, against the installed package's. `service // install` compares it on every run and replaces a differing one only under --force, and `up` installs nothing @@ -6473,7 +6562,8 @@ async function podmanEgressCanary({ podman, readInfo, run, image, pid, isAlive, if (podman.egress.proxyRunning !== true || podman.imagePresent !== true) return { checks, results: [] }; // Announced, as `runLiveProbes` announces its own containers: these start before it, and a cold first keep-id start // can take half a minute each, which is a long silence on an operator's terminal. - announce(`starting ${CANARY_PROBE_SLUGS.map((slug) => egressCanaryProbe(slug, pid)).join(" and ")} from ${image} (as the job user ${podman.user}) on the --internal network ${egressCanaryNetwork(pid)}, with ${podman.egress.proxy} attached, to read the egress allowlist back; both are removed when the canary ends`); + const probes = CANARY_PROBE_SLUGS.map((slug) => egressCanaryProbe(slug, pid)); + announce(`starting ${probes.slice(0, -1).join(", ")} and ${probes.at(-1)} from ${image} (as the job user ${podman.user}) on the --internal network ${egressCanaryNetwork(pid)}, with ${podman.egress.proxy} attached, to read the egress allowlist back; all three are removed when the canary ends`); const canary = await runEgressCanary({ run, bin: "podman", proxy: podman.egress.proxy, image, pid, user: podman.user, gate }); return { checks: [...checks, ...canary.checks], results: canary.results }; } diff --git a/worker/src/live-probes.mjs b/worker/src/live-probes.mjs index f6470ece..96436737 100644 --- a/worker/src/live-probes.mjs +++ b/worker/src/live-probes.mjs @@ -452,10 +452,11 @@ export function imagePinningVerdict({ code, output, stillAbsent, bin = "docker" } /** - * EGRESS, folded in from `doctor.mjs`'s canary: two containers on a job-shaped network behind the proxy, one that - * must reach the provider and one that must not reach an unlisted host. `results` is `[{ want, reached }]` from the - * canary's `readBack`; anything short of both readings -- the policy off, the proxy down, the canary skipped -- is - * "not read back", never a pass. + * EGRESS, folded in from `doctor.mjs`'s canary: three containers on a job-shaped network behind the proxy, one that + * must reach the provider, one that must not reach an unlisted host, and one that must not get plain HTTP through to a + * listed host on a port other than 80 (issue #508). `results` is `[{ want, reached, probe }]` from the canary's + * `readBack`, `probe` naming which of the three it is; anything short of all three readings -- the policy off, the + * proxy down, the canary skipped -- is "not read back", never a pass. * * Each venue hands in its OWN canary's readings: docker's from doctor's egress lines, the podman venue's from the canary * its `--live` runs under Podman (issue #431), so "see the egress lines above" names lines about the proxy that venue's @@ -466,15 +467,29 @@ export function egressVerdict({ armed, results, keeperBlocked = null }) { if (armed !== true) return notReadBack("egress", "PI_EGRESS is off, so there is no policy to read back"); // Issue #458: the podman venue on Podman 4.x without its keeper runs no canary, since its teardown would break the proxy. if (keeperBlocked) return notReadBack("egress", `the egress canary was not run, because ${keeperBlocked} (see the keeper line above)`); - if (!Array.isArray(results) || results.length < 2) return notReadBack("egress", "the egress canary did not run both probes (see the egress lines above)"); - // A WRONG reading fails first, whatever else is missing: an unlisted host that was reached is a finding even when - // the provider probe did not run, and reporting it as merely unread would pass doctor over it. + // Each of the three probes exactly once, by NAME (issue #508, gate round 1): a count alone read two unlisted readings, + // or three with no probe named, as a canary that ran all three. + const probes = Array.isArray(results) ? results.map((r) => r?.probe) : []; + if (probes.length !== EGRESS_PROBES.length || !EGRESS_PROBES.every((p) => probes.includes(p))) return notReadBack("egress", "the egress canary did not run all three probes (see the egress lines above)"); + // With all three probes present, a WRONG reading fails first, whatever else did not run to an answer: an unlisted + // host that was reached is a finding even when the provider probe did not run, and reporting it as merely unread + // would pass doctor over it. A probe missing altogether is "not read back" above, before any reading is judged. const wrong = results.filter((r) => typeof r.reached === "boolean" && r.reached !== r.want); - if (wrong.length > 0) return verdict("egress", false, wrong.map((r) => (r.want ? "the provider was not reached" : "an unlisted host was reached")).join("; ")); + if (wrong.length > 0) return verdict("egress", false, wrong.map((r) => EGRESS_WRONG[r.probe]).join("; ")); if (results.some((r) => typeof r.reached !== "boolean")) return notReadBack("egress", "an egress probe did not run to an answer (see the egress lines above)"); - return verdict("egress", true, "the provider was reached and an unlisted host was not"); + return verdict("egress", true, "the provider was reached, an unlisted host was not, and plain HTTP off port 80 was refused"); } +/** The canary's probes: `CANARY_PROBE_SLUGS` in doctor.mjs, which imports this module, so a test pins the two equal. */ +export const EGRESS_PROBES = Object.freeze(["provider", "unlisted", "plainhttp"]); + +/** What a wrong reading says, by the canary probe it came from. */ +const EGRESS_WRONG = Object.freeze({ + provider: "the provider was not reached", + unlisted: "an unlisted host was reached", + plainhttp: "plain HTTP to a listed host off port 80 was let through", +}); + /** * EPHEMERAL (issue #344): two runs under ONE name, each detached with `--rm`, each waited on until `docker ps -a` no * longer lists it. `first`/`second` are `{ started, id, removal: { state, ms } }` (`state` one of `gone`, `exited` diff --git a/worker/test/doctor.test.mjs b/worker/test/doctor.test.mjs index 8bc4b0ae..b8a4e2bd 100644 --- a/worker/test/doctor.test.mjs +++ b/worker/test/doctor.test.mjs @@ -6,13 +6,13 @@ import { makeWaitChecker } from "../src/wait-check.mjs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { PassThrough } from "node:stream"; -import { startAdvice, CANARY_LINES, CANARY_PROBE_SLUGS, DOCTOR_SHELL_KEYS, EGRESS_CANARY_RUNNER_MODULE, EGRESS_CANARY_STALE_RUNNER, ENV_FILE_READABLE_KEYS, RECEIVER_SERVICE_KEYS, RUN_TIMEOUTS, SERVICE_ENV_KEYS, STEERING_SERVICE_KEYS, WORKER_SERVICE_KEYS, backendChecks, collectChecks, countRetained, defaultPromptFn, dockerRunVia, egressCanaryProbeArgs, egressCanaryScript, envFileKeys, githubProtectionPreflight, jobUserChecks, liveChecks, liveRunVia, podmanLiveChecks, render, runDoctor, sandboxTombstoneChecks, serviceEnvKeys, serviceEnvLoader, sweepStaleCanaryNetworks, urlShown, resolveDoctorEnv, jobImageOf, CLI_SERVICE_KEYS, cliNotHandedLines, fileConfigures, triggersPath, valkeyPasswordUpgradeStep } from "../src/doctor.mjs"; +import { startAdvice, CANARY_LINES, CANARY_PROBE_SLUGS, DOCTOR_SHELL_KEYS, EGRESS_CANARY_RUNNER_MODULE, EGRESS_CANARY_STALE_RUNNER, ENV_FILE_READABLE_KEYS, RECEIVER_SERVICE_KEYS, RUN_TIMEOUTS, SERVICE_ENV_KEYS, STEERING_SERVICE_KEYS, WORKER_SERVICE_KEYS, backendChecks, collectChecks, countRetained, defaultPromptFn, dockerRunVia, egressCanaryPlainScript, egressCanaryProbeArgs, forgeUrlEgressChecks, egressCanaryScript, envFileKeys, githubProtectionPreflight, jobUserChecks, liveChecks, liveRunVia, podmanLiveChecks, render, runDoctor, sandboxTombstoneChecks, serviceEnvKeys, serviceEnvLoader, sweepStaleCanaryNetworks, urlShown, resolveDoctorEnv, jobImageOf, CLI_SERVICE_KEYS, cliNotHandedLines, fileConfigures, triggersPath, valkeyPasswordUpgradeStep } from "../src/doctor.mjs"; import { valkeyPasswordFor } from "../src/valkey-endpoint.mjs"; import { serviceEnvFileOf } from "../src/service-env.mjs"; import { VALKEY_SHARED_KEY as VALKEY_SHARED_NAME } from "../src/podman-stack.mjs"; import { EMPTY_PAUSE_WINDOWS, EMPTY_SCOPED_LIMITS, runInit } from "../src/init.mjs"; import { EGRESS_CANARY_NET_PREFIX, egressCanaryProbe } from "../src/egress.mjs"; -import { LIVE_PREFIX } from "../src/live-probes.mjs"; +import { LIVE_PREFIX, egressVerdict } from "../src/live-probes.mjs"; import { JOB_USER_FIX, parseDaemonFacts } from "../src/job-user.mjs"; import { OBSERVATION_FIX } from "../src/backends.mjs"; import { PODMAN_JOB_USER_FIX } from "../src/backend-podman.mjs"; @@ -155,6 +155,8 @@ const EGRESS_OK = { "docker network": 0, "docker run --rm --name pi-dispatch-egress-probe-provider": 0, "docker run --rm --name pi-dispatch-egress-probe-unlisted": 3, + // Issue #508: plain HTTP to a listed host off port 80 is refused by the proxy (exit 3). + "docker run --rm --name pi-dispatch-egress-probe-plainhttp": 3, }; const green = { ...EGRESS_OK, "docker info": 0, "docker image": 0 }; // A classic-token `gh auth status` (newer gh quotes each scope; the parser also accepts unquoted). @@ -4618,7 +4620,7 @@ test("an egress probe that did not RUN is reported as not run, and never passes const checks = await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams({ ...green, "docker run --rm --name pi-dispatch-egress-probe-unlisted": 125 })); assert.ok(checks.some((c) => /Egress policy probe for an unlisted host did not run \(docker run exited 125\)/.test(c.label) && c.warn === true)); assert.ok(!checks.some((c) => /denies an unlisted host/.test(c.label))); - assert.deepEqual(checks.filter((c) => c.readBack).map((c) => c.readBack.reached), [true, null]); + assert.deepEqual(checks.filter((c) => c.readBack).map((c) => c.readBack.reached), [true, null, false]); }); test("a job image whose runner predates issue #427 is named once, and neither direction is read as the policy", async () => { @@ -4631,6 +4633,7 @@ test("a job image whose runner predates issue #427 is named once, and neither di assert.match(stale[0].fix, /built after issue #427/); assert.ok(calls.some((c) => c.args.some((a) => /egress-probe-provider/.test(a))), "the provider probe ran"); assert.ok(!calls.some((c) => c.args.some((a) => /egress-probe-unlisted/.test(a))), "the deny probe is not run: its block would be the network's, not the policy's"); + assert.ok(!calls.some((c) => c.args.some((a) => /egress-probe-plainhttp/.test(a))), "nor the plain HTTP probe (#508): the canary stops at the first stale reading"); assert.ok(!checks.some((c) => /reaches the provider|denies an unlisted host|did not run/.test(c.label))); assert.deepEqual(checks.filter((c) => c.readBack).map((c) => c.readBack.reached), [null]); assert.ok(calls.some((c) => c.args.slice(0, 2).join(" ") === "network rm"), "and the canary network is still removed"); @@ -4655,6 +4658,161 @@ test("the canary script's exits are real: 4 only for a missing module, 3 for a b assert.equal(run(join(dir, "syntax.mjs")).status, 5); }); +// Issue #508: the plain HTTP probe's script, against a fake proxy in this process. It must send a plain forward request +// (absolute URI, never a CONNECT), and only squid's own refusal (403 + X-Squid-Error ERR_ACCESS_DENIED) counts as denied. +test("the plain HTTP canary script sends a forward GET, and only squid's access denial is a block (#508)", async () => { + const { createServer } = await import("node:http"); + const { spawn: spawnChild } = await import("node:child_process"); + const runAgainst = (answer) => + new Promise((resolve) => { + const seen = []; + const server = createServer((req, res) => { + seen.push(`${req.method} ${req.url} HTTP/${req.httpVersion}`, req.headers.host); + answer?.(res); + }); + server.on("connect", (req, socket) => { + seen.push(`CONNECT ${req.url}`); + socket.destroy(); + }); + server.listen(0, "127.0.0.1", () => { + const script = egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: `http://127.0.0.1:${server.address().port}`, timeoutMs: 300 }); + const child = spawnChild(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + child.stdout.on("data", (d) => (stdout += d)); + child.on("close", (status) => { + server.closeAllConnections(); + server.close(() => resolve({ status, stdout, seen })); + }); + }); + }); + const denied = await runAgainst((res) => res.writeHead(403, { "X-Squid-Error": "ERR_ACCESS_DENIED 0" }).end("no")); + assert.equal(denied.status, 3, denied.stdout); + assert.deepEqual(denied.seen, ["GET http://api.anthropic.com:443/ HTTP/1.1", "api.anthropic.com:443"], "a plain forward request, absolute URI, never a CONNECT"); + assert.match(denied.stdout, /^blocked 403 ERR_ACCESS_DENIED/); + for (const [why, answer] of [ + ["a TLS port spoken to in clear", (res) => res.writeHead(400).end()], + ["squid let it out and the far end hung up", (res) => res.writeHead(502, { "X-Squid-Error": "ERR_ZERO_SIZE_OBJECT 0" }).end()], + ["a 403 that is not squid's", (res) => res.writeHead(403).end()], + ]) { + const got = await runAgainst(answer); + assert.equal(got.status, 0, `${why}: ${got.stdout}`); + assert.match(got.stdout, /^reached /, why); + } + const silent = await runAgainst(null); + assert.equal(silent.status, 5, "a proxy that never answers is no reading"); + // Nothing listening: take a port, then free it. + const { createServer: net } = await import("node:net"); + const port = await new Promise((resolve) => { + const s = net().listen(0, "127.0.0.1", () => { + const p = s.address().port; + s.close(() => resolve(p)); + }); + }); + const refused = spawnSync(process.execPath, ["-e", egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: `http://127.0.0.1:${port}`, timeoutMs: 300 })], { encoding: "utf8", timeout: 30_000 }); + assert.equal(refused.status, 5, refused.stdout); + assert.match(refused.stdout, /^error ECONNREFUSED/); +}); + +test("the plain HTTP probe's argv is the unlisted probe's apart from its name and its script (#508)", () => { + const base = { pid: 42, network: "pi-dispatch-egress-doctor-42", proxy: "pi-dispatch-egress-proxy", image: "pi-job:latest" }; + const plain = egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: "http://pi-dispatch-egress-proxy:3128" }); + for (const bin of ["docker", "podman"]) { + const user = bin === "podman" ? "1234:1234" : null; + const unlisted = egressCanaryProbeArgs({ bin, ...base, user, slug: "unlisted", url: "https://example.com/" }); + const plainhttp = egressCanaryProbeArgs({ bin, ...base, user, slug: "plainhttp", url: "http://api.anthropic.com:443/", script: plain }); + assert.equal(plainhttp.at(-1), plain, bin); + assert.equal(unlisted.at(-1), egressCanaryScript("https://example.com/"), `${bin}: the default script is unchanged`); + assert.deepEqual( + plainhttp.slice(0, -1).map((a) => a.replace("plainhttp", "SLUG")), + unlisted.slice(0, -1).map((a) => a.replace("unlisted", "SLUG")), + bin, + ); + } +}); + +test("a proxy that lets plain HTTP through fails its own line and reads back as reached; an unrun one is not a reading (#508)", async () => { + const through = await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams({ ...green, "docker run --rm --name pi-dispatch-egress-probe-plainhttp": 0 })); + const line = through.find((c) => c.readBack?.probe === "plainhttp"); + assert.equal(line.ok, false); + assert.equal(line.warn, true); + assert.equal(line.label, "Egress policy lets plain HTTP through to api.anthropic.com on port 443, so a client that does not tunnel reaches every port of a listed host"); + assert.match(line.fix, /issue #508/); + assert.ok(line.fix.includes('"http_access deny !Safe_ports !CONNECT" right before "http_access allow allowed"')); + assert.ok(line.fix.includes("for a proxy you started by hand, update the egress-proxy.conf it mounts, then `docker restart pi-dispatch-egress-proxy`"), line.fix); + assert.deepEqual(line.readBack, { property: "egress", want: false, reached: true, probe: "plainhttp" }); + const held = (await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams(green))).find((c) => c.readBack?.probe === "plainhttp"); + assert.equal(held.ok, true); + assert.equal(held.label, "Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)"); + const unrun = await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams({ ...green, "docker run --rm --name pi-dispatch-egress-probe-plainhttp": 125 })); + assert.ok(unrun.some((c) => c.label === "Egress policy probe for plain HTTP to a listed host off port 80 did not run (docker run exited 125)")); + assert.deepEqual(unrun.find((c) => c.readBack?.probe === "plainhttp").readBack, { property: "egress", want: false, reached: null, probe: "plainhttp" }); +}); + +// Issue #508, gate round 1: docs/egress.md's sample doctor output is the code's own wording. The page's generated rows +// (CANARY_LINES) are the leftover sweep's lines only, so the probe lines were free to drift; they are pinned here, read +// off a real canary run and the real verdict rather than retyped. +test("docs/egress.md's sample egress lines are what doctor prints, on docker and on podman (#508)", async () => { + const doc = readFileSync(new URL("../../docs/egress.md", import.meta.url), "utf8"); + const probes = (await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams(green))).filter((c) => c.readBack); + assert.deepEqual(probes.map((c) => c.readBack.probe), ["provider", "unlisted", "plainhttp"]); + const docker = probes.map((c) => `✓ ${c.label}\n`).join(""); + const podman = probes.map((c) => `✓ podman: ${c.label}\n`).join(""); + assert.ok(doc.includes(`✓ Egress proxy health: healthy\n${docker}\`\`\``), "the docker sample, all three lines in order"); + const held = egressVerdict({ armed: true, results: probes.map((c) => c.readBack) }); + assert.equal(held.ok, true); + assert.ok(doc.includes(`${podman}✓ read back on podman: egress holds (${held.detail})\n`), "the podman sample, with the read-back's own wording"); +}); + +// Issue #508, gate rounds 1 and 2: with the egress policy on a job reaches a forge only over https on 443. git ignores the +// job's uppercase HTTP_PROXY for an http:// remote (measured in the job image), and the proxy refuses a CONNECT off 443. +test("doctor warns when egress is on and a forge URL is anything but https:// on port 443 (#508)", () => { + const on = { PI_EGRESS: "1" }; + const [gitlab] = forgeUrlEgressChecks({ ...on, GITLAB_URL: "http://gitlab.example:8929" }, ["gitlab"]); + assert.deepEqual([gitlab.ok, gitlab.warn], [false, true]); + assert.equal(gitlab.label, "GITLAB_URL (http://gitlab.example:8929) is http://, so with the egress policy on a job's git push and fetch fail (git never sends http:// through the proxy), and on port 8929, not 80, its API calls fail too (issue #508)"); + // On port 80 the API calls pass (glab and tea use HTTP_PROXY, and the proxy allows port 80): only git is named. + assert.equal(forgeUrlEgressChecks({ GITLAB_URL: "http://gitlab.example" }, ["gitlab"])[0].label, "GITLAB_URL (http://gitlab.example) is http://, so with the egress policy on a job's git push and fetch fail (git never sends http:// through the proxy) (issue #508)"); + // Any other scheme, a bare host:port among them (parsed with the host as its scheme), is not called http://. + for (const value of ["ftp://gl.example", "gitlab.internal:8443"]) { + const [other] = forgeUrlEgressChecks({ GITLAB_URL: value }, ["gitlab"]); + assert.match(other.label, /^GITLAB_URL \(.*\) is not an https:\/\/ URL, so with the egress policy on a job cannot reach it \(issue #508\)$/, value); + assert.match(other.fix, /^serve the forge over https:\/\/ on port 443/, value); + } + assert.equal(gitlab.fix, "serve the forge over https:// on port 443 and point GITLAB_URL there, or set PI_EGRESS=0 if you accept jobs without the policy (docs/egress.md)"); + const [tls] = forgeUrlEgressChecks({ GITLAB_URL: "https://gitlab.example:8443" }, ["gitlab"]); + assert.equal(tls.label, "GITLAB_URL (https://gitlab.example:8443) is https:// on port 8443, and the egress proxy refuses a CONNECT to any port but 443, so with the egress policy on every job's push, fetch and API call to it fails (issue #508)"); + for (const [why, env, forges] of [ + ["http on 80", { GITLAB_URL: "http://gitlab.example" }, ["gitlab"]], + ["http on 80 written out", { GITLAB_URL: "http://gitlab.example:80" }, ["gitlab"]], + ["forgejo on 3000, armed by default", { FORGEJO_URL: "http://forgejo.example:3000/" }, ["forgejo"]], + ]) { + assert.equal(forgeUrlEgressChecks(env, forges).length, 1, why); + } + // The credential in a URL is never printed. + assert.doesNotMatch(forgeUrlEgressChecks({ GITLAB_URL: "http://u:secret@gitlab.example:8929" }, ["gitlab"])[0].label, /secret/); + for (const [why, env, forges] of [ + ["https by default", { GITLAB_URL: "https://gitlab.example" }, ["gitlab"]], + ["https on 443 written out", { FORGEJO_URL: "https://forgejo.example:443/" }, ["forgejo"]], + ["policy off", { PI_EGRESS: "0", GITLAB_URL: "http://gitlab.example:8929" }, ["gitlab"]], + ["policy unreadable", { PI_EGRESS: "maybe", GITLAB_URL: "http://gitlab.example:8929" }, ["gitlab"]], + ["forge not in triggers", { GITLAB_URL: "http://gitlab.example:8929" }, ["forgejo"]], + ["not a URL", { GITLAB_URL: "gitlab" }, ["gitlab"]], + ]) { + assert.deepEqual(forgeUrlEgressChecks(env, forges), [], why); + } +}); + +test("the forge warning reaches doctor's output for a gitlab and a forgejo trigger (#508)", async () => { + for (const [kind, vars] of [ + ["gitlab", { GITLAB_TOKEN: "glpat_x", GITLAB_URL: "http://gitlab.example:8929" }], + ["forgejo", { FORGEJO_URL: "http://code.example.org:3000", FORGEJO_WEBHOOK_SECRET: "s", FORGEJO_TOKEN: "t" }], + ]) { + const { out, text } = capture(); + await runDoctor(imgEnv({ PI_TRIGGERS_FILE: forgeTriggersFile(kind), ...vars }), imgDeps(out, green)); + assert.match(text(), new RegExp(`⚠ ${kind.toUpperCase()}_URL \\([^)]*\\) is http://, so with the egress policy on a job's git push and fetch fail`), `${kind}: ${text()}`); + } +}); + test("the egress canary's deny probe asks for a host that RESOLVES, under a per-process name", async () => { const calls = []; await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams(green, { spawn: fakeSpawn(green, calls) })); @@ -4666,7 +4824,11 @@ test("the egress canary's deny probe asks for a host that RESOLVES, under a per- test("the egress canary carries a non-rendered readBack, so --live folds it in without a second canary", async () => { const checks = await collectChecks({ PI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-x" }, collectSeams(green)); const readBacks = checks.filter((c) => c.readBack).map((c) => c.readBack); - assert.deepEqual(readBacks, [{ property: "egress", want: true, reached: true }, { property: "egress", want: false, reached: false }]); + assert.deepEqual(readBacks, [ + { property: "egress", want: true, reached: true, probe: "provider" }, + { property: "egress", want: false, reached: false, probe: "unlisted" }, + { property: "egress", want: false, reached: false, probe: "plainhttp" }, + ]); }); // --- issue #341: the job-user line, and the probe run as the job user ------------------------------------------ @@ -5396,7 +5558,7 @@ test("docs/egress.md's canary rows ARE the table, generated (#379)", () => { // THE RETIRED PHRASES, carried over from the test this replaced. It guarded them with the note that a // correction had landed everywhere except the page four times on that branch, and dropping the guard let // all three back into the columns this test does not otherwise read. - const section = doc.slice(doc.indexOf("### Lines about leftovers"), doc.indexOf("The two policy lines each run")); + const section = doc.slice(doc.indexOf("### Lines about leftovers"), doc.indexOf("The provider and unlisted-host lines each run")); for (const retired of ["left by a doctor run that did not finish", "from an interrupted doctor", "the network itself was already gone"]) { assert.equal(section.includes(retired), false, `the page must not quote a sentence no site can emit: ${retired}`); } @@ -5628,7 +5790,7 @@ test("doctor: the probe's name comes from the SEAM, so the producer and the reap const wedged = { ...green, "docker run --rm --name pi-dispatch-egress-probe-unlisted": { code: null, output: "" }, "gh auth status": { code: 0, output: ghStatusOutput } }; await runDoctor(ghEnv({ PI_EGRESS: "1" }), ghDeps(out, wedged, calls, { pid: 777, isAlive: () => true })); const ran = calls.filter((c) => c.args[0] === "run").map((c) => c.args[c.args.indexOf("--name") + 1]); - assert.deepEqual(ran, ["pi-dispatch-egress-probe-provider-777", "pi-dispatch-egress-probe-unlisted-777"], "the injected pid names the probes"); + assert.deepEqual(ran, ["pi-dispatch-egress-probe-provider-777", "pi-dispatch-egress-probe-unlisted-777", "pi-dispatch-egress-probe-plainhttp-777"], "the injected pid names the probes"); assert.deepEqual(calls.filter((c) => c.args[0] === "rm").map((c) => c.args.at(-1)), ["pi-dispatch-egress-probe-unlisted-777"], "and the cleanup looks for that same name"); }); @@ -5814,7 +5976,8 @@ test("doctor: the slug list is pinned as LITERALS, because order is part of the // REVERSAL moves both sides together. Order matters here -- reversing it names the provider probe // `-unlisted-` while it fetches api.anthropic.com, defeating the whole point of a name someone reads // off `ps` to find out what a wedged probe was doing. - assert.deepEqual([...CANARY_PROBE_SLUGS], ["provider", "unlisted"]); + // Issue #508 appends `plainhttp` LAST, so the two names that were already read off `ps` keep their places. + assert.deepEqual([...CANARY_PROBE_SLUGS], ["provider", "unlisted", "plainhttp"]); }); test("doctor: a listing that FAILS is said, never taken as 'no leftovers' (#350)", async () => { @@ -6560,7 +6723,7 @@ const MIXED_PIN = { "✓ podman: egress proxy running under this account's Podman (pi-dispatch-egress-proxy)", // Issue #431: the one line that moved, and on purpose: the podman venue's allowlist is now read back by --live. "⚠ podman: the egress allowlist is read back by `pi-dispatch doctor --live` on this venue, not by this run", - " → run `pi-dispatch doctor --live`: its egress canary runs two containers built like a podman job's (the job user, --userns=keep-id, the venue's pinned flags) on a job-shaped --internal network under this account's Podman, one that must reach the provider through the proxy and one that must not reach an unlisted host", + " → run `pi-dispatch doctor --live`: its egress canary runs three containers built like a podman job's (the job user, --userns=keep-id, the venue's pinned flags) on a job-shaped --internal network under this account's Podman, one that must reach the provider through the proxy, one that must not reach an unlisted host, and one that must not get plain HTTP through to a listed host off port 80", // Issue #458: the one line added, with egress armed on this venue. "✓ podman: rootless network keeper running under this account's Podman on its own bridge network (pi-dispatch-netns-keeper), so a job's network teardown cannot cut the proxy's route out", // PR #463 round 3: the keeper network's options, read as they are. @@ -7885,7 +8048,8 @@ const podmanEgressFacts = (egress = {}, over = {}) => ({ const podmanEgressSeams = (canary = {}, calls = [], plan = {}) => ({ // `canary` FIRST, for its keys' place in the prefix match, and LAST, for their values: a spread keeps the first place a // key had and the last value it was given, so spreading it once would lose an override of a key the rest also carry. - spawn: fakeSpawn({ ...canary, ...toPodman(liveOk({ uid: "1234" })), ...toPodman(livePeersOk(plan)), ...podmanPlan(), "podman network": 0, ...canary }, calls), + // The plain HTTP probe (issue #508) refused by default, as a current proxy does; a test that reads it names it. + spawn: fakeSpawn({ [`${PODMAN_PROBE}plainhttp`]: 3, ...canary, ...toPodman(liveOk({ uid: "1234" })), ...toPodman(livePeersOk(plan)), ...podmanPlan(), "podman network": 0, ...canary }, calls), liveFs: liveFsAs(1234), isAlive: () => false, pid: 1, @@ -7897,7 +8061,7 @@ const PODMAN_PROBE = "podman run --name=pi-dispatch-egress-probe-"; // The podman canary's probe argv, WRITTEN OUT rather than built by the builder under test: a job's own argv (the // builder's isolation flags and bounds, the job user, keep-id and the venue's pinned flags, a job's HOME and its four // egress variables), on the canary network, with no mount, running the runner's route to the network. -const podmanProbeArgv = (slug, url) => [ +const podmanProbeArgv = (slug, url, script = egressCanaryScript(url)) => [ "run", `--name=pi-dispatch-egress-probe-${slug}-1`, "--pull=never", @@ -7933,7 +8097,7 @@ const podmanProbeArgv = (slug, url) => [ "NODE_USE_ENV_PROXY=1", "pi-job:latest", "-e", - egressCanaryScript(url), + script, ]; test("doctor --live on podman reads egress back through a canary under podman, built as a podman job is (#431)", async () => { @@ -7950,6 +8114,7 @@ test("doctor --live on podman reads egress back through a canary under podman, b "network connect pi-dispatch-egress-doctor-1", "run --name=pi-dispatch-egress-probe-provider-1 --pull=never", "run --name=pi-dispatch-egress-probe-unlisted-1 --pull=never", + "run --name=pi-dispatch-egress-probe-plainhttp-1 --pull=never", "network disconnect -f", "network rm pi-dispatch-egress-doctor-1", ]); @@ -7957,11 +8122,18 @@ test("doctor --live on podman reads egress back through a canary under podman, b const probes = calls.filter((c) => String(c.args[1]).startsWith("--name=pi-dispatch-egress-probe-")); assert.deepEqual(probes[0].args, podmanProbeArgv("provider", "https://api.anthropic.com/v1/messages")); assert.deepEqual(probes[1].args, podmanProbeArgv("unlisted", "https://example.com/")); + // Issue #508: the plain HTTP probe is a job's argv too, running the raw forward request at the job's own proxy. + assert.deepEqual(probes[2].args, podmanProbeArgv("plainhttp", "http://api.anthropic.com:443/", egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: "http://pi-dispatch-egress-proxy:3128" }))); // want and reached per probe, carried where the verdict reads them, and the lines say podman. - assert.deepEqual(held.filter((c) => c.readBack).map((c) => c.readBack), [{ property: "egress", want: true, reached: true }, { property: "egress", want: false, reached: false }]); + assert.deepEqual(held.filter((c) => c.readBack).map((c) => c.readBack), [ + { property: "egress", want: true, reached: true, probe: "provider" }, + { property: "egress", want: false, reached: false, probe: "unlisted" }, + { property: "egress", want: false, reached: false, probe: "plainhttp" }, + ]); assert.ok(labels.includes("podman: Egress policy reaches the provider (api.anthropic.com answered, so the whole path works and no key was spent)"), labels.join("\n")); assert.ok(labels.includes("podman: Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose)")); - assert.ok(held.some((c) => c.ok && c.label === "read back on podman: egress holds (the provider was reached and an unlisted host was not)"), labels.join("\n")); + assert.ok(labels.includes("podman: Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)")); + assert.ok(held.some((c) => c.ok && c.label === "read back on podman: egress holds (the provider was reached, an unlisted host was not, and plain HTTP off port 80 was refused)"), labels.join("\n")); assert.ok(labels.indexOf("podman: Egress policy reaches the provider (api.anthropic.com answered, so the whole path works and no key was spent)") < labels.findIndex((l) => l.startsWith("read back on podman: egress")), "the canary's lines come first, so the verdict's 'above' is true"); assert.equal(calls.filter((c) => c.args[0] === "info").length, 2, "podman info asked again before the canary, and again before the probes"); assert.ok(held.some((c) => c.ok && /^read back on podman: jobToJobIsolation holds \(peer1 reached the proxy/.test(c.label)), labels.join("\n")); @@ -7982,10 +8154,14 @@ test("a podman canary that reaches an unlisted host, or not the provider, fails assert.match(shut.find((c) => c.label.startsWith("read back on podman: egress")).label, /egress does NOT hold -- declared enforced, observed: the provider was not reached$/); const stale = await run({ [`${PODMAN_PROBE}provider`]: EGRESS_CANARY_STALE_RUNNER }); assert.ok(stale.some((c) => c.warn && c.label.startsWith(`podman: Egress policy: not proved, because the job image could not find ${EGRESS_CANARY_RUNNER_MODULE}`))); - assert.match(stale.find((c) => c.label.startsWith("read back on podman: egress")).label, /egress not read back: the egress canary did not run both probes \(see the egress lines above\)/); + assert.match(stale.find((c) => c.label.startsWith("read back on podman: egress")).label, /egress not read back: the egress canary did not run all three probes \(see the egress lines above\)/); const refused = await run({ [`${PODMAN_PROBE}provider`]: 125, [`${PODMAN_PROBE}unlisted`]: 3 }); assert.ok(refused.some((c) => c.label === "podman: Egress policy probe for the provider did not run (podman run exited 125)")); assert.match(refused.find((c) => c.label.startsWith("read back on podman: egress")).label, /egress not read back: an egress probe did not run to an answer/); + // Issue #508: a proxy on rules from before it lets plain HTTP through, and podman's verdict says which probe. + const plain = await run({ [`${PODMAN_PROBE}provider`]: 0, [`${PODMAN_PROBE}unlisted`]: 3, [`${PODMAN_PROBE}plainhttp`]: 0 }); + assert.ok(plain.some((c) => c.warn && !c.ok && c.label === "podman: Egress policy lets plain HTTP through to api.anthropic.com on port 443, so a client that does not tunnel reaches every port of a listed host")); + assert.equal(plain.find((c) => c.label.startsWith("read back on podman: egress")).label, "read back on podman: egress does NOT hold -- declared enforced, observed: plain HTTP to a listed host off port 80 was let through"); }); test("the podman canary is stopped by the venue's refusals, a service no longer this host's own, the policy off, and a proxy that is down (#431)", async () => { @@ -8135,14 +8311,14 @@ test("doctor --live on a podman-only deployment with egress armed: the canary's const env = liveEnv({ PI_EGRESS: "1", PI_BACKENDS: "podman" }); const { out, text } = capture(); const calls = []; - const plan = { [`${PODMAN_PROBE}provider`]: 0, [`${PODMAN_PROBE}unlisted`]: 3, ...toPodman(liveOk({ uid: "1234" })), ...toPodman(livePeersOk({})), "podman network": 0, ...podmanPlan() }; + const plan = { [`${PODMAN_PROBE}provider`]: 0, [`${PODMAN_PROBE}unlisted`]: 3, [`${PODMAN_PROBE}plainhttp`]: 3, ...toPodman(liveOk({ uid: "1234" })), ...toPodman(livePeersOk({})), "podman network": 0, ...podmanPlan() }; const code = await runDoctor(env, { ...podmanDeps(out, plan, calls), live: true, liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n", ...instantClock() }); assert.equal(code, 0, text()); assert.deepEqual(calls.filter((c) => c.cmd === "docker"), []); assert.doesNotMatch(text(), /allowlist is read back by `pi-dispatch doctor --live`/, "a --live run does not point at itself"); - assert.match(text(), /\nread back on podman: starting pi-dispatch-egress-probe-provider-7 and pi-dispatch-egress-probe-unlisted-7 from pi-job:latest \(as the job user 1234:1234\) on the --internal network pi-dispatch-egress-doctor-7, with pi-dispatch-egress-proxy attached/); - assert.match(text(), /✓ podman: Egress policy reaches the provider[^\n]*\n✓ podman: Egress policy denies an unlisted host[^\n]*\n/); - assert.ok(text().indexOf("✓ podman: Egress policy denies an unlisted host") < text().indexOf("✓ read back on podman: egress holds")); + assert.match(text(), /\nread back on podman: starting pi-dispatch-egress-probe-provider-7, pi-dispatch-egress-probe-unlisted-7 and pi-dispatch-egress-probe-plainhttp-7 from pi-job:latest \(as the job user 1234:1234\) on the --internal network pi-dispatch-egress-doctor-7, with pi-dispatch-egress-proxy attached, to read the egress allowlist back; all three are removed when the canary ends\n/); + assert.match(text(), /✓ podman: Egress policy reaches the provider[^\n]*\n✓ podman: Egress policy denies an unlisted host[^\n]*\n✓ podman: Egress policy refuses plain HTTP to a listed host off port 80[^\n]*\n/); + assert.ok(text().indexOf("✓ podman: Egress policy refuses plain HTTP to a listed host off port 80") < text().indexOf("✓ read back on podman: egress holds")); assert.equal(calls.filter((c) => c.cmd === "podman" && c.args[0] === "info").length, 3, "the section's read, the canary's re-ask, and the read-back's"); }); @@ -8164,7 +8340,7 @@ test("doctor --live on Podman 4.x without the keeper runs no canary and no peer const { out, text } = capture(); const calls = []; const info = JSON.stringify({ ...JSON.parse(PODMAN_INFO()), version: { Version: version } }); - const plan = { ...(stale ? staleKeys : {}), [`${PODMAN_PROBE}provider`]: 0, [`${PODMAN_PROBE}unlisted`]: 3, ...toPodman(liveOk({ uid: "1234" })), ...toPodman(livePeersOk({})), "podman network": 0, ...podmanPlan({ info, keeper }) }; + const plan = { ...(stale ? staleKeys : {}), [`${PODMAN_PROBE}provider`]: 0, [`${PODMAN_PROBE}unlisted`]: 3, [`${PODMAN_PROBE}plainhttp`]: 3, ...toPodman(liveOk({ uid: "1234" })), ...toPodman(livePeersOk({})), "podman network": 0, ...podmanPlan({ info, keeper }) }; const code = await runDoctor(env, { ...podmanDeps(out, plan, calls), live: true, liveFs: liveFsAs(1234), isAlive: () => false, pid: 7, nonce: "n", ...instantClock() }); const lines = calls.map((c) => [c.cmd, ...c.args].join(" ")); return { code, text: text(), lines }; @@ -8231,7 +8407,7 @@ test("the podman canary runs PI_JOB_IMAGE, whatever it names (#431)", async () = const calls = []; await podmanLiveChecks(env, podmanEgressSeams({ [`${PODMAN_PROBE}provider`]: 0, [`${PODMAN_PROBE}unlisted`]: 3 }, calls), { ...podmanEgressFacts(), jobImage: "registry.example.invalid/team/pi-job:7" }); const probes = calls.filter((c) => String(c.args[1]).startsWith("--name=pi-dispatch-egress-probe-")); - assert.equal(probes.length, 2); + assert.equal(probes.length, 3); for (const p of probes) { assert.equal(p.args.at(-3), "registry.example.invalid/team/pi-job:7", p.args.join(" ")); assert.equal(p.args.includes("pi-job:latest"), false); @@ -8461,10 +8637,13 @@ test("the pinning and ephemeral read-backs on podman name podman in what they co // --- issue #431: docker's egress canary, pinned before it was parameterised on the venue ------------------------------- // A docker deployment with egress armed, run as `doctor --live`, in three shapes: the ordinary one (a DEAD doctor's -// leftover swept, both probes answering, the teardown clean, the read-back with its peers), a job image whose runner +// leftover swept, every probe answering, the teardown clean, the read-back with its peers), a job image whose runner // predates #427, and a provider probe the bound cut short. Captured from main at 26cef96, BEFORE the canary was // parameterised on a venue runner: the text and every spawn (runtime and argv), with the only per-run values (the // jobs directory and the fixture names under it) replaced. A docker canary that moved by one byte or one spawn lands here. +// Issue #508 added, by hand and nothing else: the third probe's spawn after the unlisted one (its argv the unlisted one's +// but for name and script), its ✓ line, one more spawn in each total but the stale runner's (which stops at the first +// probe), and the read-back's wording for three probes. const dockerCanaryPinRun = async (scenario, t = null) => { const env = liveEnv({ PI_EGRESS: "1" }); const cwd = tempDir("pi-canary-pin-cwd-"); @@ -8504,7 +8683,11 @@ const dockerCanaryPinRun = async (scenario, t = null) => { const norm = (s) => String(s).replaceAll(jobs, "").replaceAll(env.PI_JOBS_DIR, "").replaceAll(cwd, "").replace(/pi-dispatch-live-7-[A-Za-z0-9]{6}/g, ""); // The in-container script is `egressCanaryScript`'s, which its own tests pin; here it is named, so the argv around it // is what this reads, and a script that is not exactly that function's output for that URL still differs. - const script = (s) => s.replace(egressCanaryScript("https://api.anthropic.com/v1/messages"), "").replace(egressCanaryScript("https://example.com/"), ""); + const script = (s) => + s + .replace(egressCanaryScript("https://api.anthropic.com/v1/messages"), "") + .replace(egressCanaryScript("https://example.com/"), "") + .replace(egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: "http://pi-dispatch-egress-proxy:3128" }), ""); const spawns = calls.map((c) => script(norm([c.cmd, ...c.args].join(" ")))); // The collection's spawns, which is where the canary lives, exactly; the read-back's (runLiveProbes, which this // change does not touch) by count. @@ -8514,7 +8697,7 @@ const dockerCanaryPinRun = async (scenario, t = null) => { const DOCKER_CANARY_PIN = { green: { code: 0, - total: 50, + total: 51, collection: [ "docker info", "docker context inspect --format={{json .Name}}|{{json .Endpoints.docker.Host}}", @@ -8530,6 +8713,7 @@ const DOCKER_CANARY_PIN = { "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker run --rm --name pi-dispatch-egress-probe-unlisted-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", + "docker run --rm --name pi-dispatch-egress-probe-plainhttp-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker network disconnect -f pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-7", "gh auth status", @@ -8546,6 +8730,7 @@ const DOCKER_CANARY_PIN = { "✓ Egress proxy health: healthy", "✓ Egress policy reaches the provider (api.anthropic.com answered, so the whole path works and no key was spent)", "✓ Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose)", + "✓ Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)", "✓ Jobs run on: local", "⚠ local: nonRoot is ASSERTED by the job image's USER directive (this repo's builds `USER pi`; an operator-built image may not), or, on a daemon that enforces bind-mount ownership with a worker uid other than 1001, the worker's own non-zero uid passed as `--user`, not enforced by it", " → not verifiable from here, so treat it as a claim rather than a control: the agent runs as a non-root user", @@ -8569,7 +8754,7 @@ const DOCKER_CANARY_PIN = { "✓ read back on local: isolation holds (CapBnd 0, NoNewPrivs 1, pids.max 512, memory.max 4294967296)", "✓ read back on local: ephemeral holds (two runs under one name each removed themselves (in 0 ms and 0 ms), and the second found nothing of the first)", "✓ read back on local: mountSet holds (/job:ro, /workspace, /outbox, /session, /opt/pi-global:ro and nothing else, in docker inspect and in /proc/self/mountinfo)", - "✓ read back on local: egress holds (the provider was reached and an unlisted host was not)", + "✓ read back on local: egress holds (the provider was reached, an unlisted host was not, and plain HTTP off port 80 was refused)", "✓ read back on local: jobToJobIsolation holds (peer1 reached the proxy and none of peer2's 3 name(s) and address(es) (pi-dispatch-live-peer2-7-n:47431 enetunreach, pi-dispatch-live-peer2-1-n:47431 enetunreach, 10.99.0.3:47431 enetunreach); peer2 answered itself before and after)", "✓ read back on local: imagePinning holds (an absent image was refused without a pull)", "✓ read back on local: nonRoot holds (Uid 1001 1001 1001 1001)", @@ -8636,7 +8821,7 @@ const DOCKER_CANARY_PIN = { "✓ read back on local: isolation holds (CapBnd 0, NoNewPrivs 1, pids.max 512, memory.max 4294967296)", "✓ read back on local: ephemeral holds (two runs under one name each removed themselves (in 0 ms and 0 ms), and the second found nothing of the first)", "✓ read back on local: mountSet holds (/job:ro, /workspace, /outbox, /session, /opt/pi-global:ro and nothing else, in docker inspect and in /proc/self/mountinfo)", - "⚠ read back on local: egress not read back: the egress canary did not run both probes (see the egress lines above)", + "⚠ read back on local: egress not read back: the egress canary did not run all three probes (see the egress lines above)", " → this property was not read back, which is not the same as holding: see the reason, fix it if you can, and re-run `pi-dispatch doctor --live`", "✓ read back on local: jobToJobIsolation holds (peer1 reached the proxy and none of peer2's 3 name(s) and address(es) (pi-dispatch-live-peer2-7-n:47431 enetunreach, pi-dispatch-live-peer2-1-n:47431 enetunreach, 10.99.0.3:47431 enetunreach); peer2 answered itself before and after)", "✓ read back on local: imagePinning holds (an absent image was refused without a pull)", @@ -8650,7 +8835,7 @@ const DOCKER_CANARY_PIN = { }, unfinished: { code: 0, - total: 51, + total: 52, collection: [ "docker info", "docker context inspect --format={{json .Name}}|{{json .Endpoints.docker.Host}}", @@ -8666,6 +8851,7 @@ const DOCKER_CANARY_PIN = { "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker run --rm --name pi-dispatch-egress-probe-unlisted-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", + "docker run --rm --name pi-dispatch-egress-probe-plainhttp-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker rm -f pi-dispatch-egress-probe-provider-7", "docker network disconnect -f pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-7", @@ -8684,6 +8870,7 @@ const DOCKER_CANARY_PIN = { "⚠ Egress policy probe for the provider did not run (docker run did not finish)", " → re-run doctor; if it persists, run the job image by hand to see why a container on this network will not start", "✓ Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose)", + "✓ Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)", "✓ Jobs run on: local", "⚠ local: nonRoot is ASSERTED by the job image's USER directive (this repo's builds `USER pi`; an operator-built image may not), or, on a daemon that enforces bind-mount ownership with a worker uid other than 1001, the worker's own non-zero uid passed as `--user`, not enforced by it", " → not verifiable from here, so treat it as a claim rather than a control: the agent runs as a non-root user", @@ -8721,7 +8908,7 @@ const DOCKER_CANARY_PIN = { }, hung: { code: 0, - total: 51, + total: 52, // The kills the provider probe got: none before its 30 s bound, then ONE, with no signal named (so the // default SIGTERM), which is `runCmdCapture`'s. The step runner would send SIGKILL. killsBeforeBound: [], @@ -8741,6 +8928,7 @@ const DOCKER_CANARY_PIN = { "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker run --rm --name pi-dispatch-egress-probe-unlisted-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", + "docker run --rm --name pi-dispatch-egress-probe-plainhttp-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker rm -f pi-dispatch-egress-probe-provider-7", "docker network disconnect -f pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-7", @@ -8759,6 +8947,7 @@ const DOCKER_CANARY_PIN = { "⚠ Egress policy probe for the provider did not run (docker run did not finish)", " → re-run doctor; if it persists, run the job image by hand to see why a container on this network will not start", "✓ Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose)", + "✓ Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)", "✓ Jobs run on: local", "⚠ local: nonRoot is ASSERTED by the job image's USER directive (this repo's builds `USER pi`; an operator-built image may not), or, on a daemon that enforces bind-mount ownership with a worker uid other than 1001, the worker's own non-zero uid passed as `--user`, not enforced by it", " → not verifiable from here, so treat it as a claim rather than a control: the agent runs as a non-root user", @@ -8796,7 +8985,7 @@ const DOCKER_CANARY_PIN = { }, enoent: { code: 0, - total: 51, + total: 52, // The kills the provider probe got: none before its 30 s bound, then ONE, with no signal named (so the // A probe that never launched is never killed. killsBeforeBound: null, @@ -8816,6 +9005,7 @@ const DOCKER_CANARY_PIN = { "docker network connect pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker run --rm --name pi-dispatch-egress-probe-provider-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker run --rm --name pi-dispatch-egress-probe-unlisted-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", + "docker run --rm --name pi-dispatch-egress-probe-plainhttp-7 --pull=never --network=pi-dispatch-egress-doctor-7 -e HTTPS_PROXY=http://pi-dispatch-egress-proxy:3128 -e NODE_USE_ENV_PROXY=1 --entrypoint node pi-job:latest -e ", "docker rm -f pi-dispatch-egress-probe-provider-7", "docker network disconnect -f pi-dispatch-egress-doctor-7 pi-dispatch-egress-proxy", "docker network rm pi-dispatch-egress-doctor-7", @@ -8834,6 +9024,7 @@ const DOCKER_CANARY_PIN = { "⚠ Egress policy probe for the provider did not run (docker run did not finish)", " → re-run doctor; if it persists, run the job image by hand to see why a container on this network will not start", "✓ Egress policy denies an unlisted host (the deny direction is the half an allowlist can silently lose)", + "✓ Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)", "✓ Jobs run on: local", "⚠ local: nonRoot is ASSERTED by the job image's USER directive (this repo's builds `USER pi`; an operator-built image may not), or, on a daemon that enforces bind-mount ownership with a worker uid other than 1001, the worker's own non-zero uid passed as `--user`, not enforced by it", " → not verifiable from here, so treat it as a claim rather than a control: the agent runs as a non-root user", diff --git a/worker/test/egress.test.mjs b/worker/test/egress.test.mjs index 4e09ce81..4b93c82a 100644 --- a/worker/test/egress.test.mjs +++ b/worker/test/egress.test.mjs @@ -525,6 +525,35 @@ test("the proxy denies a listed name resolving to this host's loopback or link-l assert.deepEqual(lines.filter((line) => /^acl allowed\s/.test(line)), ['acl allowed dstdomain -n "/etc/pi-dispatch/allowlist.conf"']); }); +// Issue #508: `allow allowed` carried no port rule, so a client forwarding plain HTTP reached ANY port of a listed host. +// The narrowing is stock squid's `Safe_ports`, one port wide, as a deny right before that allow (first match wins, so +// after it would never apply), and it names only port and method ACLs, which resolve nothing, so #428's "no unlisted +// name is resolved" still holds. `!CONNECT` keeps it off every tunnel, so the CONNECT rules are untouched. +test("plain HTTP reaches a listed host on port 80 only: `deny !Safe_ports !CONNECT` right before `allow allowed`, resolving nothing (#508)", async () => { + const { readFileSync } = await import("node:fs"); + const conf = readFileSync(new URL("../../deploy/egress-proxy.conf", import.meta.url), "utf8"); + const lines = conf.split("\n").map((line) => line.trim()).filter((line) => line !== "" && !line.startsWith("#")); + const access = lines.filter((line) => line.startsWith("http_access ")); + const safe = lines.filter((line) => /^acl Safe_ports\s/.test(line)); + assert.deepEqual(safe, ["acl Safe_ports port 80"], "defined once, with exactly that port"); + const naming = access.filter((line) => /\bSafe_ports\b/.test(line)); + assert.deepEqual(naming, ["http_access deny !Safe_ports !CONNECT"], "one rule names it, and it never matches a CONNECT"); + const at = access.indexOf(naming[0]); + assert.ok(at > access.findIndex((line) => /\bto_host_local\b/.test(line)), "after the address deny"); + assert.equal(access[at + 1], "http_access allow allowed", "directly before the allow it narrows"); + assert.ok(lines.indexOf(safe[0]) < lines.indexOf(naming[0]), "defined before use"); + // It names no ACL that resolves: every ACL on the line is a port or a method ACL. + for (const name of naming[0].split(/\s+/).slice(2).map((word) => word.replace(/^!/, ""))) { + const def = lines.filter((line) => line.startsWith(`acl ${name} `)); + assert.equal(def.length, 1, name); + assert.ok(["port", "method"].includes(def[0].split(/\s+/)[2]), `${name} is a ${def[0].split(/\s+/)[2]} acl`); + } + // The tunnel rules are as they were. + assert.ok(access.indexOf("http_access deny CONNECT !SSL_ports") < access.indexOf("http_access allow CONNECT allowed")); + assert.deepEqual(lines.filter((line) => /^acl SSL_ports\s/.test(line)), ["acl SSL_ports port 443"]); + assert.equal(access.at(-1), "http_access deny all"); +}); + // --- issue #452, gate round 3: the ONE detach helper, and the gate every teardown asks through it --------------------- const COMPAT49 = JSON.stringify({ ServerVersion: "4.9.3", ProductLicense: "Apache-2.0", OperatingSystem: "ubuntu", SecurityOptions: ["name=rootless"] }); diff --git a/worker/test/live-probes.test.mjs b/worker/test/live-probes.test.mjs index 11a675da..bcc55aa7 100644 --- a/worker/test/live-probes.test.mjs +++ b/worker/test/live-probes.test.mjs @@ -279,17 +279,51 @@ test("imagePinning holds only for a refusal with no pull attempt, and the image }); test("egress is folded from the canary: off or partial is not read back, a wrong reading fails", () => { + // The canary's three readings (issue #508 added the third), each naming its probe. + const provider = (reached) => ({ want: true, reached, probe: "provider" }); + const unlisted = (reached) => ({ want: false, reached, probe: "unlisted" }); + const plainhttp = (reached) => ({ want: false, reached, probe: "plainhttp" }); assert.equal(egressVerdict({ armed: false, results: [] }).warn, true); - assert.equal(egressVerdict({ armed: true, results: [{ want: true, reached: true }] }).warn, true, "one of two readings is not a read-back"); - assert.equal(egressVerdict({ armed: true, results: [{ want: true, reached: true }, { want: false, reached: false }] }).ok, true); - const leak = egressVerdict({ armed: true, results: [{ want: true, reached: true }, { want: false, reached: true }] }); + assert.equal(egressVerdict({ armed: true, results: [provider(true)] }).warn, true, "one of three readings is not a read-back"); + const two = egressVerdict({ armed: true, results: [provider(true), unlisted(false)] }); + assert.equal(two.warn, true, "two of three is not a read-back either"); + assert.equal(two.detail, "not read back: the egress canary did not run all three probes (see the egress lines above)"); + const holds = egressVerdict({ armed: true, results: [provider(true), unlisted(false), plainhttp(false)] }); + assert.equal(holds.ok, true); + assert.equal(holds.detail, "the provider was reached, an unlisted host was not, and plain HTTP off port 80 was refused"); + const leak = egressVerdict({ armed: true, results: [provider(true), unlisted(true), plainhttp(false)] }); assert.equal(leak.ok, false); assert.match(leak.detail, /unlisted host was reached/); - assert.equal(egressVerdict({ armed: true, results: [{ want: true, reached: true }, { want: false, reached: null }] }).warn, true, "a probe that did not run is no reading, never a deny"); + const plain = egressVerdict({ armed: true, results: [provider(true), unlisted(false), plainhttp(true)] }); + assert.equal(plain.ok, false); + assert.equal(plain.detail, "plain HTTP to a listed host off port 80 was let through"); + assert.equal(egressVerdict({ armed: true, results: [provider(true), unlisted(false), plainhttp(null)] }).warn, true, "a probe that did not run is no reading, never a deny"); assert.match(egressVerdict({ armed: null, results: [] }).detail, /could not be read/, "a malformed PI_EGRESS is not reported as off"); - const reachedWhileProviderUnread = egressVerdict({ armed: true, results: [{ want: true, reached: null }, { want: false, reached: true }] }); + const reachedWhileProviderUnread = egressVerdict({ armed: true, results: [provider(null), unlisted(true), plainhttp(false)] }); assert.equal(reachedWhileProviderUnread.ok, false); assert.notEqual(reachedWhileProviderUnread.warn, true, "a reached unlisted host is a finding even when the other probe did not run"); + const plainWhileUnread = egressVerdict({ armed: true, results: [provider(true), unlisted(null), plainhttp(true)] }); + assert.equal(plainWhileUnread.ok, false, "a wrong plain HTTP reading fails even when another probe did not run"); + assert.notEqual(plainWhileUnread.warn, true); + // Gate round 1 of #508: the three readings are counted BY NAME. A repeated probe, or readings that name none, are + // not a canary that ran all three, whatever their count. + for (const [why, results] of [ + ["a repeated probe", [provider(true), unlisted(false), unlisted(false)]], + ["no probe named", [{ want: true, reached: true }, { want: false, reached: false }, { want: false, reached: false }]], + ["four readings", [provider(true), unlisted(false), plainhttp(false), plainhttp(false)]], + ]) { + const got = egressVerdict({ armed: true, results }); + assert.equal(got.warn, true, why); + assert.equal(got.detail, "not read back: the egress canary did not run all three probes (see the egress lines above)", why); + } + // In any order. + assert.equal(egressVerdict({ armed: true, results: [plainhttp(false), provider(true), unlisted(false)] }).ok, true); +}); + +test("egressVerdict's probe names are doctor's canary slugs (#508)", async () => { + const { CANARY_PROBE_SLUGS } = await import("../src/doctor.mjs"); + const { EGRESS_PROBES } = await import("../src/live-probes.mjs"); + assert.deepEqual([...EGRESS_PROBES], [...CANARY_PROBE_SLUGS]); }); test("containerIdOf takes the ID docker run -d printed, and nothing else", () => { @@ -1420,10 +1454,10 @@ test("mountSet refuses Podman's API socket on sight, as it does docker's (#354)" }); test("egressVerdict with no readings points at the egress lines, and takes no caller's reason any more (#354, #431)", () => { - assert.equal(egressVerdict({ armed: true, results: [] }).detail, "not read back: the egress canary did not run both probes (see the egress lines above)"); + assert.equal(egressVerdict({ armed: true, results: [] }).detail, "not read back: the egress canary did not run all three probes (see the egress lines above)"); // Issue #431: the `unread` override existed for a podman venue with no canary of its own; that venue now runs one, so // a caller's reason is ignored rather than trusted, and every venue's missing reading points at its own canary lines. - assert.equal(egressVerdict({ armed: true, results: [], unread: "the canary runs on docker only" }).detail, "not read back: the egress canary did not run both probes (see the egress lines above)"); + assert.equal(egressVerdict({ armed: true, results: [], unread: "the canary runs on docker only" }).detail, "not read back: the egress canary did not run all three probes (see the egress lines above)"); assert.equal(egressVerdict({ armed: false, results: [] }).detail, "not read back: PI_EGRESS is off, so there is no policy to read back"); });