diff --git a/.github/scripts/openshell-conformance.mjs b/.github/scripts/openshell-conformance.mjs index cbe78787..7bbb70ed 100644 --- a/.github/scripts/openshell-conformance.mjs +++ b/.github/scripts/openshell-conformance.mjs @@ -17,8 +17,9 @@ * AND ONE CASE OF ITS OWN: the real job image's real runner, keyed, on the sandbox layout, offline under a cost cap of 0, * through the bundle, which must end on its own signed `cost-cap` line with the session brought back, and (issue #686) * whose session lines must reach the live transcript publisher as verified records over the exec's stderr. And (issue - * #686, part 2) the same runner STEERED: one message through the real relay down the exec's stdin as a frame, its ack - * back over the stderr transcript and its id on the signed exit line, still at zero spend. + * #686, part 2) the same runner STEERED: two messages through the real relay down the exec's stdin as frames, the + * second written mid-run, their acks back over the stderr transcript and their ids on the signed exit line, still at + * zero spend (issue #725: on a zero-rated fake model the workflow's stand-in serves, so the run lives long enough). * * STRICTER THAN THE HARNESS, as podman-conformance.mjs is: any abstention among the read-back properties fails. Run it * with the worker STOPPED: the harness calls the real `reap`, which deletes every sandbox carrying this worker's owner @@ -45,6 +46,7 @@ const { buildRecord, makeLogSink } = await load("worker/src/run-history.mjs"); const { networkPolicyFor } = await load("worker/src/openshell-policy.mjs"); const { makeLiveTranscriptOpener } = await load("worker/src/live-transcript.mjs"); const { makeInboxRelay } = await load("worker/src/inbox-relay.mjs"); +const { KEYLESS_API_KEY } = await load("worker/src/model-endpoints.mjs"); const refuse = (why) => { console.error(`openshell-conformance: ${why}`); @@ -121,12 +123,58 @@ case "$c" in echo "probe bother=$(node -e 'fetch("http://host.openshell.internal:18080/bundle-elsewhere",{headers:{authorization:"Bearer "+process.env.BUNDLE_TOKEN},signal:AbortSignal.timeout(10000)}).then(r=>console.log(r.status),e=>console.log("refused",e.cause?.code??e.name))')" echo "probe bkey=$(node -e 'fetch("http://host.docker.internal:18080/key-at-bundle-host",{headers:{"x-api-key":process.env.ANTHROPIC_API_KEY},signal:AbortSignal.timeout(10000)}).then(r=>console.log(r.status),e=>console.log("refused",e.cause?.code??e.name))')" exit 0 ;; + netpath) + # Issue #725 diagnostics: the routes a model call can take from this sandbox to the stand-in, each printed and + # none asserted. The policy probes above fetch with the image's plain node and a GET; a job's provider call is a + # streamed POST from the runner's own binary, after pi has replaced the fetch dispatcher (docs/egress.md, issue + # #427), through the OpenAI SDK. Each step below changes one of those, so the first one to fail names the cause. + # Under /opt: the sandbox policy reads only its listed paths (sandboxPolicy's read_only), so a file at / is unreadable + # and the first run printed nothing. Each binary's exit and the head of its stderr are printed, so a load failure + # is a line too. + node /opt/pi-dispatch-probe/netpath.mjs node 2>/tmp/netpath.err; echo "probe netpath node exit=$?" + head -c 2000 /tmp/netpath.err | head -n 8 | sed 's/^/probe netpath node stderr=/' + /opt/pi-dispatch/runner-node /opt/pi-dispatch-probe/netpath.mjs runner 2>/tmp/netpath.err; echo "probe netpath runner exit=$?" + head -c 2000 /tmp/netpath.err | head -n 8 | sed 's/^/probe netpath runner stderr=/' + exit 0 ;; *) exit "$c" ;; esac `, { mode: 0o755 }, ); -nodeFs.writeFileSync(join(context, "Dockerfile"), `FROM ${image}\nUSER root\nCOPY entrypoint.sh /entrypoint.sh\nRUN chmod 0755 /entrypoint.sh\nUSER pi\n`); +// Issue #725 diagnostics, run by the `netpath` mode: one line per route, `probe netpath =`. The +// POST carries a `tool` message, so the stand-in answers it at once ("done") rather than holding it for the steering +// case's signal. Errors are printed by name and cause code (and a short cause message), never a body. +nodeFs.writeFileSync( + join(context, "netpath.mjs"), + `const who = process.argv[2]; +const base = "http://host.openshell.internal:18080"; +const body = JSON.stringify({ model: "m1", stream: true, messages: [{ role: "user", content: "x" }, { role: "assistant", content: null, tool_calls: [{ id: "c", type: "function", function: { name: "read", arguments: "{}" } }] }, { role: "tool", tool_call_id: "c", content: "x" }] }); +const why = (e) => { const c = e?.cause; return [e?.name, e?.constructor?.name, c?.code ?? c?.cause?.code ?? "", String(c?.message ?? e?.message ?? "").slice(0, 160)].join(" "); }; +const step = async (label, fn) => { try { console.log("probe netpath " + who + " " + label + "=" + (await fn())); } catch (e) { console.log("probe netpath " + who + " " + label + "=error " + why(e)); } }; +const get = async () => { const r = await fetch(base + "/", { signal: AbortSignal.timeout(10000) }); return "status " + r.status; }; +const post = (auth) => async () => { const r = await fetch(base + "/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json", ...(auth ? { authorization: "Bearer keyless" } : {}) }, body, signal: AbortSignal.timeout(15000) }); const t = await r.text(); return "status " + r.status + " " + (r.headers.get("content-type") ?? "") + " " + (t.includes("[DONE]") ? "complete" : "partial") + " " + t.length; }; +await step("get", get); +await step("post-sse-noauth", post(false)); +await step("post-sse", post(true)); +if (who === "runner") { + await step("load-pi", async () => { const m = await import("/app/image/runner/src/env-proxy.mjs"); return "env proxy restored " + (await m.loadPiThenRestore()); }); + await step("pi-get", get); + await step("pi-post-sse", post(true)); + await step("pi-openai-sdk", async () => { + const { createRequire } = await import("node:module"); + const { pathToFileURL } = await import("node:url"); + const at = createRequire("/app/node_modules/@earendil-works/pi-ai/package.json").resolve("openai"); + const OpenAI = (await import(pathToFileURL(at).href)).default; + const client = new OpenAI({ apiKey: "keyless", baseURL: base + "/v1", maxRetries: 0, timeout: 15000 }); + const stream = await client.chat.completions.create({ ...JSON.parse(body) }); + let n = 0; + for await (const _ of stream) n += 1; + return "chunks " + n; + }); +} +`, +); +nodeFs.writeFileSync(join(context, "Dockerfile"), `FROM ${image}\nUSER root\nCOPY entrypoint.sh /entrypoint.sh\nCOPY netpath.mjs /opt/pi-dispatch-probe/netpath.mjs\nRUN chmod 0755 /entrypoint.sh && chmod 0755 /opt/pi-dispatch-probe && chmod 0644 /opt/pi-dispatch-probe/netpath.mjs\nUSER pi\n`); const probeImage = `pi-dispatch-openshell-probe:${nonce}`; const built = await run(engine, ["build", "--pull=false", "-q", "-t", probeImage, context]); if (built.code !== 0) refuse(`the probe image did not build: ${built.stderr.trim().split("\n").slice(-1)[0]}`); @@ -393,22 +441,40 @@ for (const [property, why] of Object.entries(UNVERIFIED_BY_THIS_HARNESS)) consol if (!ok) failed = true; } +// Issue #725 diagnostics: the model-call routes from a copy-mode sandbox under the probes' policy, printed and never +// asserted (the steering case below is the assertion). Run before it so its answer is in the log whatever it does. +{ + output.set("last", ""); + const result = await backend.runContainer({ job: JOB, prepared: jobDirFor("netpath"), name: `pi-job-conformance-${nonce}-netpath`, signal: new AbortController().signal, size: { memMiB: 1024, cpuCenti: 100 } }); + const lines = String(output.get("last") ?? "").split("\n").filter((l) => l.startsWith("probe netpath ")); + console.log(`diag the model-call routes (exit ${result.code}):\n${lines.map((l) => ` ${l.slice("probe netpath ".length)}`).join("\n") || " none printed"}`); +} + // Issue #686, part 2: the real runner STEERED over the exec's stdin. The real relay (inbox-relay.mjs) over an in-memory // inbox holding one queued steer, the real publisher with the steer ack consumer (the opener start.mjs builds), and the // bundle handed both, as start.mjs hands them. A fresh job's relay is released at the spawn (an empty upload has no -// prior line), so the FIRST message's frame travels right behind the key frame, before the runner has loaded pi. Still at -// a cost cap of 0: pi polls its steering queue once at the start of its loop, BEFORE the first model call the cap -// refuses, so that message is persisted (and acked) at zero spend. Asserted: the job ends on its own signed cost-cap -// line, the ack came back through the stderr transcript (the ack consumer verified one), the signed exit line's -// deliveredIds names it, the inbox calls it delivered, and the stream never broke and carried no forged frame. -// A SECOND message is queued the moment the first one's ack arrives, and the relay is ticked from that very mark (not a -// later turn of the event loop), so its frame is written mid-run, on the open pipe, before the exit line's close (M1) -// can run: the tick's whole pass is microtasks over this in-memory inbox, and the close waits for a stdout read. Asserted: -// the relay wrote two frames, and writing the second mid-run broke nothing (no stream break, no unverified frame, no -// relay fault); held 6 of 6 on Docker. Whether the runner still READS it is not asserted: at zero spend the run is over -// within milliseconds of that ack (its only model call is refused), so it ends `delivered` or `written` (the settle then -// calls it late), printed. A delivery mid-run is shown with spend by the runner's own test -// (image/runner/test/inbox.test.mjs, "the real run-job on the stdin frame channel"). +// prior line), so the FIRST message's frame travels right behind the key frame, before the runner has loaded pi, and pi +// takes it at the start of its loop, before its first model call. +// A SECOND message is queued the moment the first one's ack arrives, and the relay is ticked from that very mark, so its +// frame is written mid-run, on the open pipe. +// Issue #725: the model is a zero-rated fake at the workflow's stand-in (provider `steer-fake`, keyless, every rate 0). +// Its answer to the first request is HELD until this script touches STEER_SIGNAL_FILE, which it does once the relay has +// written the second frame, and then asks for `read` of the job's prompt file; its answer to a request carrying that +// tool result says "done". This case used to run on the job provider under a cost cap of 0, which refuses the only +// model call, so the run ended at the cap within milliseconds of the first ack; the second message then raced the exit +// line's close of stdin (M1) and was left `queued` with one frame written on some runs. Now the run cannot pass its +// first model call before the second frame is on the pipe, and pi delivers it with its second request, so both are +// asserted delivered. Still at zero spend: the cap +// stays 0 and a zero-rated model runs under it (usage-meter.mjs, `callCostBound`), and no cost refusal may appear. +// Asserted: the job ends clean (exit 0) on its own signed exit line, whose deliveredIds names BOTH messages; the acks +// came back through the stderr transcript (the ack consumer verified them); the inbox calls both delivered; the relay +// wrote two frames; and the stream never broke and carried no forged frame. The delivery with spend is still shown by +// the runner's own test (image/runner/test/inbox.test.mjs, "the real run-job on the stdin frame channel"). +// The keyless provider's declaration, as an operator writes one for a local model server (docs/egress.md, "Local model +// servers"): `apiKey` is the keyless placeholder the worker fills, so no job credential (ANTHROPIC_API_KEY here) is +// given to this provider or made into an OpenShell provider for it. +const STEER_FAKE_MODEL = { id: "m1", name: "m1", api: "openai-completions", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 100000, maxTokens: 1000, compat: { maxTokensField: "max_tokens" } }; +const STEER_FAKE_MODELS = { providers: { "steer-fake": { baseUrl: "http://host.openshell.internal:18080/v1", apiKey: KEYLESS_API_KEY, api: "openai-completions", models: [STEER_FAKE_MODEL] } } }; { const logs = join(scratch, "logs-steer"); nodeFs.mkdirSync(logs); @@ -456,23 +522,65 @@ for (const [property, why] of Object.entries(UNVERIFIED_BY_THIS_HARNESS)) consol return h; }; const openInboxRelay = (args) => (relayRef = makeInboxRelay({ redis: {}, host: "conformance", log, pollMs: 500, inbox, ...args })); - const real = bundleFor(image, { ...POLICY_SOURCES, openJobLog: makeLogSink({ logsDir: logs, enabled: false }), openLiveTranscript, openInboxRelay, log }); - const dirs = jobDirFor("Reply with the single word ok."); + // The stand-in's release (the workflow sets the path for both): removed first so a rerun starts held, touched once + // the relay has written the second frame. Without the variable the stand-in answers after its 30 s cap anyway. + // The stand-in's completion requests so far (the diagnostics above sent some), so a failure reports this run's own. + const standinCompletions = () => { + let seen = []; + try { + seen = nodeFs.readFileSync(process.env.STANDIN_LOG ?? "", "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)); + } catch {} + return seen.filter((e) => e.path === "/v1/chat/completions").map((e) => `${e.method} ${e.path}`); + }; + const completionsBefore = standinCompletions().length; + const signalFile = process.env.STEER_SIGNAL_FILE ?? ""; + if (signalFile !== "") nodeFs.rmSync(signalFile, { force: true }); + const signal = setInterval(() => { + if ((relayRef?.counts().written ?? 0) < 2) return; + clearInterval(signal); + if (signalFile !== "") nodeFs.writeFileSync(signalFile, ""); + }, 50); + // The overlay holding the fake's models.json, uploaded with the job in copy mode as an operator's overlay is. + const overlay = join(scratch, "overlay-steer"); + nodeFs.mkdirSync(overlay); + nodeFs.writeFileSync(join(overlay, "models.json"), `${JSON.stringify(STEER_FAKE_MODELS)}\n`); + // The pickup's endpoint snapshot, as the processor hands it (index.mjs): the same two endpoints the policy is built + // from (so the policy is the probes'), the stand-in's first alias declared keyless. The credential gate inside + // runContainer reads it: a provider pi does not know passes only as keyless (env-allowlist.mjs, `keylessEndpointsFor`). + const modelEndpoints = { endpoints: POLICY_SOURCES.readEndpoints().map((e) => ({ ...e, keyless: e.id === "lab-a" })), models: STEER_FAKE_MODELS }; + const real = bundleFor(image, { ...POLICY_SOURCES, globalPiDir: overlay, openJobLog: makeLogSink({ logsDir: logs, enabled: false }), openLiveTranscript, openInboxRelay, log }); + const dirs = jobDirFor("Read your prompt file, then reply with the single word done."); output.set("last", ""); - const result = await real.runContainer({ job: { ...JOB, maxCostMicros: 0, steer: true }, prepared: dirs, name: `pi-job-conformance-${nonce}-steer`, signal: new AbortController().signal, exitAuth: true, transcriptStderr: true, steerStdin: true, jobId: `conformance-steer-${nonce}`, size: { memMiB: 2048, cpuCenti: 200 } }); + const steerJob = { ...JOB, provider: "steer-fake", model: "m1", maxTurns: 2, maxCostMicros: 0, steer: true }; + const result = await real.runContainer({ job: steerJob, prepared: dirs, modelEndpoints, name: `pi-job-conformance-${nonce}-steer`, signal: new AbortController().signal, exitAuth: true, transcriptStderr: true, steerStdin: true, jobId: `conformance-steer-${nonce}`, size: { memMiB: 2048, cpuCenti: 200 } }); + clearInterval(signal); const said = String(output.get("last") ?? ""); const ackVerified = handles[0]?.stats?.().steer?.verified ?? 0; const exitLine = said.split("\n").filter((l) => l.startsWith('{"event":"exit"')).at(-1) ?? ""; - const named = exitLine.includes(`"deliveredIds":["${id}"`); + let exit = null; + try { + exit = JSON.parse(exitLine); + } catch {} + const deliveredIds = Array.isArray(exit?.steer?.deliveredIds) ? exit.steer.deliveredIds : []; + const named = deliveredIds.includes(id) && deliveredIds.includes(second); + // At zero spend by construction: the runner's guard refused nothing (no `cost_refused` line, the exit line's own + // `costRefused` 0, measured so on the runner against this stand-in), and the run did not end at the cap. + const refusedCost = /"event":"cost_refused"/.test(said) || (exit?.tokens?.costRefused ?? 0) !== 0 || (exit?.tokens?.cost ?? 0) !== 0 || result.exitReason === "cost-cap"; const broken = /"event":"steer_(stream_broken|frame_unverified|stdin_shared)"/.test(said) || events.some(([e]) => e === "steer_relay_fault" || e === "steer_skipped" || e === "steer_frame_refused" || e === "steer_acks_unread"); const secondOutcome = state[second] ?? "never queued"; const written = relayRef?.counts().written ?? 0; - const secondHonest = (secondOutcome === "delivered" || secondOutcome === "written") && written >= 2; - const ok = result.code === 2 && result.exitReason === "cost-cap" && result.exitAuth === "verified" && (result.steer?.delivered ?? 0) >= 1 && ackVerified >= 1 && named && state[id] === "delivered" && !broken && second in state && secondHonest; - console.log(`${ok ? "ok " : "FAIL "} steering over the exec's stdin: exit ${result.code} ${result.exitReason} (${result.exitAuth}), runner delivered ${result.steer?.delivered ?? "none"}, acks verified over stderr ${ackVerified}, exit line names the first ${named}, inbox says ${state[id]}; the mid-run second: ${secondOutcome} (${written} frames written, 2 wanted); marks ${JSON.stringify(marks)}, broken or refused ${broken}`); + // Delivered and nothing less: the exit line names it (so the runner saw pi persist it), and the worker marks what a + // verified exit line names before the relay stops, after draining the acks, so `written` here would be a defect. + const secondDelivered = secondOutcome === "delivered" && written >= 2; + const ok = result.code === 0 && result.exitAuth === "verified" && !refusedCost && (result.steer?.delivered ?? 0) >= 2 && ackVerified >= 2 && named && state[id] === "delivered" && !broken && secondDelivered; + console.log(`${ok ? "ok " : "FAIL "} steering over the exec's stdin: exit ${result.code} ${result.exitReason ?? "none"} (${result.exitAuth}), cost refused ${refusedCost}, runner delivered ${result.steer?.delivered ?? "none"}, acks verified over stderr ${ackVerified}, exit line names both ${named} (${JSON.stringify(deliveredIds)}), inbox says ${state[id]}; the mid-run second: ${secondOutcome} (${written} frames written, 2 wanted); marks ${JSON.stringify(marks)}, broken or refused ${broken}`); if (!ok) { console.log(said.split("\n").slice(-20).join("\n")); console.log(JSON.stringify(events.filter(([e]) => /steer|transcript/.test(e)))); + // Whether the model calls reached the stand-in at all (issue #725): its log's completion requests, by method and + // path only (the log holds no body; the credential headers are left out here too). + const calls = standinCompletions().slice(completionsBefore); + console.log(`the stand-in saw ${calls.length} completion request(s) during this run${calls.length > 0 ? `: ${JSON.stringify(calls)}` : ""}`); failed = true; } } diff --git a/.github/workflows/openshell-conformance.yml b/.github/workflows/openshell-conformance.yml index 689f17b6..ce093018 100644 --- a/.github/workflows/openshell-conformance.yml +++ b/.github/workflows/openshell-conformance.yml @@ -97,13 +97,50 @@ jobs: # host aliases. It records what each request carried in its credential headers (only fake values reach it), so the # script can tell a swapped value from a placeholder, and asks git for Basic auth on info/refs. Issue #704 part 3: # its second alias (host.docker.internal) is the secret-bundle case's endpoint host, read off the same log. + # Issue #725: it is also the steering case's model, a zero-rated OpenAI-compatible fake (`POST /v1/chat/completions`, + # streamed). A request whose body holds no `tool` message (the run's first, or pi's retry of it) is HELD until the + # conformance script touches STEER_SIGNAL_FILE, which it does once the relay has written the second message's + # frame (at most 30 s, then it answers anyway and the case fails on its own assertions), and then asks for `read` + # of the job's prompt file; any request carrying a tool result answers "done". Held on that event and not a + # timer, so the second frame is written mid-run however slow the host; chosen by the body and not a request + # count, so a retry is held again. The body is read, never logged: the log line stays method, path and the two + # credential headers, as for every other route, and every other route answers exactly as before. - name: A stand-in server on the gateway host run: | cat > "$RUNNER_TEMP/standin.mjs" <<'JS' import { createServer } from "node:http"; - import { appendFileSync } from "node:fs"; + import { appendFileSync, existsSync } from "node:fs"; + const signalled = async () => { + for (let waited = 0; waited < 30000 && !existsSync(process.env.STEER_SIGNAL_FILE ?? ""); waited += 100) await new Promise((r) => setTimeout(r, 100)); + }; + const completion = (req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", async () => { + let messages = []; + try { + messages = JSON.parse(body).messages ?? []; + } catch {} + const first = !messages.some((m) => m?.role === "tool"); + res.writeHead(200, { "content-type": "text/event-stream" }); + const chunk = (payload) => res.write(`data: ${JSON.stringify(payload)}\n\n`); + const base = { id: "r", object: "chat.completion.chunk", created: 0, model: "m1" }; + const usage = { prompt_tokens: 10, completion_tokens: 3, total_tokens: 13 }; + if (first) { + await signalled(); + chunk({ ...base, choices: [{ index: 0, delta: { role: "assistant", tool_calls: [{ index: 0, id: "call_read", type: "function", function: { name: "read", arguments: JSON.stringify({ path: "/sandbox/job/prompt.md" }) } }] }, finish_reason: null }] }); + chunk({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage }); + } else { + chunk({ ...base, choices: [{ index: 0, delta: { role: "assistant", content: "done" }, finish_reason: null }] }); + chunk({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage }); + } + res.write("data: [DONE]\n\n"); + res.end(); + }); + }; createServer((req, res) => { appendFileSync(process.env.STANDIN_LOG, JSON.stringify({ method: req.method, path: req.url, xApiKey: req.headers["x-api-key"] ?? null, authorization: req.headers.authorization ?? null }) + "\n"); + if (req.method === "POST" && req.url === "/v1/chat/completions") return completion(req, res); if (req.url.includes("/info/refs") && !req.headers.authorization) { res.writeHead(401, { "WWW-Authenticate": 'Basic realm="standin"' }); return res.end(); @@ -112,10 +149,10 @@ jobs: res.end(""); }).listen(18080, "0.0.0.0"); JS - STANDIN_LOG="$RUNNER_TEMP/standin.jsonl" nohup node "$RUNNER_TEMP/standin.mjs" > "$RUNNER_TEMP/standin.log" 2>&1 & + STEER_SIGNAL_FILE="$RUNNER_TEMP/steer-second-written" STANDIN_LOG="$RUNNER_TEMP/standin.jsonl" nohup node "$RUNNER_TEMP/standin.mjs" > "$RUNNER_TEMP/standin.log" 2>&1 & - name: Conformance (the real bundle, the real runner) - run: STANDIN_LOG="$RUNNER_TEMP/standin.jsonl" GATEWAY_LOG="$RUNNER_TEMP/gateway.log" PI_JOB_IMAGE=pi-job:ci node .github/scripts/openshell-conformance.mjs + run: STEER_SIGNAL_FILE="$RUNNER_TEMP/steer-second-written" STANDIN_LOG="$RUNNER_TEMP/standin.jsonl" GATEWAY_LOG="$RUNNER_TEMP/gateway.log" PI_JOB_IMAGE=pi-job:ci node .github/scripts/openshell-conformance.mjs # Issue #686 part 3: bind mode. First on the gateway as it is, whose default settings must refuse it before spend # (the probe create refused with upstream's allow_driver_config sentence). diff --git a/specs/design.md b/specs/design.md index 9966c259..c3f857b0 100644 --- a/specs/design.md +++ b/specs/design.md @@ -11151,7 +11151,22 @@ a tunnel. host can be made ready before #684; they add the engine floor (Docker 28, Podman 5) and the job image's `sandboxLayout` to the plan's list. The boot half of the version floor is #684's, with the blessing. The CI lane is ADVISORY (`the openshell venue holds its declarations`), promoted to required in a later issue once it has passed - across a round. + across a round. **AMENDED 2026-10-21 (#725)**: the copy-mode steering case runs on a zero-rated fake model the + lane's stand-in serves (`POST /v1/chat/completions` on the gateway host, a keyless overlay provider `steer-fake` + uploaded as the job's overlay, every rate 0, the cap still 0). Its answer to a request with no tool result is HELD + until the conformance script signals that the relay has written the second frame (a file the script touches, at + most 30 s), then asks for `read` of the job's prompt file; a request carrying the tool result is answered "done". + The hold is on that event and not a timer, so the second frame is written mid-run however slow the host, and it is + chosen by the request body and not a count, so a retry of the first request is held again. It ran on the job + provider under a cost cap of 0, which refuses the only model call, so the run ended at the cap within milliseconds + of the first ack and the second message raced the exit line's close of stdin: left `queued` with one frame written + on some runs, a red lane on a truthful outcome. The case now requires a clean exit 0 on the signed exit line naming + both messages, both delivered, two frames written, no cost refusal and no broken stream. REJECTED accepting `queued` + as the honest outcome: it would stop proving the mid-run write the case exists for; REJECTED a relay change (a tick + asked for mid-pass run again): nothing in production ticks from a mark, the failure was the pipe already closed, and + it would only add back-to-back passes when Valkey is slow. Residual, named: this case is the first to stream a + model's answer (server-sent events) through OpenShell 0.1.4's `protocol: rest` layer, so a red run here may be that + layer's handling of a streamed response rather than steering. - **Credentials and git push (#684 part 2, AMENDED 2026-10-19)**: the "Credentials through per-job providers" plan above, as built and measured on OpenShell 0.1.4. Per job, `secretNamesOf` takes every env name the redactor does not list as non-secret (`NON_SECRET_ENV_KEYS`, the one list), and the adapter imports an ENDPOINTLESS profile for them @@ -13685,3 +13700,4 @@ a tunnel. | 2026-10-21 (#712) | Issue #712, PR C: doctor results per host. **NEW `DES-DOCTOR-RESULTS-PUBLISHED`**: every doctor run publishes its checks per host (`doctor:h:`, 7 days), and the worker runs `pi-dispatch doctor --publish` as a child a minute after boot and every `PI_DOCTOR_INTERVAL_HOURS`; what is published (ids and levels; label and fix on a warn or fail only, redacted: paths to ``, URLs to scheme and host, secret values and token shapes to `[redacted]`, controls out, 300 characters), how ids are made (named where a console sentence matches, else `
.` over 33 pinned sections), best effort, the child's bounds (120 s kill, no overlap, killed on close, names and integers in its log lines); why the worker reruns doctor (a host nobody looks at is the one nobody has checked), why the redaction is stricter than the terminal's, why the labels are for both roles; rejected: doctor in the worker's process, raw labels, no TTL, a HASH, publishing only on hand runs, fix text only when fixed; residuals named. **`DES-OPERATOR-CONSOLE` AMENDED**, a dated "Doctor results" bullet: the snapshot's `doctor`, read at most every 15 s and never pruned from the console, the Fleet page's Doctor panel, the quoted "does not load" sentences, the HOSTS view's lines, both roles, not on `/api/state`. **`DES-RETENTION-SWEEPS-ON-A-TIMER` UNCHANGED, checked**: its interval idiom is reused by the doctor spawner, two divergences marked there (a delayed first run, a kill on close); the sweep itself is untouched. **Code evidence**: worker/src/doctor-results.mjs; worker/src/doctor-spawner.mjs; worker/src/doctor.mjs -> collectChecks, publishResults; worker/src/start.mjs; admin/src/doctor-view.mjs; admin/src/console/panels.mjs -> fleetDoctor, doctorReason; admin/src/dashboard.ts -> doctorLines. Review round 1, folded in before merge: **`DES-DOCTOR-RESULTS-PUBLISHED` AMENDED**, a Review round 1 bullet (the JSON.parse quote cut, the path scanner, live hosts before the index and the writer's prune, this host's record first in a quoted sentence, `by` and `workerRun`, the refused worker name, the internal `--by-worker`, the stderr tail and the process-group kill) and the residuals; **`DES-CLI-SURFACE` AMENDED**: doctor stays on the read-only tier, stated as read-only on the HOST with one write, its own results to Valkey, a display key nothing decides on. Review round 2, folded in before merge: **`DES-DOCTOR-RESULTS-PUBLISHED` AMENDED**, a Review round 2 bullet (invisible characters deleted first, the JSON.parse cut anchored to the message's start, more credential forms, numeric knob values not secrets, `--by-worker` bounded) and the residuals named: fullwidth and relative paths, the scp user name, the detached child that outlives a crashed worker. | | 2026-10-21 | Issue #712, part B: the Venues view. **`DES-OPERATOR-CONSOLE` AMENDED**, a dated bullet "The Venues view": the Fleet page's Venues panel, the hosts tables' Venues column and the panel's VENUES view (`v` from HOSTS) from one model, `venuesView`, over the snapshot's host rows (`hostRowOf` keeps `backends` and `caps`); every property as the table's question and word with `DECLARATION_TEXT`, `PROPERTY_LABEL`, `GATEWAY_TEXT`, each bolted by `admin/test/venues-view.test.mjs`; nothing resolved live (two rejected options: a live check from the console, a column on the HOSTS lines); the run lists' venue (a console column, a panel cell off `local` only, a `/dispatch runs` column only when a run is off `local`) and the `backend` runs filter (by `HOST_VENUE_NAME_RE`, a newer worker's venue included); a race fixed: a 7-day read that lands while VENUES is open is kept for HOSTS; the README images regenerated and the fixture's records naming `local`. **`DES-HOST-REGISTRY` UNCHANGED, checked**: the new field is display only, so deleting `host:*` still changes nothing (its contract is `INT-HOST-REGISTRY-CONTRACT`'s). **Code evidence**: worker/src/backends.mjs -> HOST_GATEWAY_WORDS, hostBackendsField, parseHostBackends; worker/src/backend-openshell.mjs -> openshellBootVersions (`gatewayLocal`); worker/src/start.mjs (the `backends` field); admin/src/read-model.mjs -> hostRowOf, venuesView, DECLARATION_TEXT, PROPERTY_LABEL, GATEWAY_TEXT; admin/src/console/panels.mjs -> fleetVenues, blessedText, runVenue; admin/src/dashboard.ts -> venuesPanelView, recordVenue; admin/src/render.mjs (VENUE); admin/src/run-filter.mjs (`backend`); launch/fixture-deployment.mjs. | | 2026-10-21 (#712) | Issue #712, part D: OpenShell run facts and the secret profiles view. **`DES-OPENSHELL-VENUE` AMENDED**, a dated "Recorded run facts" bullet: the stage sets `openshell` (`mode`, the written policy's endpoint hosts, the providers made, the keys they bind) once every provider exists, `collectRun` carries it on every result, the settle carries it to the record; the hosts are operator config (allowlist, model endpoints, the forge's fixed host; the repository is a push-rule path and is not read), held to the host-name charset and capped at 32 with the total counted; counts for credentials, never names; MCP connection state is named as not recorded (#684 records none). **`DES-OPERATOR-CONSOLE` AMENDED**, a dated bullet: the run page's "Sandbox policy", "Rule hosts" and "Secrets profile" rows and the panel's RUN_DETAIL lines; the configuration's "Secret profiles in use" table, the panel's secret profiles section and `/dispatch secrets list`'s profile lines (`secretProfilesView`: source, the roots by `resolverRootsStatus`, which the add's judgement now reads too, the triggers and projects, the last resolve as refused, resolved or not reached, and the hosts advertising `s:` through `hostRowOf`'s new `caps`); no resolver path on the console or the panel; `dispatch_runs` returns the facts and the profile name rebuilt, never a resolver path. Review round 1: the add's `resolverRootsJudgement` stays strict on its fs seam (no realpath throws) and only the view tolerates a row it cannot judge (`unresolved`); `hostRowOf` carries `capsSeen` so a host that publishes no list is counted; a `BUNDLE_SHARED_REASONS` refusal on a bundled run is said as either read's; `dispatch_runs`' list carries only the host count; no facts for a stage thrown or stopped before the exec or a detached run, said so; the bundles view's lexical roots check named as a residual. **`DES-CONTAINER-BACKEND-REGISTRY` UNCHANGED, checked**: no `serves` word or custody changes; the facts are recorded, not declared. **`DES-PROJECT-SECRET-BUNDLES` UNCHANGED, checked**: the bundle's own record keys and surfaces are as #704 left them; the openshell `placeholders` count includes the bundle hosts' keys beside `bundle.placeholders`, which it does not replace. **Code evidence**: worker/src/backend-openshell.mjs -> stageRun, collectRun; worker/src/index.mjs; worker/src/run-history.mjs -> recordedOpenshell, recordedSecretsProfile; worker/src/capabilities.mjs -> secretProfileNeeded (exported); admin/src/read-model.mjs -> secretProfilesView, resolverRootsStatus; admin/src/console/panels.mjs; admin/src/dashboard.ts; admin/src/secrets-command.ts. | +| 2026-10-21 (#725) | Issue #725: the openshell lane's steering race. **`DES-OPENSHELL-VENUE` AMENDED**, the "Doctor and CI" bullet: the copy-mode steering case runs on a zero-rated keyless fake model at the lane's stand-in (`POST /v1/chat/completions`), whose first answer is held until the script signals the second frame written (at most 30 s) and then asks for `read`, chosen by the request body so a retry is held again; the case requires exit 0 with both messages delivered and named on the signed exit line, two frames written and no cost refusal, instead of a `cost-cap` exit that raced the second write. Rejected: accepting `queued` as the outcome, and a relay rerun change (not the cause: the pipe was already closed). Residual named: the first streamed model answer through OpenShell 0.1.4's rest layer. The lane stays ADVISORY. **UNCHANGED, checked**: `DES-STEER-VIA-READONLY-INBOX-MOUNT` and `INT-STEER-INBOX-CONTRACT` (no relay or inbox change), `DES-DOLLAR-RESERVE-AND-SETTLE` (a zero-rated model under a cap of 0 is the meter's existing rule), the stand-in's other routes (answered and logged as before). **Code evidence**: .github/scripts/openshell-conformance.mjs; .github/workflows/openshell-conformance.yml. |