From 189adf785f9406bbe64880fdc29456778f263d8e Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Sat, 26 Sep 2026 00:21:10 +0800 Subject: [PATCH 1/4] fix(presets): admit reviewed ACP endpoint connections (cherry picked from commit 21fba171ad24d56219cb52d06eff215673df3c7c) --- docs/provider-credential-references.md | 30 ++++----- .../control-core/src/inference-bindings.ts | 8 ++- packages/control-core/src/presets.ts | 5 +- .../control-core/test/preset-endpoint.test.ts | 62 +++++++++++++++++++ 4 files changed, 88 insertions(+), 17 deletions(-) diff --git a/docs/provider-credential-references.md b/docs/provider-credential-references.md index 99f905c8..ae913e5b 100644 --- a/docs/provider-credential-references.md +++ b/docs/provider-credential-references.md @@ -57,20 +57,22 @@ style without a connection is refused before any run record is written. A submission that names no connection keeps the ordinary router route, so approving a credential never changes where an otherwise identical submission runs. -`model_api` is Harbor's own agent argument: at a custom endpoint Harbor requires -it and `pi` writes it into its `models.json` `api` value. The grant stores the -value it reviewed, and admission requires the built record to carry exactly that -value, so a record whose wire API style changed after the build is denied. Only -the submission names the connection and its wire API style; no preset field, no -alias table and no submission-supplied URL is added. - -The run record for such a connection is the ordinary `openai/` route with -the reviewed base URL and an opaque key reference. Admission and restart recheck -that record against the approved grant. A model outside the grant, another preset -version, another worker image, another operator, a changed base URL, a changed -host list or a changed wire API style is denied. A connection the submission -names but the registry does not hold for this exact subject is denied as well, -and the run never falls back to the router. +For agents that accept Harbor's `model_api` option, the built record carries the +approved value. For example, `pi` writes it into the `api` field of its +`models.json`. Admission checks that value again at each start. + +Harbor's ACP agent does not accept `model_api`. A reviewed ACP source configures +its own client. The submission must still name the API style approved by the +grant, but the built record does not add an unsupported ACP option. At each start, +admission requires exactly one matching grant for the agent import path and +version, model, operator, worker image and credential reference. It refuses an +ambiguous grant or an ACP record with an injected `model_api` option. The +separate source allowlist checks the pinned executable source. + +The run record uses `openai/`, the reviewed base URL and an opaque key +reference. Admission checks the URL and hosts again at each start. A connection +without an approved grant for the exact subject is denied; the run cannot fall +back to the router. ## Deployment caveat diff --git a/packages/control-core/src/inference-bindings.ts b/packages/control-core/src/inference-bindings.ts index 56203adf..57f7d899 100644 --- a/packages/control-core/src/inference-bindings.ts +++ b/packages/control-core/src/inference-bindings.ts @@ -379,12 +379,16 @@ export class InferenceBindings { kwargs && typeof kwargs === "object" && !Array.isArray(kwargs) ? (kwargs as Record).model_api : undefined; + // ACP has no native model_api option. Its pinned source configures the client; + // admission still requires one unambiguous reviewed grant for the native + // agent identity, model, actor and image at every start or restart. + const isAcp = subject.import_path === "harbor.agents.installed.acp:AcpAgent"; const matches = - modelId === null + modelId === null || (isAcp && modelApi !== undefined) ? undefined : binding?.uses.filter( (use) => - use.model_api === modelApi && + (isAcp || use.model_api === modelApi) && this.presetMatches(use, subject, modelId, actor, image), ); const grant = matches?.[0]; diff --git a/packages/control-core/src/presets.ts b/packages/control-core/src/presets.ts index aa62862d..2a19c34d 100644 --- a/packages/control-core/src/presets.ts +++ b/packages/control-core/src/presets.ts @@ -112,7 +112,10 @@ export function presetEndpointAgent( ); if (preset.reasoning_option !== null && reasoningEffort !== "default") kwargs[preset.reasoning_option] = reasoningEffort; - kwargs.model_api = modelApi; + // Harbor's ACP runner takes a pinned source, not a model_api argument. The + // reviewed grant selects the wire API style; the source configures its client. + if (fragment.import_path !== "harbor.agents.installed.acp:AcpAgent") + kwargs.model_api = modelApi; return { ...base, ...(fragment.name ? { name: fragment.name } : {}), diff --git a/packages/control-core/test/preset-endpoint.test.ts b/packages/control-core/test/preset-endpoint.test.ts index a5a64178..b956ccb9 100644 --- a/packages/control-core/test/preset-endpoint.test.ts +++ b/packages/control-core/test/preset-endpoint.test.ts @@ -198,6 +198,68 @@ describe("reviewed endpoint connections for native presets", () => { expect(admit(agent)?.source).toBe(source); }); + it("admits a pinned ACP source without adding an unsupported model_api option", () => { + const acpPath = "harbor.agents.installed.acp:AcpAgent"; + const acpVersion = "1.2.3"; + const sourceConfig = { + repo_url: "https://github.com/example/agent.git", + ref: "a".repeat(40), + source_dir: "agent", + manifest_path: "harbor-agent.json", + }; + const preset = { + ...catalog.agent("pi", "0.84.4"), + harbor_agent: { + import_path: acpPath, + kwargs: { version: acpVersion, source: sourceConfig }, + }, + reasoning_option: null, + }; + const grant = grantOf({ agent_import_path: acpPath, agent_version: acpVersion }); + const policy = policyOf([grant]); + const subject = { import_path: acpPath, version: acpVersion }; + expect( + policy.presetConnection(ref, subject, modelId, actor, image, modelApi), + ).toMatchObject(connection()); + expect( + policy.presetConnection(ref, subject, modelId, actor, image, "openai-responses"), + ).toBeNull(); + const agent = presetEndpointAgent( + undefined, + preset, + modelId, + connection(), + modelApi, + "default", + ); + expect(agent.kwargs).toEqual({ version: acpVersion, source: sourceConfig }); + expect(admit(agent, [grant])).toEqual({ ref, source }); + expect(() => admit(agent, [grantOf({ ...grant, agent_version: "other" })])).toThrow( + "not reviewed", + ); + expect(() => + admit( + { ...agent, kwargs: { ...(agent.kwargs as object), model_api: modelApi } }, + [grant], + ), + ).toThrow("not reviewed"); + expect(() => + admit(agent, [grant, grantOf({ ...grant, model_api: "openai-responses" })]), + ).toThrow("not reviewed"); + expect(() => + admit( + { + ...agent, + env: { + OPENAI_BASE_URL: "https://other.invalid/v1", + OPENAI_API_KEY: `\${${ref}}`, + }, + }, + [grant], + ), + ).toThrow("not reviewed"); + }); + it("keeps the router record when the submission names no connection", () => { const router = catalog.buildJobConfig("run-1", routerSubmission, "/data"); expect(router.agents?.[0]?.model_name).toBe("huggingface/example/model:endpoint"); From 86d893dbbb48a12f9cef6eaace1982bf58d2a79f Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:27:19 +0800 Subject: [PATCH 2/4] feat(inference): deliver scoped verifier judge keys to reviewed runs Co-Authored-By: GPT-6 Sol Generated-By: pi 0.87.1 (https://pi.dev) (cherry picked from commit 222867e7080d743901325ac024ac6b342287b6f8) --- apps/control-api/src/config.ts | 42 ++++++++++- apps/control-api/src/runtime.ts | 1 + apps/control-api/test/config.test.ts | 41 ++++++++++ docs/CONTROL_SERVICE.md | 22 +++++- .../control-core/src/inference-bindings.ts | 11 +++ packages/hf-adapters/src/inference-secrets.ts | 74 ++++++++++++++++--- packages/hf-adapters/src/jobs.ts | 18 ++++- .../test/inference-secrets.test.ts | 59 +++++++++++++++ packages/hf-adapters/test/jobs.test.ts | 74 +++++++++++++++++++ 9 files changed, 327 insertions(+), 15 deletions(-) diff --git a/apps/control-api/src/config.ts b/apps/control-api/src/config.ts index 9165cc8e..d452df1a 100644 --- a/apps/control-api/src/config.ts +++ b/apps/control-api/src/config.ts @@ -1,8 +1,35 @@ import { resolve } from "node:path"; import { parsePresetSources, type PresetSourceV1 } from "@harbor-hf/control-core"; -import type { ParentHardware } from "@harbor-hf/hf-adapters"; +import type { ParentHardware, ReviewedVerifierGrant } from "@harbor-hf/hf-adapters"; import { z } from "zod"; +const verifierGrantSchema = z.strictObject({ + ref: z.string().regex(/^INFERENCE_API_KEY_[A-Z0-9_]{1,48}$/), + worker_image: z.string().regex(/@sha256:[a-f0-9]{64}$/), + benchmark: z.strictObject({ + name: z.string().min(1).max(120), + preset: z.string().min(1).max(120), + }), + dataset_repo: z + .string() + .regex(/^https:\/\/huggingface\.co\/datasets\/[A-Za-z0-9_./-]+\.git@[a-f0-9]{40}$/), + dataset_path: z.string().min(1).max(240), + model: z.string().regex(/^[A-Za-z0-9_./-]{1,160}$/), + base_url: z + .string() + .url() + .refine((value) => { + const url = new URL(value); + return ( + url.protocol === "https:" && + !url.username && + !url.password && + !url.search && + !url.hash + ); + }), +}); + const schema = z.object({ NODE_ENV: z.enum(["development", "test", "production"]).default("production"), PORT: z.coerce.number().int().min(1).max(65535).default(7860), @@ -14,6 +41,7 @@ const schema = z.object({ HARBOR_HF_AUTH_PATH: z.string().min(1).default("/tmp/harbor-hf/auth.sqlite"), HARBOR_HF_PRESETS_ROOT: z.string().min(1).default("./presets"), HARBOR_HF_PRESET_SOURCES: z.string().default("[]"), + HARBOR_HF_VERIFIER_GRANTS: z.string().default("[]"), HARBOR_HF_LAUNCH_PYTHON: z.string().min(1).optional(), HARBOR_HF_APPROVED_AGENT_SOURCES: z.string().default("[]"), HARBOR_HF_MAX_ACTIVE_JOBS: z.coerce.number().int().min(1).max(1024).default(16), @@ -80,6 +108,7 @@ export interface AppConfig { * presets for very popular benchmarks; anything else comes from a source. */ preset_sources: PresetSourceV1[]; + verifier_grants?: ReviewedVerifierGrant[]; launch_python?: string; approved_agent_sources?: Record[]; max_active_jobs: number; @@ -183,6 +212,16 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): AppCon operator_org_subject: parsed.HARBOR_HF_OPERATOR_ORG_SUBJECT ?? null, }; } + const verifierGrants = z + .array(verifierGrantSchema) + .max(8) + .parse(JSON.parse(parsed.HARBOR_HF_VERIFIER_GRANTS)); + const subjects = new Set(); + for (const grant of verifierGrants) { + const subject = `${grant.benchmark.name}\u0000${grant.benchmark.preset}`; + if (subjects.has(subject)) throw new Error("duplicate verifier credential grant"); + subjects.add(subject); + } return { node_env: parsed.NODE_ENV, port: parsed.PORT, @@ -194,6 +233,7 @@ export function loadConfig(environment: NodeJS.ProcessEnv = process.env): AppCon auth_path: resolve(parsed.HARBOR_HF_AUTH_PATH), presets_root: resolve(parsed.HARBOR_HF_PRESETS_ROOT), preset_sources: parsePresetSources(JSON.parse(parsed.HARBOR_HF_PRESET_SOURCES)), + verifier_grants: verifierGrants, ...(parsed.HARBOR_HF_LAUNCH_PYTHON ? { launch_python: parsed.HARBOR_HF_LAUNCH_PYTHON } : {}), diff --git a/apps/control-api/src/runtime.ts b/apps/control-api/src/runtime.ts index 88a6a62b..d3974b43 100644 --- a/apps/control-api/src/runtime.ts +++ b/apps/control-api/src/runtime.ts @@ -141,6 +141,7 @@ async function composeRuntime( parentImage: config.parent_image ?? "", hardware: config.parent_hardware, timeoutSeconds: config.parent_timeout_seconds, + verifierGrants: config.verifier_grants ?? [], }) : new ReadOnlyHuggingFaceJobs({ namespace: config.namespace, diff --git a/apps/control-api/test/config.test.ts b/apps/control-api/test/config.test.ts index 2609dbf6..661b70a9 100644 --- a/apps/control-api/test/config.test.ts +++ b/apps/control-api/test/config.test.ts @@ -82,6 +82,47 @@ describe("control API configuration", () => { ).toThrow(); }); + it("loads only pinned, HTTPS verifier credential grants", () => { + const grant = { + ref: "INFERENCE_API_KEY_JUDGE", + worker_image: `example/parent@sha256:${"a".repeat(64)}`, + benchmark: { name: "example-benchmark", preset: "one-task" }, + dataset_repo: `https://huggingface.co/datasets/example-org/tasks.git@${"b".repeat(40)}`, + dataset_path: "tasks", + model: "judge-model", + base_url: "https://judge.invalid/v1", + }; + expect(loadConfig(environment).verifier_grants).toEqual([]); + expect( + loadConfig({ + ...environment, + HARBOR_HF_VERIFIER_GRANTS: JSON.stringify([grant]), + }).verifier_grants, + ).toEqual([grant]); + expect(() => + loadConfig({ + ...environment, + HARBOR_HF_VERIFIER_GRANTS: JSON.stringify([grant, grant]), + }), + ).toThrow("duplicate verifier credential grant"); + for (const bad of [ + { + ...grant, + dataset_repo: "https://huggingface.co/datasets/example-org/tasks.git@main", + }, + { ...grant, base_url: "http://judge.invalid/v1" }, + { ...grant, base_url: "https://user:pass@judge.invalid/v1" }, + { ...grant, ref: "HF_TOKEN" }, + { ...grant, extra: "not reviewed" }, + ]) + expect(() => + loadConfig({ + ...environment, + HARBOR_HF_VERIFIER_GRANTS: JSON.stringify([bad]), + }), + ).toThrow(); + }); + it("rejects development authentication in production", () => { expect(() => loadConfig({ diff --git a/docs/CONTROL_SERVICE.md b/docs/CONTROL_SERVICE.md index 201c499a..ce865a93 100644 --- a/docs/CONTROL_SERVICE.md +++ b/docs/CONTROL_SERVICE.md @@ -127,7 +127,7 @@ Do not create a resource per run. SQLite files and HF Jobs are temporary. The three-table projection rebuilds from run records, Harbor results, attempt cost receipts, and current Job observations. -The Space has two secrets: +The Space requires two core secrets. Reviewed agent and verifier connections can use additional, separately registered secrets: - `HF_TOKEN` is a purpose-scoped control credential with access to the Bucket, HF Jobs, and approved private Hugging Face Dataset Git repositories when they @@ -155,6 +155,7 @@ The service reads these Space variables: | `HARBOR_HF_BUCKET_ROOT` | no | `/data` | local filesystem store root | | `HARBOR_HF_PRESETS_ROOT` | no | `./presets` | reviewed presets for very popular benchmarks | | `HARBOR_HF_PRESET_SOURCES` | no | `[]` | pinned external preset sources as JSON; see [preset sources](preset-sources.md) | +| `HARBOR_HF_VERIFIER_GRANTS` | no | `[]` | reviewed, benchmark-scoped verifier credential grants as JSON; no secret values | | `HARBOR_HF_LAUNCH_PYTHON` | no | worker package `.venv/bin/python`; `/opt/harbor-launch/bin/python` in the control image | pinned native launch inspector | | `HARBOR_HF_APPROVED_AGENT_SOURCES` | no | `[]` | reviewed native ACP source objects as JSON; never credentials | | `HARBOR_HF_WRITE_MODE` | no | `disabled` | permit Job lifecycle changes | @@ -207,6 +208,25 @@ credential files, run configuration, Bucket objects, projections, browser responses, or logs. It must not enter trial agent environments or agent source installation. The separate inference credential path remains unchanged. +### Verifier judge credential + +Some task verifiers need a different model and key from the agent. Set +`HARBOR_HF_VERIFIER_GRANTS` only after reviewing the benchmark source and the +registered credential reference. Each JSON object names `ref`, `worker_image`, +`benchmark` (`name` and `preset`), `dataset_repo`, `dataset_path`, `model`, and +`base_url`. Pin the Dataset Git URL to a full commit and the worker image to a +SHA256 digest. Keep the source credential in the existing Space secret store; +never put its value in this variable. + +At parent Job start, the service checks the exact benchmark, dataset, image, +operator, and active credential reference again. It sends the selected key as +an ephemeral `AGENT_JUDGE_API_KEY` Job secret. It sends the reviewed URL and +model as `AGENT_JUDGE_API_URL` and `AGENT_JUDGE_MODEL`. Harbor then resolves +the task's `${AGENT_JUDGE_API_KEY:-}` verifier environment entry from the +parent Job environment. The run record and trial configuration contain no +judge key. A different dataset or a verifier environment override blocks the +credential. This grant does not replace the separate reviewed agent connection. + Admission and repository inspection occur before model inference. A missing token, denied repository, missing commit, malformed source, or native Harbor resolution failure stops the launch. The service does not fall back to another diff --git a/packages/control-core/src/inference-bindings.ts b/packages/control-core/src/inference-bindings.ts index 57f7d899..c5234049 100644 --- a/packages/control-core/src/inference-bindings.ts +++ b/packages/control-core/src/inference-bindings.ts @@ -203,6 +203,17 @@ export class InferenceBindings { return this.#manifest.bindings.map(({ ref, source_env }) => ({ ref, source_env })); } + /** Select an active source owned by this operator for a separately scoped verifier grant. */ + reviewedSource(ref: string, actor: string): string | null { + const binding = this.#manifest.bindings.find( + (entry) => + entry.ref === ref && + entry.enabled && + entry.uses.some((use) => use.operator_subjects.includes(actor)), + ); + return binding?.source_env ?? null; + } + assertSourceTransitionFrom(previous: InferenceBindings): void { for (const binding of this.sourceIdentities()) { const prior = previous diff --git a/packages/hf-adapters/src/inference-secrets.ts b/packages/hf-adapters/src/inference-secrets.ts index fef1e053..f858d023 100644 --- a/packages/hf-adapters/src/inference-secrets.ts +++ b/packages/hf-adapters/src/inference-secrets.ts @@ -1,6 +1,20 @@ -import { InferenceBindingDenied, type RunRecordV1 } from "@harbor-hf/contracts"; +import { + InferenceBindingDenied, + validateHarborJobConfig, + type RunRecordV1, +} from "@harbor-hf/contracts"; import type { InferenceBindings } from "@harbor-hf/control-core"; +export interface ReviewedVerifierGrant { + ref: string; + worker_image: string; + benchmark: { name: string; preset: string }; + dataset_repo: string; + dataset_path: string; + model: string; + base_url: string; +} + /** Ephemeral delivery boundary. No environment enumeration, serialization or HF token path. * The Jobs adapter supplies the transport; runtime derives reviewed policy from the private registry. */ @@ -9,15 +23,39 @@ export async function withSelectedInferenceSecret( image: string, policy: () => InferenceBindings | Promise, readSelected: (source: string) => string | undefined, - deliver: (secrets: Readonly>) => Promise, + deliver: ( + secrets: Readonly>, + environment: Readonly>, + ) => Promise, + verifierGrants: readonly ReviewedVerifierGrant[] = [], ): Promise { let selected: ReturnType; + let verifierSource: string | null = null; + let verifierGrant: ReviewedVerifierGrant | undefined; try { - selected = (await policy()).selected( - run.harbor_job_config, - run.submitted_by, - image, + const reviewed = await policy(); + selected = reviewed.selected(run.harbor_job_config, run.submitted_by, image); + verifierGrant = verifierGrants.find( + (grant) => + grant.benchmark.name === run.submission?.benchmark?.name && + grant.benchmark.preset === run.submission?.benchmark?.preset, ); + if (verifierGrant) { + const config = validateHarborJobConfig(run.harbor_job_config); + const datasets = config.datasets ?? []; + if ( + verifierGrant.worker_image !== image || + datasets.length !== 1 || + datasets[0]?.repo !== verifierGrant.dataset_repo || + datasets[0]?.path !== verifierGrant.dataset_path || + Object.keys(config.verifier?.env ?? {}).length !== 0 || + !selected || + selected.ref === verifierGrant.ref + ) + throw new InferenceBindingDenied(); + verifierSource = reviewed.reviewedSource(verifierGrant.ref, run.submitted_by); + if (!verifierSource) throw new InferenceBindingDenied(); + } } catch { // This check precedes delivery, so callers can safely block just this start. throw new InferenceBindingDenied(); @@ -34,11 +72,27 @@ export async function withSelectedInferenceSecret( throw new InferenceBindingDenied("Selected inference credential is missing"); secrets[selected.ref] = value; } + const verifierEnvironment: Record = {}; try { - return await deliver(secrets); - } catch { - // Never return transport errors that may contain ephemeral secret arguments. - throw new Error("Reviewed inference delivery failed"); + if (verifierGrant && verifierSource) { + let value: string | undefined; + try { + value = readSelected(verifierSource); + } catch { + throw new InferenceBindingDenied(); + } + if (typeof value !== "string" || !value || value === secrets[selected?.ref ?? ""]) + throw new InferenceBindingDenied(); + secrets.AGENT_JUDGE_API_KEY = value; + verifierEnvironment.AGENT_JUDGE_API_URL = verifierGrant.base_url; + verifierEnvironment.AGENT_JUDGE_MODEL = verifierGrant.model; + } + try { + return await deliver(secrets, verifierEnvironment); + } catch { + // Never return transport errors that may contain ephemeral secret arguments. + throw new Error("Reviewed inference delivery failed"); + } } finally { for (const key of Object.keys(secrets)) delete secrets[key]; } diff --git a/packages/hf-adapters/src/jobs.ts b/packages/hf-adapters/src/jobs.ts index 9f385b26..e68529df 100644 --- a/packages/hf-adapters/src/jobs.ts +++ b/packages/hf-adapters/src/jobs.ts @@ -8,7 +8,10 @@ import { cancelJob, runJob, type SpaceHardwareFlavor } from "@huggingface/hub"; import { isolatedGitSourceEnvironment } from "./git-environment.js"; import { inspectJob, listJobPages, observation } from "./jobs-read.js"; -import { withSelectedInferenceSecret } from "./inference-secrets.js"; +import { + withSelectedInferenceSecret, + type ReviewedVerifierGrant, +} from "./inference-secrets.js"; export type ParentHardware = SpaceHardwareFlavor; @@ -30,6 +33,7 @@ export interface HuggingFaceJobsOptions extends ReadOnlyHuggingFaceJobsOptions { hardware?: SpaceHardwareFlavor; localRoot?: string; timeoutSeconds?: number; + verifierGrants?: readonly ReviewedVerifierGrant[]; } export class ReadOnlyHuggingFaceJobs implements JobsPort { @@ -60,7 +64,12 @@ export class HuggingFaceJobs implements JobsPort { constructor(private readonly options: HuggingFaceJobsOptions) { if (!IMMUTABLE_IMAGE.test(options.parentImage)) throw new Error("parent image must use an immutable sha256 digest"); - this.options = Object.freeze({ ...options }); + this.options = Object.freeze({ + ...options, + ...(options.verifierGrants + ? { verifierGrants: structuredClone(options.verifierGrants) } + : {}), + }); this.hardware = options.hardware ?? "cpu-basic"; this.localRoot = options.localRoot ?? "/data"; this.timeoutSeconds = options.timeoutSeconds ?? 86_400; @@ -104,7 +113,8 @@ export class HuggingFaceJobs implements JobsPort { this.options.parentImage, policy, readSelected, - (secrets) => this.#startParent(run.run_id, secrets), + (secrets, environment) => this.#startParent(run.run_id, secrets, environment), + this.options.verifierGrants, ); } @@ -112,6 +122,7 @@ export class HuggingFaceJobs implements JobsPort { async #startParent( runId: string, inferenceSecrets: Readonly>, + verifierEnvironment: Readonly> = {}, ): Promise { assertRunId(runId); const value = await runJob({ @@ -124,6 +135,7 @@ export class HuggingFaceJobs implements JobsPort { HARBOR_HF_BUCKET_ID: this.options.bucketId, HARBOR_HF_NAMESPACE: this.options.namespace, ...isolatedGitSourceEnvironment(), + ...verifierEnvironment, }, secrets: { HF_TOKEN: this.options.accessToken, diff --git a/packages/hf-adapters/test/inference-secrets.test.ts b/packages/hf-adapters/test/inference-secrets.test.ts index aa130313..172eff1e 100644 --- a/packages/hf-adapters/test/inference-secrets.test.ts +++ b/packages/hf-adapters/test/inference-secrets.test.ts @@ -109,6 +109,65 @@ describe("ephemeral selected inference delivery", () => { }); }); +it("delivers a separate judge credential only for the exact reviewed benchmark and source", async () => { + const { record, manifest } = run("EXAMPLE"); + const source = `https://huggingface.co/datasets/example-org/tasks.git@${"b".repeat(40)}`; + record.harbor_job_config.datasets = [{ repo: source, path: "tasks" }]; + const judge = { + ref: "INFERENCE_API_KEY_JUDGE", + worker_image: image, + benchmark: { name: "synthetic", preset: "synthetic" }, + dataset_repo: source, + dataset_path: "tasks", + base_url: "https://judge.invalid/v1", + model: "model-judge", + }; + manifest.bindings.push({ + ...manifest.bindings[0]!, + ref: judge.ref, + source_env: "INFERENCE_SECRET_JUDGE", + label: "Synthetic judge", + }); + const original = JSON.stringify(record); + const policy = () => new InferenceBindings(manifest); + const read = vi.fn((name: string) => `synthetic-value-${name}`); + const deliver = vi.fn( + async ( + secrets: Readonly>, + environment: Readonly>, + ) => { + expect(secrets).toEqual({ + INFERENCE_API_KEY_EXAMPLE: "synthetic-value-INFERENCE_SECRET_EXAMPLE", + AGENT_JUDGE_API_KEY: "synthetic-value-INFERENCE_SECRET_JUDGE", + }); + expect(environment).toEqual({ + AGENT_JUDGE_API_URL: "https://judge.invalid/v1", + AGENT_JUDGE_MODEL: "model-judge", + }); + return "delivered"; + }, + ); + await expect( + withSelectedInferenceSecret(record, image, policy, read, deliver, [judge]), + ).resolves.toBe("delivered"); + expect(JSON.stringify(record)).toBe(original); + expect(read.mock.calls.map(([name]) => name)).toEqual([ + "INFERENCE_SECRET_EXAMPLE", + "INFERENCE_SECRET_JUDGE", + ]); + expect(deliver.mock.calls[0]?.[0]).toEqual({}); + record.harbor_job_config.datasets = [{ repo: source, path: "other" }]; + await expect( + withSelectedInferenceSecret(record, image, policy, read, deliver, [judge]), + ).rejects.toThrow("not reviewed"); + record.harbor_job_config.datasets = [{ repo: source, path: "tasks" }]; + manifest.bindings[1]!.enabled = false; + await expect( + withSelectedInferenceSecret(record, image, policy, read, deliver, [judge]), + ).rejects.toThrow("not reviewed"); + expect(deliver).toHaveBeenCalledTimes(1); +}); + it("passes no secrets or source reads for a native no-inference configuration", async () => { const { record } = run("EXAMPLE"); record.harbor_job_config.agents = []; diff --git a/packages/hf-adapters/test/jobs.test.ts b/packages/hf-adapters/test/jobs.test.ts index 87b11fa5..c95c21ab 100644 --- a/packages/hf-adapters/test/jobs.test.ts +++ b/packages/hf-adapters/test/jobs.test.ts @@ -224,6 +224,80 @@ describe("reviewed provider parent transport", () => { /^Reviewed inference delivery failed$/, ); }); + + it("delivers a separately scoped verifier key and route to the same ephemeral parent", async () => { + const { + fixture, + actor, + image: worker, + } = await import("../../control-core/test/inference-fixture.js"); + const { InferenceBindings } = await import("@harbor-hf/control-core"); + const { validateRunRecord } = await import("@harbor-hf/contracts"); + const data = fixture(); + data.manifest.bindings.push({ + ...data.manifest.bindings[0]!, + ref: "INFERENCE_API_KEY_JUDGE", + source_env: "INFERENCE_SECRET_JUDGE", + }); + const source = `https://huggingface.co/datasets/example-org/tasks.git@${"b".repeat(40)}`; + const record = validateRunRecord({ + schema_version: "v1", + run_id: runId, + created_at: "2026-01-01T00:00:00Z", + submitted_by: actor, + role: "diagnostic", + harbor_revision: "a".repeat(40), + submission: { + benchmark: { name: "synthetic", preset: "one" }, + cost_ceiling_usd: 1, + }, + harbor_job_config: { ...data.job(), datasets: [{ repo: source, path: "tasks" }] }, + }); + const requests: Record[] = []; + const jobs = new HuggingFaceJobs({ + namespace: "example", + accessToken: controlToken, + inferenceToken, + bucketId: "example/bucket", + parentImage: worker, + verifierGrants: [ + { + ref: "INFERENCE_API_KEY_JUDGE", + worker_image: worker, + benchmark: { name: "synthetic", preset: "one" }, + dataset_repo: source, + dataset_path: "tasks", + base_url: "https://judge.invalid/v1", + model: "judge-model", + }, + ], + fetch: (async (_input, init) => { + requests.push(JSON.parse(String(init?.body))); + return new Response(JSON.stringify(apiJob()), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch, + }); + const read = (name: string) => `synthetic-${name}`; + await jobs.startReviewedParent( + record, + () => new InferenceBindings(data.manifest), + read, + ); + expect(requests[0]?.secrets).toMatchObject({ + INFERENCE_API_KEY_EXAMPLE: "synthetic-INFERENCE_SECRET_EXAMPLE", + AGENT_JUDGE_API_KEY: "synthetic-INFERENCE_SECRET_JUDGE", + }); + expect(requests[0]?.environment).toMatchObject({ + AGENT_JUDGE_API_URL: "https://judge.invalid/v1", + AGENT_JUDGE_MODEL: "judge-model", + }); + expect(JSON.stringify(requests[0]?.environment)).not.toContain( + "synthetic-INFERENCE_SECRET_JUDGE", + ); + expect(JSON.stringify(record)).not.toContain("synthetic-INFERENCE_SECRET_JUDGE"); + }); }); describe("paginated Job reads", () => { From c81e04d5b1c931836c46e920442b872c8d98120f Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Sat, 26 Sep 2026 05:42:24 +0800 Subject: [PATCH 3/4] fix(jobs): filter Harbor jobs before pagination Co-Authored-By: GPT-6 Sol Generated-By: pi 0.87.1 (https://pi.dev) (cherry picked from commit 5a0b33ce1cd22c63280be8b3f74f5463b3d5fdaa) --- packages/control-core/test/control.test.ts | 7 ++- packages/hf-adapters/src/jobs-read.ts | 54 ++++++++++--------- packages/hf-adapters/test/jobs.test.ts | 62 +++++++++++++++------- 3 files changed, 79 insertions(+), 44 deletions(-) diff --git a/packages/control-core/test/control.test.ts b/packages/control-core/test/control.test.ts index 604108a6..bb32dff7 100644 --- a/packages/control-core/test/control.test.ts +++ b/packages/control-core/test/control.test.ts @@ -1536,10 +1536,13 @@ describe("reconciliation", () => { namespace: "example", accessToken: "synthetic", fetch: async (input) => { - if (String(input).includes("cursor=second")) + const url = new URL(String(input)); + if (url.searchParams.has("cursor")) return fails ? new Response("unavailable", { status: 503 }) - : new Response(JSON.stringify([raw("parent", "existing-parent")])); + : new Response("[]"); + if (url.searchParams.get("label") === "harbor-hf-role=parent") + return new Response(JSON.stringify([raw("parent", "existing-parent")])); return new Response( JSON.stringify( Array.from({ length: 100 }, (_, i) => raw("trial", `child-${i}`)), diff --git a/packages/hf-adapters/src/jobs-read.ts b/packages/hf-adapters/src/jobs-read.ts index 140f3496..54bab5fa 100644 --- a/packages/hf-adapters/src/jobs-read.ts +++ b/packages/hf-adapters/src/jobs-read.ts @@ -109,33 +109,39 @@ export async function listJobPages( options: ReadOnlyHuggingFaceJobsOptions, ): Promise { const first = endpoint(options); - let url: URL | null = first; - const seen = new Set(); const values = new Map(); - while (url) { - if (seen.has(url.href) || seen.size >= 1000) - throw new Error("Jobs pagination loop or page limit"); - seen.add(url.href); - const response = await read(options, url); - const page: unknown = await response.json(); - if (!Array.isArray(page)) throw new Error("Invalid Jobs page"); - for (const input of page) { - const job = observation(input); - if (job) { - const previous = values.get(job.id); - if ( - previous && - (previous.run_id !== job.run_id || - previous.role !== job.role || - previous.created_at !== job.created_at) - ) - throw new Error("Conflicting Job in pagination"); - // New Jobs can shift offset-based pages while this list is in flight. - // Keep the later observation when the same stable identity overlaps. - values.set(job.id, job); + // The account has many unrelated Jobs. Filter at the Hub before pagination; + // still collect both roles so late children and parent liveness remain visible. + for (const role of ["parent", "trial"] as const) { + const filtered = new URL(first); + filtered.searchParams.set("label", `harbor-hf-role=${role}`); + let url: URL | null = filtered; + const seen = new Set(); + while (url) { + if (seen.has(url.href) || seen.size >= 1000) + throw new Error("Jobs pagination loop or page limit"); + seen.add(url.href); + const response = await read(options, url); + const page: unknown = await response.json(); + if (!Array.isArray(page)) throw new Error("Invalid Jobs page"); + for (const input of page) { + const job = observation(input); + if (job) { + if (job.role !== role) throw new Error("Jobs label filter was not applied"); + const previous = values.get(job.id); + if ( + previous && + (previous.run_id !== job.run_id || + previous.role !== job.role || + previous.created_at !== job.created_at) + ) + throw new Error("Conflicting Job in pagination"); + // New Jobs can shift cursor pages. Keep the later stable observation. + values.set(job.id, job); + } } + url = nextPage(response.headers.get("link"), url, filtered); } - url = nextPage(response.headers.get("link"), url, first); } return [...values.values()]; } diff --git a/packages/hf-adapters/test/jobs.test.ts b/packages/hf-adapters/test/jobs.test.ts index c95c21ab..1df2b57b 100644 --- a/packages/hf-adapters/test/jobs.test.ts +++ b/packages/hf-adapters/test/jobs.test.ts @@ -79,13 +79,17 @@ describe("HuggingFaceJobs", () => { status: { stage: "STOPPED", failureCount: 0 }, }), ); + const role = new URL(String(_input)).searchParams.get("label"); return new Response( - JSON.stringify([ - apiJob(), - apiJob("trial"), - { ...apiJob(), id: "unlabelled", labels: null }, - { ...apiJob(), id: "unrelated", labels: { other: "value" } }, - ]), + JSON.stringify( + role === "harbor-hf-role=parent" + ? [ + apiJob(), + { ...apiJob(), id: "unlabelled", labels: null }, + { ...apiJob(), id: "unrelated", labels: { other: "value" } }, + ] + : [apiJob("trial")], + ), ); }; const jobs = new HuggingFaceJobs({ @@ -98,19 +102,20 @@ describe("HuggingFaceJobs", () => { }); expect(await jobs.list()).toHaveLength(2); await jobs.cancel("parent-job"); - expect(methods).toEqual(["GET", "POST"]); + expect(methods).toEqual(["GET", "GET", "POST"]); }); it("observes owned Jobs without allowing lifecycle changes", async () => { const jobs = new ReadOnlyHuggingFaceJobs({ namespace: "example", accessToken: controlToken, - fetch: async () => + fetch: async (input) => new Response( - JSON.stringify([ - apiJob(), - { ...apiJob(), id: "unrelated", labels: { other: "value" } }, - ]), + JSON.stringify( + new URL(String(input)).searchParams.get("label") === "harbor-hf-role=parent" + ? [apiJob(), { ...apiJob(), id: "unrelated", labels: { other: "value" } }] + : [], + ), ), }); expect(await jobs.list()).toHaveLength(1); @@ -312,6 +317,7 @@ describe("paginated Job reads", () => { it("reads 100 children then the live parent, following relative and multi-rel Links", async () => { const transport = vi .fn() + .mockResolvedValueOnce(new Response(JSON.stringify([apiJob()]))) .mockResolvedValueOnce( new Response( JSON.stringify( @@ -327,19 +333,36 @@ describe("paginated Job reads", () => { }, ), ) - .mockResolvedValueOnce(new Response(JSON.stringify([apiJob()]))); + .mockResolvedValueOnce(new Response("[]")); const values = await reader(transport).list(); expect(values).toHaveLength(101); - expect(values.at(-1)?.role).toBe("parent"); + expect(values[0]?.role).toBe("parent"); + expect(values.at(-1)?.role).toBe("trial"); + expect(transport.mock.calls[0]?.[0]).toBe( + "https://huggingface.co/api/jobs/example?label=harbor-hf-role%3Dparent", + ); expect(transport.mock.calls[1]?.[0]).toBe( + "https://huggingface.co/api/jobs/example?label=harbor-hf-role%3Dtrial", + ); + expect(transport.mock.calls[2]?.[0]).toBe( "https://huggingface.co/api/jobs/example?cursor=second", ); - expect(transport.mock.calls[1]?.[1]).toMatchObject({ + expect(transport.mock.calls[2]?.[1]).toMatchObject({ redirect: "error", headers: { Authorization: `Bearer ${controlToken}` }, }); }); + it("rejects a Hub response that ignores the requested Job role", async () => { + const transport = vi + .fn() + .mockResolvedValue(new Response(JSON.stringify([apiJob("trial")]))); + await expect(reader(transport).list()).rejects.toThrow( + "Jobs label filter was not applied", + ); + expect(transport).toHaveBeenCalledTimes(1); + }); + it.each([ "https://untrusted.invalid/api/jobs/example", "//untrusted.invalid/api/jobs/example", @@ -374,7 +397,7 @@ describe("paginated Job reads", () => { const transport = vi .fn() .mockResolvedValueOnce( - new Response(JSON.stringify([apiJob("trial")]), { + new Response(JSON.stringify([apiJob()]), { headers: { link: "; rel=next" }, }), ) @@ -394,7 +417,9 @@ describe("paginated Job reads", () => { .fn() .mockResolvedValue(new Response("[]", { headers: { link } })); await expect(reader(transport).list()).rejects.toThrow(); - expect(transport).toHaveBeenCalledTimes(1); + expect(transport).toHaveBeenCalledTimes( + link.startsWith("") ? 2 : 1, + ); }); it("coalesces a stable Job repeated by moving pagination", async () => { @@ -415,7 +440,8 @@ describe("paginated Job reads", () => { }, ]), ), - ); + ) + .mockResolvedValueOnce(new Response("[]")); await expect(reader(transport).list()).resolves.toEqual([ expect.objectContaining({ From a7167215c1e4fe7cad5fee73e932f8c64aa955f5 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Sat, 26 Sep 2026 06:17:58 +0800 Subject: [PATCH 4/4] fix(runs): scope submission projection refresh Co-Authored-By: GPT-6 Sol Generated-By: pi 0.87.1 (https://pi.dev) (cherry picked from commit c450f033d441a85750ff790ffb9aa0d577d02ab0) --- packages/control-core/src/service.ts | 2 +- packages/control-core/test/control.test.ts | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/control-core/src/service.ts b/packages/control-core/src/service.ts index f3d09222..13dcc9b5 100644 --- a/packages/control-core/src/service.ts +++ b/packages/control-core/src/service.ts @@ -605,7 +605,7 @@ export class ControlService { return { created: true, run: record }; }), ); - await this.refresh(); + await this.projection.rebuild(this.store, await this.jobs.list(), record.run_id); return result; } diff --git a/packages/control-core/test/control.test.ts b/packages/control-core/test/control.test.ts index bb32dff7..ee3c1b6d 100644 --- a/packages/control-core/test/control.test.ts +++ b/packages/control-core/test/control.test.ts @@ -154,6 +154,20 @@ describe("run submission", () => { expect(projection.run(result.run.run_id)?.status).toBe("queued"); }); + it("updates only the submitted run projection", async () => { + const first = await submit("first-run"); + const rebuild = vi.spyOn(projection, "rebuild"); + const second = await submit("second-run"); + expect(rebuild).toHaveBeenCalledExactlyOnceWith( + store, + expect.any(Array), + second.run.run_id, + ); + expect(projection.run(first.run.run_id)?.status).toBe("queued"); + expect(projection.run(second.run.run_id)?.status).toBe("queued"); + rebuild.mockRestore(); + }); + it("loads the nine-trial diagnostic preset unchanged through both launch paths", async () => { const benchmark = { name: "terminal-bench-2-1", preset: "three-tasks-3-trials" }; const canary = presets.benchmark(benchmark.name, benchmark.preset);