Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/control-web/e2e/control.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ const presets = {
n_concurrent_trials: 1,
environment: {
type: "hf-sandbox",
kwargs: { flavor: "cpu-upgrade", job_timeout: "30m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
},
},
},
Expand Down
4 changes: 3 additions & 1 deletion apps/control-web/src/launch-draft.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ export function initialDraft(): NativeObject {
agents: [],
environment: {
type: "hf-sandbox",
kwargs: { flavor: "cpu-basic", job_timeout: "30m" },
// Temporary stopgap for https://github.com/huggingface/sandbox-server/pull/21.
// Remove after the fixed server passes a foreground canary longer than 30 minutes.
kwargs: { flavor: "cpu-basic", job_timeout: "none" },
},
};
}
Expand Down
24 changes: 20 additions & 4 deletions apps/control-web/src/launch-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -750,11 +750,14 @@ function LaunchForm({ search }: { search: string }) {
})
}
/>
<label className="text-sm">
Sandbox idle timeout
<div>
<label className="text-sm" htmlFor="sandbox-idle-timeout">
Sandbox idle timeout
</label>
<input
className={inputClass}
value={String(envKwargs.job_timeout ?? "30m")}
id="sandbox-idle-timeout"
value={String(envKwargs.job_timeout ?? "none")}
onChange={(event) =>
change({
...draft,
Expand All @@ -766,7 +769,20 @@ function LaunchForm({ search }: { search: string }) {
})
}
/>
</label>
<span className="mt-1 block text-xs text-slate-400">
Keep this set to <code>none</code> until the fixed Sandbox server is
deployed. See{" "}
<a
className="underline"
href="https://github.com/huggingface/sandbox-server/pull/21"
rel="noreferrer"
target="_blank"
>
sandbox-server PR 21
</a>
.
</span>
</div>
<label className="text-sm">
Post-trial cost limit (USD)
<input
Expand Down
2 changes: 1 addition & 1 deletion apps/control-web/test/App.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ const presets: PresetsResponse = {
n_concurrent_trials: 1,
environment: {
type: "hf-sandbox",
kwargs: { flavor: "cpu-upgrade", job_timeout: "30m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
},
},
},
Expand Down
6 changes: 3 additions & 3 deletions apps/control-web/test/launch.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ const draft = {
],
environment: {
type: "hf-sandbox",
kwargs: { flavor: "cpu-upgrade", job_timeout: "30m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
},
};
const catalog = {
Expand Down Expand Up @@ -331,7 +331,7 @@ describe("launch page", () => {
n_concurrent_trials: 1,
environment: {
type: "hf-sandbox",
kwargs: { flavor: "cpu-basic", job_timeout: "15m" },
kwargs: { flavor: "cpu-basic", job_timeout: "none" },
},
},
},
Expand All @@ -358,7 +358,7 @@ describe("launch page", () => {
target: { value: "0.5" },
});
expect(loadDraft().environment).toMatchObject({
kwargs: { flavor: "cpu-upgrade", job_timeout: "15m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
});
fireEvent.change(screen.getByLabelText("Native environment"), {
target: { value: '{"type":"hf-sandbox","kwargs":{"flavor":"cpu-basic"}}' },
Expand Down
15 changes: 11 additions & 4 deletions docs/2026-09-04-simplification-implementation-spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ A preset submission creates this immutable record:
"import_path": "harbor_hf_agents.hf_sandbox:LabeledHFSandboxEnvironment",
"kwargs": {
"flavor": "cpu-basic",
"job_timeout": "30m",
"job_timeout": "none",
"run_label": "run-19ecb4608a42c1e9f4610f25"
}
}
Expand Down Expand Up @@ -210,9 +210,16 @@ A benchmark preset contains `schema_version`, `benchmark`, `preset`,
`n_concurrent_trials` and the native `environment` object. The environment type
is `hf-sandbox`. Its native `kwargs.flavor` value is limited to `cpu-basic` or
`cpu-upgrade`, so a preset cannot select paid accelerator hardware. The
environment also keeps the reviewed native `kwargs.job_timeout` value. The job
can contain timeout multipliers, `retry` and `artifacts`. It cannot set paths,
agents, credentials, user agents, custom environments or source jobs.
environment also keeps the reviewed native `kwargs.job_timeout` value. This
value is temporarily `none` because
[huggingface/sandbox-server#21](https://github.com/huggingface/sandbox-server/pull/21)
is not yet deployed. Remove this stopgap only after the correction is merged,
the fixed server is deployed, and a foreground command runs for more than 30
minutes in a canary. Follow the
[stopgap removal checklist](2026-09-09-sandbox-idle-timeout-stopgap-removal.md).
The job can contain timeout multipliers, `retry` and
`artifacts`. It cannot set paths, agents, credentials, user agents, custom
environments or source jobs.

An agent preset contains `schema_version`, `agent`, `version`, `harbor_agent`,
`reasoning_option` and `reasoning_values`. `harbor_agent` selects `name` or
Expand Down
114 changes: 114 additions & 0 deletions docs/2026-09-09-sandbox-idle-timeout-stopgap-removal.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
---
title: Remove the Sandbox idle-timeout stopgap
author: Harbor-HF maintainers
date: 2026-09-09
tags: [sandbox, timeout, maintenance]
---

# Remove the Sandbox idle-timeout stopgap

## Purpose

Harbor-HF temporarily forces native Harbor configuration to use:

```yaml
environment:
type: hf-sandbox
kwargs:
job_timeout: none
```

The deployed Sandbox server can treat a live foreground command as idle and stop
it after 30 minutes. [sandbox-server PR 21] corrects the server activity check.
[Harbor-HF PR 191] adds this stopgap without adding a keepalive loop or a second
lifecycle implementation.

This stopgap disables the Sandbox idle limit. Harbor phase timeouts, cancellation,
cleanup, and the Hugging Face 24-hour Job limit still bound work.

[sandbox-server PR 21]: https://github.com/huggingface/sandbox-server/pull/21
[Harbor-HF PR 191]: https://github.com/huggingface/harbor-hf/pull/191

## Ownership

- Hugging Face Sandbox owns the server correction and deployment.
- Harbor owns `environment.kwargs.job_timeout` and its timeout semantics.
- Harbor-HF only sets and reviews the native Harbor value.

Do not add provider, model, agent, harness, or benchmark conditions. Do not add a
keepalive request loop.

## Removal gates

Complete all checks before removal:

- [ ] Merge sandbox-server PR 21, or an equivalent correction. Record the merged
commit.
- [ ] Build and deploy a Sandbox server release that contains the correction.
Record the release version and immutable commit or image digest.
- [ ] Confirm from runtime evidence that a new HF Sandbox starts that exact fixed
server build. A merge or mutable download URL is not deployment evidence.
- [ ] Run one foreground command for more than 30 minutes with no keepalive API
traffic. Confirm that it stays alive, returns complete output, exits normally,
and has no idle-shutdown log.
- [ ] Run a true-idle canary with a finite timeout. Confirm that the server still
removes an inactive Sandbox.
- [ ] Confirm that Harbor cancellation, phase timeouts, and Sandbox cleanup still
work with the fixed server.

If a gate fails, keep `job_timeout: none` and record the exact server revision,
command, duration, and logs. Do not restore a finite timeout from merge status
alone.

## Removal changes

After all gates pass, restore the reviewed `30m` idle timeout in one hard cutover:

1. Change `job_timeout` from `none` to `30m` in all files under
`presets/benchmarks/`.
2. Change the `job_timeout` constant from `none` to `30m` in
`packages/contracts/schemas/benchmark-preset-v1.schema.json`.
3. Regenerate `packages/contracts/src/generated/benchmark-preset-v1.ts`.
4. In `packages/control-core/src/direct-config.ts`:
- remove `SANDBOX_IDLE_TIMEOUT_STOPGAP` and `SANDBOX_IDLE_TIMEOUT_FIX`;
- remove the temporary rejection of finite values;
- restore positive duration validation such as `30m`;
- restore `30m` as the default native value; and
- remove the stopgap comments.
5. In `apps/control-web/src/launch-draft.ts`, restore `30m` as the new draft
default and remove the stopgap comments.
6. In `apps/control-web/src/launch-page.tsx`, restore the `30m` fallback and
remove the stopgap notice and upstream link.
7. Update the related contract, control-core, browser, and end-to-end tests:
- `packages/contracts/test/contracts.test.ts`;
- `packages/control-core/test/control.test.ts`;
- `packages/control-core/test/direct-config.test.ts`;
- `apps/control-web/test/App.test.tsx`;
- `apps/control-web/test/launch.test.tsx`; and
- `apps/control-web/e2e/control.spec.ts`.
8. Remove the temporary stopgap text and links from:
- `docs/DESIGN_PRINCIPLES.md`;
- `docs/CONFIGURABLE_LAUNCH.md`; and
- `docs/2026-09-04-simplification-implementation-spec.md`.
9. Delete this TODO document in the same removal pull request. Put the completed
gate evidence in that pull request description.

## Removal verification

Run these checks after the cutover:

```bash
npm run format:check
npm run lint
npm run typecheck
npm test
npm run build
npm run check:generated
npm run test:e2e
uv run python scripts/check_public_privacy.py .
git diff --check
```

Also run the foreground and true-idle canaries from the removal gates against the
fixed deployed server. CI must pass its AMD64 Docker and secret-backed privacy
checks before merge.
11 changes: 9 additions & 2 deletions docs/CONFIGURABLE_LAUNCH.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,15 @@ receive neither the control credential nor the canonical Bucket mount.
## HF restrictions

- Sandbox flavors: `cpu-basic` and `cpu-upgrade`.
- `environment.kwargs.job_timeout` is an **idle timeout**, for example `30m`.
It does not replace agent, trial, or parent wall-clock limits.
- `environment.kwargs.job_timeout` must be `none` as a temporary stopgap for
[huggingface/sandbox-server#21](https://github.com/huggingface/sandbox-server/pull/21).
The current server can stop a running foreground command when its idle timer
expires. Remove this restriction only after the correction is merged, the fixed
server is deployed, and a foreground command runs for more than 30 minutes in a
canary. Follow the
[stopgap removal checklist](2026-09-09-sandbox-idle-timeout-stopgap-removal.md).
Harbor phase timeouts, cleanup, and the Hugging Face 24-hour Job limit still
bound the work.
- Each task needs a prebuilt Docker image. Image builds, resource overrides, and
enforced network allowlists are not offered by this page.
- There are no separate hosted maximums for source count, agent count, attempts,
Expand Down
11 changes: 10 additions & 1 deletion docs/DESIGN_PRINCIPLES.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,9 +180,18 @@ environment:
type: hf-sandbox
kwargs:
flavor: cpu-upgrade
job_timeout: 30m
job_timeout: none
```

The `none` value is a temporary stopgap for
[huggingface/sandbox-server#21](https://github.com/huggingface/sandbox-server/pull/21).
The current Sandbox server can stop a running foreground command when its idle
timer expires. Remove the stopgap only after the correction is merged, the fixed
server is deployed, and a foreground command runs for more than 30 minutes in a
canary. Follow the
[stopgap removal checklist](2026-09-09-sandbox-idle-timeout-stopgap-removal.md).
Harbor phase timeouts and cleanup still bound the work during this period.

Harbor-HF MAY replace `type: hf-sandbox` with its labeled environment adapter
when it compiles the final job. The compiler MUST preserve the native
environment arguments and MUST add only the `run_label` argument required by
Expand Down
2 changes: 1 addition & 1 deletion packages/contracts/schemas/benchmark-preset-v1.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
"required": ["flavor", "job_timeout"],
"properties": {
"flavor": { "enum": ["cpu-basic", "cpu-upgrade"] },
"job_timeout": { "const": "30m" }
"job_timeout": { "const": "none" }
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion packages/contracts/src/generated/benchmark-preset-v1.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ environment: {
type: "hf-sandbox"
kwargs: {
flavor: ("cpu-basic" | "cpu-upgrade")
job_timeout: "30m"
job_timeout: "none"
}
}
timeout_multiplier?: number
Expand Down
2 changes: 1 addition & 1 deletion packages/contracts/test/contracts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ describe("contracts", () => {
n_concurrent_trials: 1,
environment: {
type: "hf-sandbox",
kwargs: { flavor: "cpu-upgrade", job_timeout: "30m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
},
},
} as const;
Expand Down
18 changes: 14 additions & 4 deletions packages/control-core/src/direct-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ import {
ROUTER_URL,
} from "./hf-config.js";

const SANDBOX_IDLE_TIMEOUT_STOPGAP = "none";
const SANDBOX_IDLE_TIMEOUT_FIX =
"https://github.com/huggingface/sandbox-server/pull/21";

function record(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value))
throw new Error(`${label} must be an object`);
Expand Down Expand Up @@ -121,11 +125,10 @@ export function prepareDirectJobConfig(
throw new Error("HF Sandbox flavor must be a string");
if (
kwargs.job_timeout !== undefined &&
(typeof kwargs.job_timeout !== "string" ||
!/^[1-9][0-9]*(s|m|h)$/.test(kwargs.job_timeout))
kwargs.job_timeout !== SANDBOX_IDLE_TIMEOUT_STOPGAP
)
throw new Error(
"Sandbox idle timeout must be a positive duration, for example 30m",
`Sandbox idle timeout must stay disabled with job_timeout="none" until ${SANDBOX_IDLE_TIMEOUT_FIX} is merged and deployed`,
);
const verifier = record(input.verifier ?? {}, "verifier");
admittedEnvironment(verifier.env);
Expand All @@ -139,7 +142,14 @@ export function prepareDirectJobConfig(
environment: {
import_path: LABELED_ENVIRONMENT,
env: admittedEnvironment(environment.env),
kwargs: { flavor: "cpu-basic", job_timeout: "30m", ...kwargs, run_label: runId },
// Remove this stopgap after the fixed Sandbox server is deployed and a
// foreground command has passed a canary longer than 30 minutes.
kwargs: {
flavor: "cpu-basic",
job_timeout: SANDBOX_IDLE_TIMEOUT_STOPGAP,
...kwargs,
run_label: runId,
},
},
});
}
6 changes: 3 additions & 3 deletions packages/control-core/test/control.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ describe("run submission", () => {
presets.benchmark("terminal-bench-2-1", "one-task-1-trial").job.environment,
).toEqual({
type: "hf-sandbox",
kwargs: { flavor: "cpu-upgrade", job_timeout: "30m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
});
const result = await submit();
expect(result.created).toBe(true);
Expand All @@ -132,7 +132,7 @@ describe("run submission", () => {
import_path: "harbor_hf_agents.hf_sandbox:LabeledHFSandboxEnvironment",
kwargs: {
flavor: "cpu-upgrade",
job_timeout: "30m",
job_timeout: "none",
run_label: result.run.run_id,
},
},
Expand Down Expand Up @@ -246,7 +246,7 @@ describe("run submission", () => {
n_attempts: 1,
n_concurrent_trials: 8,
environment: {
kwargs: { flavor: "cpu-upgrade", job_timeout: "30m" },
kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
},
});
});
Expand Down
Loading