[sandbox audit] Give each pooled sandbox its own capability token

README.md

@@ -54,10 +54,11 @@ DELETE /v1/files/delete?path=&recursive=
 POST /v1/files/mkdir?path=
 
 # host mode (many sandboxes per job)
-POST   /v1/sandboxes        {count?, env?, max_procs?, max_mem_mb?}  → {"sandboxes":[{id,uid,home}]}
+POST   /v1/sandboxes        {count?, env?, max_procs?, max_mem_mb?}  → {"sandboxes":[{id,token,uid,home}]}
 GET    /v1/sandboxes                                                 → live sandbox list
 DELETE /v1/sandboxes                                                 → delete all
 DELETE /v1/sandboxes/{id}                                            → delete one (frees the uid)
+GET    /v1/sandboxes/{id}/token                                      → recover its capability token
 # every dedicated route above also exists scoped to a sandbox, e.g.:
 POST   /v1/sandboxes/{id}/exec        ...   GET /v1/sandboxes/{id}/processes
 GET    /v1/sandboxes/{id}/files/read  ...   PUT /v1/sandboxes/{id}/files/write
@@ -78,6 +79,7 @@ the home, and it assigns ownership through the resulting descriptor rather than
 | `SBX_PORT` | `8000` | listen port (the client uses 49983 to keep common dev ports free) |
 | `SBX_TOKEN` | **required** | all endpoints except `/health` require this value in the `X-Sandbox-Token` header (constant-time compare); removed from the env before any child process spawns. The server refuses to start without it, unless launched with `--allow-no-auth` (local development only — it is an argv flag, not an env var, so a Job's user-supplied env can never set it) |
 | `SBX_IDLE_TIMEOUT` | unset | seconds of inactivity (no authed request, no running process) before clean exit |
+| `SBX_COMPAT_HOST_TOKEN` | `1` | host mode: whether the host token is still accepted on per-sandbox routes, for clients that predate per-sandbox tokens. Set to `0` to require scoped tokens |
 
 ## Security model
 
@@ -94,10 +96,24 @@ Two layers when running on HF Jobs:
    `HMAC-SHA256(user_hf_token, nonce)` with the nonce stored in job labels — so
    reconnection is stateless and the HF token itself never enters the sandbox.
 
-`SBX_TOKEN` is **one token per server process**, not per sandbox. In dedicated mode the job
-*is* the sandbox, so the two coincide. In host mode the same token authorizes every
-`/v1/sandboxes/{id}/*` route for every sandbox on the host plus the pool-management routes
-(`POST`/`GET`/`DELETE /v1/sandboxes`), so a leak is host-wide.
+In dedicated mode the job *is* the sandbox, so `SBX_TOKEN` is already scoped to it.
+
+In host mode there are two kinds of credential:
+
+- **`SBX_TOKEN` is the host management token.** It creates, lists and deletes sandboxes, and
+  recovers their tokens. It is held by whoever runs the pool.
+- **Each sandbox gets its own random 256-bit capability token**, returned by `POST
+  /v1/sandboxes` and recoverable with `GET /v1/sandboxes/{id}/token`. It authorizes that
+  sandbox's routes and nothing else — not a sibling, not the pool. This is the credential to
+  hand to whoever operates a single sandbox, including into a browser or WebSocket client via
+  the port proxy.
+
+The host token is *also* accepted on per-sandbox routes while `SBX_COMPAT_HOST_TOKEN=1` (the
+default), so clients that predate per-sandbox tokens keep working when this binary is
+published under them — every job fetches the binary fresh, so a hard break would break every
+old client at once. That is a management credential having authority over the sandboxes it
+created, not a sandbox credential reaching a sibling. Set `SBX_COMPAT_HOST_TOKEN=0` to close
+it once clients have upgraded.
 
 ### Known limitations
 

scripts/token-scope-regression.sh

@@ -0,0 +1,115 @@
+#!/bin/sh
+# Live regression for per-sandbox capability tokens.
+#
+# The property under test: a credential handed out for one pooled sandbox must
+# not address a sibling, and must not manage the pool. The host token stays a
+# management credential.
+#
+#   docker run --rm -v "$PWD:/src" -w /src sh scripts/token-scope-regression.sh
+set -eu
+
+BIN=${BIN:-target/x86_64-unknown-linux-musl/release/sbx-server}
+PORT=${PORT:-49401}
+HOST_TOKEN=host-management-token
+U="http://127.0.0.1:$PORT"
+failures=0
+
+say()  { printf '\n=== %s\n' "$1"; }
+pass() { printf '  ok    %s\n' "$1"; }
+fail() { printf '  FAIL  %s\n' "$1"; failures=$((failures + 1)); }
+
+# $1 expected status, $2 token, $3 description, rest: curl args
+expect() {
+    want=$1
+    tok=$2
+    desc=$3
+    shift 3
+    got=$(curl -s -o /tmp/body -w '%{http_code}' -H "X-Sandbox-Token: $tok" "$@" || echo 000)
+    if [ "$got" = "$want" ]; then
+        pass "$desc ($got)"
+    else
+        fail "$desc (wanted $want, got $got: $(head -c 160 /tmp/body))"
+    fi
+}
+
+field() { sed "s/.*\"$2\":\"\([^\"]*\)\".*/\1/" "$1"; }
+
+command -v curl >/dev/null || { apt-get update -qq && apt-get install -y -qq curl >/dev/null; }
+[ -f "$BIN" ] || cargo build --release --target x86_64-unknown-linux-musl
+
+SBX_PORT=$PORT SBX_TOKEN=$HOST_TOKEN SBX_HOST_MODE=1 SBX_CAPACITY=8 "$BIN" &
+server=$!
+trap 'kill $server 2>/dev/null || true' EXIT
+sleep 1
+
+say "creating two sandboxes"
+curl -s -H "X-Sandbox-Token: $HOST_TOKEN" -X POST "$U/v1/sandboxes" -d '{"count":2}' >/tmp/created
+# Two {"id":...,"token":...} objects; split them so each can be read separately.
+sed 's/},{/}\n{/g' /tmp/created | grep '"id"' >/tmp/objs
+sed -n '1p' /tmp/objs >/tmp/a
+sed -n '2p' /tmp/objs >/tmp/b
+A_ID=$(field /tmp/a id); A_TOK=$(field /tmp/a token)
+B_ID=$(field /tmp/b id); B_TOK=$(field /tmp/b token)
+echo "  A=$A_ID  B=$B_ID"
+
+[ -n "$A_TOK" ] && [ "$A_TOK" != "$A_ID" ] &&
+    pass "create returns a per-sandbox token" ||
+    fail "create returned no token (got: $(head -c 200 /tmp/created))"
+[ "$A_TOK" != "$B_TOK" ] && pass "the two tokens differ" || fail "both sandboxes share a token"
+[ "$A_TOK" != "$HOST_TOKEN" ] && pass "not the host token" || fail "the sandbox token IS the host token"
+[ "${#A_TOK}" = 64 ] && pass "256 bits of token" || fail "unexpected token length ${#A_TOK}"
+
+say "a sandbox's own token works on its own routes"
+expect 200 "$A_TOK" "exec"        -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"echo hi"}'
+expect 200 "$A_TOK" "files write" -X PUT  "$U/v1/sandboxes/$A_ID/files/write?path=f" -d 'x'
+expect 200 "$A_TOK" "files read"       "$U/v1/sandboxes/$A_ID/files/read?path=f"
+expect 200 "$A_TOK" "processes"        "$U/v1/sandboxes/$A_ID/processes"
+
+say "A's token must not address B"
+expect 403 "$A_TOK" "exec in B"    -X POST   "$U/v1/sandboxes/$B_ID/exec" -d '{"cmd":"id"}'
+expect 403 "$A_TOK" "read B"                 "$U/v1/sandboxes/$B_ID/files/read?path=f"
+expect 403 "$A_TOK" "write into B" -X PUT    "$U/v1/sandboxes/$B_ID/files/write?path=pwned" -d 'x'
+expect 403 "$A_TOK" "delete B"     -X DELETE "$U/v1/sandboxes/$B_ID"
+expect 403 "$A_TOK" "proxy into B"           "$U/v1/sandboxes/$B_ID/proxy/9000/"
+expect 403 "$A_TOK" "list B's processes"     "$U/v1/sandboxes/$B_ID/processes"
+
+say "a sandbox token must not manage the pool"
+expect 403 "$A_TOK" "create"          -X POST   "$U/v1/sandboxes" -d '{"count":1}'
+expect 403 "$A_TOK" "list sandboxes"            "$U/v1/sandboxes"
+expect 403 "$A_TOK" "delete all"      -X DELETE "$U/v1/sandboxes"
+# Token recovery must be management-gated, or the scoping would be trivially bypassable.
+expect 403 "$A_TOK" "recover B's token"         "$U/v1/sandboxes/$B_ID/token"
+expect 403 "$A_TOK" "recover its own token"     "$U/v1/sandboxes/$A_ID/token"
+
+say "the host token manages the pool and recovers tokens"
+expect 200 "$HOST_TOKEN" "list sandboxes"    "$U/v1/sandboxes"
+expect 200 "$HOST_TOKEN" "recover A's token" "$U/v1/sandboxes/$A_ID/token"
+grep -q "$A_TOK" /tmp/body && pass "recovered token matches" || fail "recovered a different token"
+
+say "an unknown token is refused"
+expect 403 "not-a-real-token" "garbage token" -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"id"}'
+expect 403 "" "empty token"                   -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"id"}'
+
+say "the compat window (host token on scoped routes) is on by default"
+expect 200 "$HOST_TOKEN" "host token on a scoped route" -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"echo hi"}'
+kill $server 2>/dev/null || true
+wait $server 2>/dev/null || true
+
+say "SBX_COMPAT_HOST_TOKEN=0 closes it"
+SBX_PORT=$PORT SBX_TOKEN=$HOST_TOKEN SBX_HOST_MODE=1 SBX_COMPAT_HOST_TOKEN=0 "$BIN" &
+server=$!
+sleep 1
+curl -s -H "X-Sandbox-Token: $HOST_TOKEN" -X POST "$U/v1/sandboxes" -d '{"count":1}' >/tmp/created
+C_ID=$(sed 's/.*"id":"\([^"]*\)".*/\1/' /tmp/created)
+C_TOK=$(sed 's/.*"token":"\([^"]*\)".*/\1/' /tmp/created)
+expect 403 "$HOST_TOKEN" "host token refused on a scoped route" -X POST "$U/v1/sandboxes/$C_ID/exec" -d '{"cmd":"id"}'
+expect 200 "$C_TOK"      "the scoped token still works"         -X POST "$U/v1/sandboxes/$C_ID/exec" -d '{"cmd":"echo hi"}'
+expect 200 "$HOST_TOKEN" "management still works"                         "$U/v1/sandboxes"
+
+say "result"
+if [ "$failures" -eq 0 ]; then
+    echo "all checks passed"
+else
+    echo "$failures check(s) failed"
+    exit 1
+fi

src/main.rs

@@ -65,6 +65,9 @@ pub struct State {
     /// sandbox). Picks the idle policy: per-sandbox eviction + empty-host shutdown, vs
     /// the whole-job activity watchdog.
     pub host_mode: bool,
+    /// Whether the host management token is still accepted on per-sandbox routes
+    /// (see [`authorize`]). Transitional; `SBX_COMPAT_HOST_TOKEN=0` turns it off.
+    pub compat_host_token: bool,
 }
 
 /// Constant-time string comparison.
@@ -76,8 +79,75 @@ fn ct_eq(a: &str, b: &str) -> bool {
     a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
 }
 
-fn authorized(state: &State, request: &Request) -> bool {
-    state.auth.accepts(request.header("x-sandbox-token"))
+/// What a presented credential is allowed to address.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Scope {
+    /// The host management token (`SBX_TOKEN`), or the dedicated-mode token.
+    Host,
+    /// A capability token bound to exactly one pooled sandbox.
+    Sandbox,
+}
+
+/// Authorize a request and decide what it may address.
+///
+/// Dedicated mode has one credential and one sandbox, so there is nothing to
+/// scope. Host mode has two kinds:
+///
+/// - pool lifecycle (`/v1/sandboxes`, and token recovery) is management, so it
+///   needs the host token;
+/// - a scoped route accepts that sandbox's own capability token.
+///
+/// The host token is *also* accepted on scoped routes for now, so that clients
+/// which predate per-sandbox tokens keep working when this binary is published
+/// under them (every job fetches the binary fresh, so a hard break would break
+/// every old client at once). That is a management credential legitimately
+/// having authority over the sandboxes it created — not a sandbox credential
+/// reaching a sibling, which is what this change closes. Set
+/// `SBX_COMPAT_HOST_TOKEN=0` to refuse it and require scoped tokens today.
+/// Which credential a host-mode route requires.
+enum RouteAuth<'a> {
+    /// Pool lifecycle and token recovery: the host management token.
+    Management,
+    /// Scoped to one sandbox: that sandbox's capability token.
+    Sandbox(&'a str),
+    /// Not a host-mode route.
+    Unknown,
+}
+
+fn host_route_auth<'a>(segments: &[&'a str]) -> RouteAuth<'a> {
+    match segments {
+        ["v1", "sandboxes"] | ["v1", "sandboxes", _, "token"] => RouteAuth::Management,
+        ["v1", "sandboxes", id, ..] => RouteAuth::Sandbox(id),
+        _ => RouteAuth::Unknown,
+    }
+}
+
+/// Decide what a credential presented on a scoped route may address.
+fn classify_scoped_token(provided: &str, sandbox_token: &str, host: &Auth, compat: bool) -> Option<Scope> {
+    if ct_eq(provided, sandbox_token) {
+        return Some(Scope::Sandbox);
+    }
+    if compat && host.accepts(Some(provided)) {
+        return Some(Scope::Host);
+    }
+    None
+}
+
+fn authorize(state: &State, provided: Option<&str>, segments: &[&str]) -> Option<Scope> {
+    if !state.host_mode {
+        return state.auth.accepts(provided).then_some(Scope::Host);
+    }
+    match host_route_auth(segments) {
+        RouteAuth::Management => state.auth.accepts(provided).then_some(Scope::Host),
+        RouteAuth::Sandbox(id) => {
+            let entry = state.sandboxes.get(id)?;
+            classify_scoped_token(provided?, &entry.token, &state.auth, state.compat_host_token)
+        }
+        // Not a host-mode route. Whether it *exists* is the route gate's
+        // question, not authorization's: check the host credential so a valid
+        // caller gets an accurate 404 while an invalid one still gets 403.
+        RouteAuth::Unknown => state.auth.accepts(provided).then_some(Scope::Host),
+    }
 }
 
 /// Whether a route exists in this server mode.
@@ -115,17 +185,17 @@ fn route(
         );
     }
 
+    let segments: Vec<&str> = path.trim_matches('/').split('/').collect();
     // Authentication first, so an unauthenticated caller learns nothing about
     // which routes this server serves.
-    if !authorized(state, request) {
+    if authorize(state, request.header("x-sandbox-token"), &segments).is_none() {
         return resp.error(403, "invalid or missing X-Sandbox-Token");
     }
     if request.header("transfer-encoding").is_some() {
         return resp.error(411, "chunked request bodies not supported; send Content-Length");
     }
     state.last_activity_ms.store(now_ms(), Ordering::Relaxed);
 
-    let segments: Vec<&str> = path.trim_matches('/').split('/').collect();
     if !route_mode_allowed(state.host_mode, &segments) {
         return resp.error(404, &format!("no route in this server mode: {method} {path}"));
     }
@@ -151,6 +221,12 @@ fn route(
         ("POST", ["v1", "sandboxes"]) => sandboxes::handle_create(state, request, reader, resp),
         ("GET", ["v1", "sandboxes"]) => resp.json(200, &state.sandboxes.list()),
         ("DELETE", ["v1", "sandboxes"]) => sandboxes::handle_delete_all(state, resp),
+        // Recover a sandbox's capability token with the host token, so a client
+        // that reconnects to an existing sandbox does not need local state.
+        ("GET", ["v1", "sandboxes", id, "token"]) => match state.sandboxes.get(id) {
+            Some(entry) => resp.json(200, &serde_json::json!({"id": entry.id, "token": entry.token})),
+            None => resp.error(404, &format!("no such sandbox: {id}")),
+        },
         ("DELETE", ["v1", "sandboxes", id]) => {
             let id = id.to_string();
             sandboxes::handle_delete(state, &id, resp)
@@ -256,6 +332,9 @@ fn main() {
     let capacity = std::env::var("SBX_CAPACITY").ok().and_then(|v| v.parse().ok()).unwrap_or(usize::MAX);
     // Host mode multiplexes many sandboxes; dedicated mode is one sandbox == the job.
     let host_mode = std::env::var("SBX_HOST_MODE").map(|v| v == "1").unwrap_or(false);
+    // Transitional: accept the host token on per-sandbox routes for clients that
+    // predate per-sandbox tokens. Set to 0 to require scoped tokens.
+    let compat_host_token = std::env::var("SBX_COMPAT_HOST_TOKEN").map(|v| v != "0").unwrap_or(true);
 
     let state = Arc::new(State {
         auth,
@@ -264,6 +343,7 @@ fn main() {
         procs: exec::ProcRegistry::default(),
         sandboxes: sandboxes::SandboxRegistry::with_capacity(capacity),
         host_mode,
+        compat_host_token,
     });
 
     // Idle watchdog: stop billing for an abandoned sandbox/host before the job timeout.
@@ -313,10 +393,11 @@ fn main() {
     // Mode and auth state on the first line: a server that silently serves the
     // wrong surface, or no authentication at all, is the hazard worth seeing.
     eprintln!(
-        "sbx-server {VERSION} listening on 0.0.0.0:{port} (mode: {}, auth: {}, landlock: {})",
+        "sbx-server {VERSION} listening on 0.0.0.0:{port} (mode: {}, auth: {}, landlock: {}{})",
         if host_mode { "host" } else { "dedicated" },
         if matches!(state.auth, Auth::Required(_)) { "required" } else { "DISABLED" },
-        if landlock_ok { "enabled" } else { "UNAVAILABLE — uid isolation only" }
+        if landlock_ok { "enabled" } else { "UNAVAILABLE — uid isolation only" },
+        if host_mode && compat_host_token { ", host-token compat: on" } else { "" }
     );
     // Host mode reuses one set of system-dir fds across every sandbox ruleset.
     // Open them now so the cost (and any missing-dir surface) lands at startup
@@ -408,4 +489,71 @@ mod tests {
             assert_eq!(ct_eq(a, b), a == b, "ct_eq({a:?}, {b:?})");
         }
     }
+
+    /// Only the sandbox's own capability token addresses it. A sibling's token
+    /// is refused even though it is a perfectly valid credential elsewhere --
+    /// this is the property the change exists for.
+    #[test]
+    fn a_sandbox_token_addresses_only_its_own_sandbox() {
+        let host = Auth::Required("host-management-token".to_string());
+        let mine = "sandbox-a-token";
+        let theirs = "sandbox-b-token";
+
+        assert_eq!(classify_scoped_token(mine, mine, &host, false), Some(Scope::Sandbox));
+        assert_eq!(classify_scoped_token(theirs, mine, &host, false), None, "a sibling's token must not work");
+        assert_eq!(classify_scoped_token("", mine, &host, false), None);
+        assert_eq!(classify_scoped_token("sandbox-a-toke", mine, &host, false), None, "a prefix must not work");
+    }
+
+    #[test]
+    fn the_host_token_on_a_scoped_route_depends_on_the_compat_window() {
+        let host = Auth::Required("host-management-token".to_string());
+        let sandbox = "sandbox-a-token";
+
+        // Transitional: a management credential may address its sandboxes.
+        assert_eq!(
+            classify_scoped_token("host-management-token", sandbox, &host, true),
+            Some(Scope::Host)
+        );
+        // With the window closed, scoped routes require scoped tokens.
+        assert_eq!(classify_scoped_token("host-management-token", sandbox, &host, false), None);
+        // Either way it is still recognised as the host credential, never as the sandbox's.
+        assert_ne!(
+            classify_scoped_token("host-management-token", sandbox, &host, true),
+            Some(Scope::Sandbox)
+        );
+    }
+
+    #[test]
+    fn pool_lifecycle_and_token_recovery_are_management_routes() {
+        let management = ["/v1/sandboxes"];
+        for path in management {
+            assert!(
+                matches!(host_route_auth(&segments(path)), RouteAuth::Management),
+                "{path} must require the host token"
+            );
+        }
+        assert!(matches!(host_route_auth(&segments("/v1/sandboxes/abc/token")), RouteAuth::Management));
+
+        // Everything else scoped under a sandbox id belongs to that sandbox.
+        for path in [
+            "/v1/sandboxes/abc",
+            "/v1/sandboxes/abc/exec",
+            "/v1/sandboxes/abc/processes",
+            "/v1/sandboxes/abc/files/read",
+            "/v1/sandboxes/abc/proxy/8000/ws",
+        ] {
+            match host_route_auth(&segments(path)) {
+                RouteAuth::Sandbox(id) => assert_eq!(id, "abc", "{path}"),
+                _ => panic!("{path} should be scoped to a sandbox"),
+            }
+        }
+    }
+
+    /// Token recovery must be management-gated: if a sandbox's own token could
+    /// read `/v1/sandboxes/<other>/token`, the whole scoping would be moot.
+    #[test]
+    fn token_recovery_is_not_reachable_with_a_sandbox_token() {
+        assert!(matches!(host_route_auth(&segments("/v1/sandboxes/victim/token")), RouteAuth::Management));
+    }
 }

src/proxy.rs

@@ -289,6 +289,7 @@ mod tests {
             env: HashMap::new(),
             max_procs: 16,
             max_mem_mb: 16,
+            token: "sandbox-capability-token".to_string(),
             landlock_fd: -1,
             last_activity_ms: AtomicI64::new(0),
             idle_timeout_ms: 0,