diff --git a/README.md b/README.md index d48fb5d..325ef4e 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,11 @@ DELETE /v1/files/delete?path=&recursive= POST /v1/files/mkdir?path= # host mode (many sandboxes per job) -POST /v1/sandboxes {count?, env?, max_procs?, max_mem_mb?} → {"sandboxes":[{id,uid,home}]} +POST /v1/sandboxes {count?, env?, max_procs?, max_mem_mb?} → {"sandboxes":[{id,token,uid,home}]} GET /v1/sandboxes → live sandbox list DELETE /v1/sandboxes → delete all DELETE /v1/sandboxes/{id} → delete one (frees the uid) +GET /v1/sandboxes/{id}/token → recover its capability token # every dedicated route above also exists scoped to a sandbox, e.g.: POST /v1/sandboxes/{id}/exec ... GET /v1/sandboxes/{id}/processes GET /v1/sandboxes/{id}/files/read ... PUT /v1/sandboxes/{id}/files/write @@ -78,6 +79,7 @@ the home, and it assigns ownership through the resulting descriptor rather than | `SBX_PORT` | `8000` | listen port (the client uses 49983 to keep common dev ports free) | | `SBX_TOKEN` | **required** | all endpoints except `/health` require this value in the `X-Sandbox-Token` header (constant-time compare); removed from the env before any child process spawns. The server refuses to start without it, unless launched with `--allow-no-auth` (local development only — it is an argv flag, not an env var, so a Job's user-supplied env can never set it) | | `SBX_IDLE_TIMEOUT` | unset | seconds of inactivity (no authed request, no running process) before clean exit | +| `SBX_COMPAT_HOST_TOKEN` | `1` | host mode: whether the host token is still accepted on per-sandbox routes, for clients that predate per-sandbox tokens. Set to `0` to require scoped tokens | ## Security model @@ -94,10 +96,24 @@ Two layers when running on HF Jobs: `HMAC-SHA256(user_hf_token, nonce)` with the nonce stored in job labels — so reconnection is stateless and the HF token itself never enters the sandbox. -`SBX_TOKEN` is **one token per server process**, not per sandbox. In dedicated mode the job -*is* the sandbox, so the two coincide. In host mode the same token authorizes every -`/v1/sandboxes/{id}/*` route for every sandbox on the host plus the pool-management routes -(`POST`/`GET`/`DELETE /v1/sandboxes`), so a leak is host-wide. +In dedicated mode the job *is* the sandbox, so `SBX_TOKEN` is already scoped to it. + +In host mode there are two kinds of credential: + +- **`SBX_TOKEN` is the host management token.** It creates, lists and deletes sandboxes, and + recovers their tokens. It is held by whoever runs the pool. +- **Each sandbox gets its own random 256-bit capability token**, returned by `POST + /v1/sandboxes` and recoverable with `GET /v1/sandboxes/{id}/token`. It authorizes that + sandbox's routes and nothing else — not a sibling, not the pool. This is the credential to + hand to whoever operates a single sandbox, including into a browser or WebSocket client via + the port proxy. + +The host token is *also* accepted on per-sandbox routes while `SBX_COMPAT_HOST_TOKEN=1` (the +default), so clients that predate per-sandbox tokens keep working when this binary is +published under them — every job fetches the binary fresh, so a hard break would break every +old client at once. That is a management credential having authority over the sandboxes it +created, not a sandbox credential reaching a sibling. Set `SBX_COMPAT_HOST_TOKEN=0` to close +it once clients have upgraded. ### Known limitations diff --git a/scripts/token-scope-regression.sh b/scripts/token-scope-regression.sh new file mode 100644 index 0000000..dd059df --- /dev/null +++ b/scripts/token-scope-regression.sh @@ -0,0 +1,115 @@ +#!/bin/sh +# Live regression for per-sandbox capability tokens. +# +# The property under test: a credential handed out for one pooled sandbox must +# not address a sibling, and must not manage the pool. The host token stays a +# management credential. +# +# docker run --rm -v "$PWD:/src" -w /src sh scripts/token-scope-regression.sh +set -eu + +BIN=${BIN:-target/x86_64-unknown-linux-musl/release/sbx-server} +PORT=${PORT:-49401} +HOST_TOKEN=host-management-token +U="http://127.0.0.1:$PORT" +failures=0 + +say() { printf '\n=== %s\n' "$1"; } +pass() { printf ' ok %s\n' "$1"; } +fail() { printf ' FAIL %s\n' "$1"; failures=$((failures + 1)); } + +# $1 expected status, $2 token, $3 description, rest: curl args +expect() { + want=$1 + tok=$2 + desc=$3 + shift 3 + got=$(curl -s -o /tmp/body -w '%{http_code}' -H "X-Sandbox-Token: $tok" "$@" || echo 000) + if [ "$got" = "$want" ]; then + pass "$desc ($got)" + else + fail "$desc (wanted $want, got $got: $(head -c 160 /tmp/body))" + fi +} + +field() { sed "s/.*\"$2\":\"\([^\"]*\)\".*/\1/" "$1"; } + +command -v curl >/dev/null || { apt-get update -qq && apt-get install -y -qq curl >/dev/null; } +[ -f "$BIN" ] || cargo build --release --target x86_64-unknown-linux-musl + +SBX_PORT=$PORT SBX_TOKEN=$HOST_TOKEN SBX_HOST_MODE=1 SBX_CAPACITY=8 "$BIN" & +server=$! +trap 'kill $server 2>/dev/null || true' EXIT +sleep 1 + +say "creating two sandboxes" +curl -s -H "X-Sandbox-Token: $HOST_TOKEN" -X POST "$U/v1/sandboxes" -d '{"count":2}' >/tmp/created +# Two {"id":...,"token":...} objects; split them so each can be read separately. +sed 's/},{/}\n{/g' /tmp/created | grep '"id"' >/tmp/objs +sed -n '1p' /tmp/objs >/tmp/a +sed -n '2p' /tmp/objs >/tmp/b +A_ID=$(field /tmp/a id); A_TOK=$(field /tmp/a token) +B_ID=$(field /tmp/b id); B_TOK=$(field /tmp/b token) +echo " A=$A_ID B=$B_ID" + +[ -n "$A_TOK" ] && [ "$A_TOK" != "$A_ID" ] && + pass "create returns a per-sandbox token" || + fail "create returned no token (got: $(head -c 200 /tmp/created))" +[ "$A_TOK" != "$B_TOK" ] && pass "the two tokens differ" || fail "both sandboxes share a token" +[ "$A_TOK" != "$HOST_TOKEN" ] && pass "not the host token" || fail "the sandbox token IS the host token" +[ "${#A_TOK}" = 64 ] && pass "256 bits of token" || fail "unexpected token length ${#A_TOK}" + +say "a sandbox's own token works on its own routes" +expect 200 "$A_TOK" "exec" -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"echo hi"}' +expect 200 "$A_TOK" "files write" -X PUT "$U/v1/sandboxes/$A_ID/files/write?path=f" -d 'x' +expect 200 "$A_TOK" "files read" "$U/v1/sandboxes/$A_ID/files/read?path=f" +expect 200 "$A_TOK" "processes" "$U/v1/sandboxes/$A_ID/processes" + +say "A's token must not address B" +expect 403 "$A_TOK" "exec in B" -X POST "$U/v1/sandboxes/$B_ID/exec" -d '{"cmd":"id"}' +expect 403 "$A_TOK" "read B" "$U/v1/sandboxes/$B_ID/files/read?path=f" +expect 403 "$A_TOK" "write into B" -X PUT "$U/v1/sandboxes/$B_ID/files/write?path=pwned" -d 'x' +expect 403 "$A_TOK" "delete B" -X DELETE "$U/v1/sandboxes/$B_ID" +expect 403 "$A_TOK" "proxy into B" "$U/v1/sandboxes/$B_ID/proxy/9000/" +expect 403 "$A_TOK" "list B's processes" "$U/v1/sandboxes/$B_ID/processes" + +say "a sandbox token must not manage the pool" +expect 403 "$A_TOK" "create" -X POST "$U/v1/sandboxes" -d '{"count":1}' +expect 403 "$A_TOK" "list sandboxes" "$U/v1/sandboxes" +expect 403 "$A_TOK" "delete all" -X DELETE "$U/v1/sandboxes" +# Token recovery must be management-gated, or the scoping would be trivially bypassable. +expect 403 "$A_TOK" "recover B's token" "$U/v1/sandboxes/$B_ID/token" +expect 403 "$A_TOK" "recover its own token" "$U/v1/sandboxes/$A_ID/token" + +say "the host token manages the pool and recovers tokens" +expect 200 "$HOST_TOKEN" "list sandboxes" "$U/v1/sandboxes" +expect 200 "$HOST_TOKEN" "recover A's token" "$U/v1/sandboxes/$A_ID/token" +grep -q "$A_TOK" /tmp/body && pass "recovered token matches" || fail "recovered a different token" + +say "an unknown token is refused" +expect 403 "not-a-real-token" "garbage token" -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"id"}' +expect 403 "" "empty token" -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"id"}' + +say "the compat window (host token on scoped routes) is on by default" +expect 200 "$HOST_TOKEN" "host token on a scoped route" -X POST "$U/v1/sandboxes/$A_ID/exec" -d '{"cmd":"echo hi"}' +kill $server 2>/dev/null || true +wait $server 2>/dev/null || true + +say "SBX_COMPAT_HOST_TOKEN=0 closes it" +SBX_PORT=$PORT SBX_TOKEN=$HOST_TOKEN SBX_HOST_MODE=1 SBX_COMPAT_HOST_TOKEN=0 "$BIN" & +server=$! +sleep 1 +curl -s -H "X-Sandbox-Token: $HOST_TOKEN" -X POST "$U/v1/sandboxes" -d '{"count":1}' >/tmp/created +C_ID=$(sed 's/.*"id":"\([^"]*\)".*/\1/' /tmp/created) +C_TOK=$(sed 's/.*"token":"\([^"]*\)".*/\1/' /tmp/created) +expect 403 "$HOST_TOKEN" "host token refused on a scoped route" -X POST "$U/v1/sandboxes/$C_ID/exec" -d '{"cmd":"id"}' +expect 200 "$C_TOK" "the scoped token still works" -X POST "$U/v1/sandboxes/$C_ID/exec" -d '{"cmd":"echo hi"}' +expect 200 "$HOST_TOKEN" "management still works" "$U/v1/sandboxes" + +say "result" +if [ "$failures" -eq 0 ]; then + echo "all checks passed" +else + echo "$failures check(s) failed" + exit 1 +fi diff --git a/src/main.rs b/src/main.rs index 9ca7f83..d6629c8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,6 +65,9 @@ pub struct State { /// sandbox). Picks the idle policy: per-sandbox eviction + empty-host shutdown, vs /// the whole-job activity watchdog. pub host_mode: bool, + /// Whether the host management token is still accepted on per-sandbox routes + /// (see [`authorize`]). Transitional; `SBX_COMPAT_HOST_TOKEN=0` turns it off. + pub compat_host_token: bool, } /// Constant-time string comparison. @@ -76,8 +79,75 @@ fn ct_eq(a: &str, b: &str) -> bool { a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 } -fn authorized(state: &State, request: &Request) -> bool { - state.auth.accepts(request.header("x-sandbox-token")) +/// What a presented credential is allowed to address. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Scope { + /// The host management token (`SBX_TOKEN`), or the dedicated-mode token. + Host, + /// A capability token bound to exactly one pooled sandbox. + Sandbox, +} + +/// Authorize a request and decide what it may address. +/// +/// Dedicated mode has one credential and one sandbox, so there is nothing to +/// scope. Host mode has two kinds: +/// +/// - pool lifecycle (`/v1/sandboxes`, and token recovery) is management, so it +/// needs the host token; +/// - a scoped route accepts that sandbox's own capability token. +/// +/// The host token is *also* accepted on scoped routes for now, so that clients +/// which predate per-sandbox tokens keep working when this binary is published +/// under them (every job fetches the binary fresh, so a hard break would break +/// every old client at once). That is a management credential legitimately +/// having authority over the sandboxes it created — not a sandbox credential +/// reaching a sibling, which is what this change closes. Set +/// `SBX_COMPAT_HOST_TOKEN=0` to refuse it and require scoped tokens today. +/// Which credential a host-mode route requires. +enum RouteAuth<'a> { + /// Pool lifecycle and token recovery: the host management token. + Management, + /// Scoped to one sandbox: that sandbox's capability token. + Sandbox(&'a str), + /// Not a host-mode route. + Unknown, +} + +fn host_route_auth<'a>(segments: &[&'a str]) -> RouteAuth<'a> { + match segments { + ["v1", "sandboxes"] | ["v1", "sandboxes", _, "token"] => RouteAuth::Management, + ["v1", "sandboxes", id, ..] => RouteAuth::Sandbox(id), + _ => RouteAuth::Unknown, + } +} + +/// Decide what a credential presented on a scoped route may address. +fn classify_scoped_token(provided: &str, sandbox_token: &str, host: &Auth, compat: bool) -> Option { + if ct_eq(provided, sandbox_token) { + return Some(Scope::Sandbox); + } + if compat && host.accepts(Some(provided)) { + return Some(Scope::Host); + } + None +} + +fn authorize(state: &State, provided: Option<&str>, segments: &[&str]) -> Option { + if !state.host_mode { + return state.auth.accepts(provided).then_some(Scope::Host); + } + match host_route_auth(segments) { + RouteAuth::Management => state.auth.accepts(provided).then_some(Scope::Host), + RouteAuth::Sandbox(id) => { + let entry = state.sandboxes.get(id)?; + classify_scoped_token(provided?, &entry.token, &state.auth, state.compat_host_token) + } + // Not a host-mode route. Whether it *exists* is the route gate's + // question, not authorization's: check the host credential so a valid + // caller gets an accurate 404 while an invalid one still gets 403. + RouteAuth::Unknown => state.auth.accepts(provided).then_some(Scope::Host), + } } /// Whether a route exists in this server mode. @@ -115,9 +185,10 @@ fn route( ); } + let segments: Vec<&str> = path.trim_matches('/').split('/').collect(); // Authentication first, so an unauthenticated caller learns nothing about // which routes this server serves. - if !authorized(state, request) { + if authorize(state, request.header("x-sandbox-token"), &segments).is_none() { return resp.error(403, "invalid or missing X-Sandbox-Token"); } if request.header("transfer-encoding").is_some() { @@ -125,7 +196,6 @@ fn route( } state.last_activity_ms.store(now_ms(), Ordering::Relaxed); - let segments: Vec<&str> = path.trim_matches('/').split('/').collect(); if !route_mode_allowed(state.host_mode, &segments) { return resp.error(404, &format!("no route in this server mode: {method} {path}")); } @@ -151,6 +221,12 @@ fn route( ("POST", ["v1", "sandboxes"]) => sandboxes::handle_create(state, request, reader, resp), ("GET", ["v1", "sandboxes"]) => resp.json(200, &state.sandboxes.list()), ("DELETE", ["v1", "sandboxes"]) => sandboxes::handle_delete_all(state, resp), + // Recover a sandbox's capability token with the host token, so a client + // that reconnects to an existing sandbox does not need local state. + ("GET", ["v1", "sandboxes", id, "token"]) => match state.sandboxes.get(id) { + Some(entry) => resp.json(200, &serde_json::json!({"id": entry.id, "token": entry.token})), + None => resp.error(404, &format!("no such sandbox: {id}")), + }, ("DELETE", ["v1", "sandboxes", id]) => { let id = id.to_string(); sandboxes::handle_delete(state, &id, resp) @@ -256,6 +332,9 @@ fn main() { let capacity = std::env::var("SBX_CAPACITY").ok().and_then(|v| v.parse().ok()).unwrap_or(usize::MAX); // Host mode multiplexes many sandboxes; dedicated mode is one sandbox == the job. let host_mode = std::env::var("SBX_HOST_MODE").map(|v| v == "1").unwrap_or(false); + // Transitional: accept the host token on per-sandbox routes for clients that + // predate per-sandbox tokens. Set to 0 to require scoped tokens. + let compat_host_token = std::env::var("SBX_COMPAT_HOST_TOKEN").map(|v| v != "0").unwrap_or(true); let state = Arc::new(State { auth, @@ -264,6 +343,7 @@ fn main() { procs: exec::ProcRegistry::default(), sandboxes: sandboxes::SandboxRegistry::with_capacity(capacity), host_mode, + compat_host_token, }); // Idle watchdog: stop billing for an abandoned sandbox/host before the job timeout. @@ -313,10 +393,11 @@ fn main() { // Mode and auth state on the first line: a server that silently serves the // wrong surface, or no authentication at all, is the hazard worth seeing. eprintln!( - "sbx-server {VERSION} listening on 0.0.0.0:{port} (mode: {}, auth: {}, landlock: {})", + "sbx-server {VERSION} listening on 0.0.0.0:{port} (mode: {}, auth: {}, landlock: {}{})", if host_mode { "host" } else { "dedicated" }, if matches!(state.auth, Auth::Required(_)) { "required" } else { "DISABLED" }, - if landlock_ok { "enabled" } else { "UNAVAILABLE — uid isolation only" } + if landlock_ok { "enabled" } else { "UNAVAILABLE — uid isolation only" }, + if host_mode && compat_host_token { ", host-token compat: on" } else { "" } ); // Host mode reuses one set of system-dir fds across every sandbox ruleset. // Open them now so the cost (and any missing-dir surface) lands at startup @@ -408,4 +489,71 @@ mod tests { assert_eq!(ct_eq(a, b), a == b, "ct_eq({a:?}, {b:?})"); } } + + /// Only the sandbox's own capability token addresses it. A sibling's token + /// is refused even though it is a perfectly valid credential elsewhere -- + /// this is the property the change exists for. + #[test] + fn a_sandbox_token_addresses_only_its_own_sandbox() { + let host = Auth::Required("host-management-token".to_string()); + let mine = "sandbox-a-token"; + let theirs = "sandbox-b-token"; + + assert_eq!(classify_scoped_token(mine, mine, &host, false), Some(Scope::Sandbox)); + assert_eq!(classify_scoped_token(theirs, mine, &host, false), None, "a sibling's token must not work"); + assert_eq!(classify_scoped_token("", mine, &host, false), None); + assert_eq!(classify_scoped_token("sandbox-a-toke", mine, &host, false), None, "a prefix must not work"); + } + + #[test] + fn the_host_token_on_a_scoped_route_depends_on_the_compat_window() { + let host = Auth::Required("host-management-token".to_string()); + let sandbox = "sandbox-a-token"; + + // Transitional: a management credential may address its sandboxes. + assert_eq!( + classify_scoped_token("host-management-token", sandbox, &host, true), + Some(Scope::Host) + ); + // With the window closed, scoped routes require scoped tokens. + assert_eq!(classify_scoped_token("host-management-token", sandbox, &host, false), None); + // Either way it is still recognised as the host credential, never as the sandbox's. + assert_ne!( + classify_scoped_token("host-management-token", sandbox, &host, true), + Some(Scope::Sandbox) + ); + } + + #[test] + fn pool_lifecycle_and_token_recovery_are_management_routes() { + let management = ["/v1/sandboxes"]; + for path in management { + assert!( + matches!(host_route_auth(&segments(path)), RouteAuth::Management), + "{path} must require the host token" + ); + } + assert!(matches!(host_route_auth(&segments("/v1/sandboxes/abc/token")), RouteAuth::Management)); + + // Everything else scoped under a sandbox id belongs to that sandbox. + for path in [ + "/v1/sandboxes/abc", + "/v1/sandboxes/abc/exec", + "/v1/sandboxes/abc/processes", + "/v1/sandboxes/abc/files/read", + "/v1/sandboxes/abc/proxy/8000/ws", + ] { + match host_route_auth(&segments(path)) { + RouteAuth::Sandbox(id) => assert_eq!(id, "abc", "{path}"), + _ => panic!("{path} should be scoped to a sandbox"), + } + } + } + + /// Token recovery must be management-gated: if a sandbox's own token could + /// read `/v1/sandboxes//token`, the whole scoping would be moot. + #[test] + fn token_recovery_is_not_reachable_with_a_sandbox_token() { + assert!(matches!(host_route_auth(&segments("/v1/sandboxes/victim/token")), RouteAuth::Management)); + } } diff --git a/src/proxy.rs b/src/proxy.rs index 02a9313..bf76900 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -289,6 +289,7 @@ mod tests { env: HashMap::new(), max_procs: 16, max_mem_mb: 16, + token: "sandbox-capability-token".to_string(), landlock_fd: -1, last_activity_ms: AtomicI64::new(0), idle_timeout_ms: 0, diff --git a/src/sandboxes.rs b/src/sandboxes.rs index 88e7394..4733df8 100644 --- a/src/sandboxes.rs +++ b/src/sandboxes.rs @@ -41,6 +41,14 @@ pub struct SandboxEntry { pub env: HashMap, pub max_procs: u64, pub max_mem_mb: u64, + /// Capability token for this sandbox alone. + /// + /// The host token (`SBX_TOKEN`) is a management credential: it creates, + /// lists and deletes sandboxes, and it can address any of them. This one is + /// bound to a single sandbox, so it can be handed to whoever operates that + /// sandbox — including into a browser or WebSocket client via the port + /// proxy — without also conferring authority over its siblings or the host. + pub token: String, /// Landlock ruleset fd confining this sandbox (-1 if Landlock unavailable). pub landlock_fd: i32, /// Last time a request targeted this sandbox; drives idle eviction. @@ -66,14 +74,16 @@ pub struct SandboxRegistry { reserved: AtomicUsize, } -fn random_id() -> String { - let mut buf = [0u8; 8]; - if std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut buf)).is_err() { - // /dev/urandom always exists on Linux; fallback just in case - let t = now_ms() as u64; - buf.copy_from_slice(&t.to_le_bytes()); - } - buf.iter().map(|b| format!("{b:02x}")).collect() +/// `n` bytes from the kernel CSPRNG, hex-encoded. +/// +/// Fails rather than substituting anything weaker: this produces both sandbox +/// ids (which name home directories and appear in URLs) and per-sandbox +/// capability tokens, so a degraded source here would be a predictable +/// credential, which is worse than a failed create. +fn random_hex(n: usize) -> std::io::Result { + let mut buf = vec![0u8; n]; + std::fs::File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut buf))?; + Ok(buf.iter().map(|b| format!("{b:02x}")).collect()) } impl SandboxRegistry { @@ -120,7 +130,8 @@ impl SandboxRegistry { format!("sandbox uid pool exhausted (max {} concurrent sandboxes per host)", UID_MAX - UID_BASE), )); } - let id = random_id(); + let id = random_hex(8)?; + let token = random_hex(32)?; let home = format!("{HOMES_DIR}/{id}"); std::fs::create_dir_all(&home)?; std::fs::set_permissions(&home, std::fs::Permissions::from_mode(0o700))?; @@ -147,6 +158,7 @@ impl SandboxRegistry { env, max_procs: max_procs.unwrap_or(DEFAULT_MAX_PROCS), max_mem_mb: max_mem_mb.unwrap_or(DEFAULT_MAX_MEM_MB), + token, landlock_fd, last_activity_ms: AtomicI64::new(now_ms()), idle_timeout_ms, @@ -353,7 +365,9 @@ pub fn handle_create( let mut rejected = 0usize; for _ in 0..count { match state.sandboxes.create(env.clone(), max_procs, max_mem_mb, idle_timeout_ms) { - Ok(entry) => created.push(serde_json::json!({"id": entry.id, "uid": entry.uid, "home": entry.home})), + Ok(entry) => created.push( + serde_json::json!({"id": entry.id, "token": entry.token, "uid": entry.uid, "home": entry.home}), + ), // Host full: report how many we couldn't place so the client packs them // onto another host (or boots a duplicate). Not an error. Err(CreateError::Full) => {