From 6c889d7019f80e38ea2fc70397963375d9b97e80 Mon Sep 17 00:00:00 2001 From: Sixing Lin <2682191498@qq.com> Date: Tue, 4 Aug 2026 21:20:42 +0800 Subject: [PATCH 1/4] feat(sandbox): generate envd access token for secure sandboxes --- Cargo.lock | 2 + Cargo.toml | 2 + config/default.toml | 5 + crates/aenv/src/client/files.rs | 24 +++- crates/aenv/src/client/mod.rs | 8 +- crates/aenv/src/client/sandboxes.rs | 49 +++++--- crates/aenv/src/commands/connect.rs | 19 +-- crates/aenv/src/commands/exec.rs | 3 +- crates/aenv/src/commands/start.rs | 30 +++-- crates/aenv/src/grpc/mod.rs | 40 +++++- docs/src/concepts/sandboxes.md | 2 + docs/src/configuration/env-vars.md | 1 + docs/src/configuration/reference.md | 10 ++ docs/src/getting-started/aenv-cli.md | 2 + src/api/generated/src/models.rs | 15 +++ src/api/impls/sandbox.rs | 12 ++ src/api/openapi.yml | 3 + src/api/proxy.rs | 130 ++++++++++++++++++++ src/bin/server.rs | 2 + src/cfg.rs | 66 ++++++++++ src/orchestrator/launch_plan.rs | 7 +- src/orchestrator/persistence/file_backed.rs | 1 + src/orchestrator/service.rs | 105 +++++++++++++--- src/orchestrator/store/metadata.rs | 5 + src/orchestrator/tests.rs | 34 ++++- src/orchestrator/types.rs | 1 + src/sandbox/access.rs | 98 +++++++++++++++ src/sandbox/backend.rs | 13 +- src/sandbox/envd.rs | 76 ++++++++++-- src/sandbox/firecracker/config.rs | 11 +- src/sandbox/firecracker/factory.rs | 16 ++- src/sandbox/firecracker/mmds.rs | 28 ++++- src/sandbox/firecracker/sandbox.rs | 120 ++++++++++++++---- src/sandbox/mock.rs | 12 +- src/sandbox/mod.rs | 8 +- tests/common/mod.rs | 4 + tests/integration/orchestrator.rs | 81 ++++++++++++ tests/integration/snapshot.rs | 2 + thirdparty/envd/src/lib.rs | 7 +- thirdparty/envd/src/transport.rs | 17 ++- thirdparty/envd/tests/filesystem_test.rs | 2 +- thirdparty/envd/tests/process_test.rs | 6 +- 42 files changed, 953 insertions(+), 126 deletions(-) create mode 100644 src/sandbox/access.rs diff --git a/Cargo.lock b/Cargo.lock index 04075b60..bead1f50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,8 @@ dependencies = [ "flate2", "futures", "headers", + "hex", + "hmac", "http 1.4.0", "http-body-util", "hyper 1.9.0", diff --git a/Cargo.toml b/Cargo.toml index 44ba9361..62083f75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,8 @@ dashmap = "6.1.0" io-uring = "0.7.9" tokio-tungstenite = "0.28" sha2 = "0.10" +hmac = "0.12" +hex = "0.4" semver = "1" iroh = { version = "=1.0.0-rc.0" } iroh-blobs = { version = "=0.101.0" } diff --git a/config/default.toml b/config/default.toml index e6cd5065..795e48e8 100644 --- a/config/default.toml +++ b/config/default.toml @@ -187,6 +187,11 @@ init_timeout_secs = 60 # envd readiness polling interval in milliseconds. poll_ms = 3 +[sandbox] +# Shared secret used to derive per-sandbox envd access tokens. All nodes in a +# cluster must use the same value. Prefer AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED. +# access_token_hash_seed = "replace-with-a-secret" + [orchestrator] # Expired-sandbox eviction interval in milliseconds (1000 = 1 second). auto_evict_interval_ms = 1000 diff --git a/crates/aenv/src/client/files.rs b/crates/aenv/src/client/files.rs index 83042e0b..ac5173ae 100644 --- a/crates/aenv/src/client/files.rs +++ b/crates/aenv/src/client/files.rs @@ -20,6 +20,7 @@ use crate::progress::TransferProgress; const API_KEY_HEADER: &str = "X-API-Key"; const SANDBOX_ID_HEADER: &str = "x-agentenv-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; +const ACCESS_TOKEN_HEADER: &str = "X-Access-Token"; const FILESYSTEM_SERVICE: &str = "filesystem.Filesystem"; const MAX_ERROR_BODY_BYTES: usize = 64 * 1024; pub(crate) const TRANSFER_IDLE_TIMEOUT: Duration = Duration::from_secs(60); @@ -31,7 +32,12 @@ pub struct EnvdFilesClient { } impl EnvdFilesClient { - fn new(base_url: &str, api_key: &str, sandbox_id: &str) -> Result { + fn new( + base_url: &str, + api_key: &str, + sandbox_id: &str, + envd_access_token: Option<&str>, + ) -> Result { let mut headers = HeaderMap::new(); headers.insert( API_KEY_HEADER, @@ -42,6 +48,12 @@ impl EnvdFilesClient { HeaderValue::from_str(sandbox_id).context("invalid sandbox ID header value")?, ); headers.insert(TARGET_PORT_HEADER, HeaderValue::from_static(ENVD_PORT_STR)); + if let Some(token) = envd_access_token { + let mut token = + HeaderValue::from_str(token).context("invalid envd access token header value")?; + token.set_sensitive(true); + headers.insert(ACCESS_TOKEN_HEADER, token); + } headers.insert( USER_AGENT, HeaderValue::from_str(&format!("aenv/{}", env!("CARGO_PKG_VERSION"))) @@ -60,7 +72,7 @@ impl EnvdFilesClient { Ok(Self { base_url: base_url.trim_end_matches('/').to_string(), http: client, - transport: Transport::new(base_url, api_key, sandbox_id)?, + transport: Transport::new(base_url, api_key, sandbox_id, envd_access_token)?, }) } @@ -298,7 +310,13 @@ fn format_envd_response_error(status: reqwest::StatusCode, content: &str) -> any impl Client { pub fn files(&self, sandbox_id: &str) -> Result { - EnvdFilesClient::new(&self.base, &self.api_key, sandbox_id) + let sandbox = self.get_sandbox(sandbox_id)?; + EnvdFilesClient::new( + &self.base, + &self.api_key, + sandbox_id, + sandbox.envd_access_token.as_deref(), + ) } } diff --git a/crates/aenv/src/client/mod.rs b/crates/aenv/src/client/mod.rs index 75e29089..55dba491 100644 --- a/crates/aenv/src/client/mod.rs +++ b/crates/aenv/src/client/mod.rs @@ -35,8 +35,12 @@ impl Client { }) } - pub fn transport(&self, sandbox_id: &str) -> Result { - Transport::new(&self.base, &self.api_key, sandbox_id) + pub fn transport( + &self, + sandbox_id: &str, + envd_access_token: Option<&str>, + ) -> Result { + Transport::new(&self.base, &self.api_key, sandbox_id, envd_access_token) } fn url(&self, path: &str) -> String { diff --git a/crates/aenv/src/client/sandboxes.rs b/crates/aenv/src/client/sandboxes.rs index 50bd64b4..5e321837 100644 --- a/crates/aenv/src/client/sandboxes.rs +++ b/crates/aenv/src/client/sandboxes.rs @@ -10,6 +10,8 @@ pub struct NewSandbox<'a> { pub template_id: &'a str, #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub secure: Option, } #[derive(Debug, Serialize)] @@ -23,12 +25,16 @@ pub struct NewColdSandbox<'a> { pub memory_mb: Option, #[serde(skip_serializing_if = "Option::is_none", rename = "diskSizeMB")] pub disk_size_mb: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub secure: Option, } -#[derive(Debug, Deserialize)] +#[derive(Deserialize)] pub struct Sandbox { #[serde(rename = "sandboxID")] pub sandbox_id: String, + #[serde(default, rename = "envdAccessToken")] + pub envd_access_token: Option, } #[derive(Debug, Serialize)] @@ -37,9 +43,11 @@ pub struct RefreshSandbox { pub duration: Option, } -#[derive(Debug, Deserialize)] +#[derive(Deserialize)] pub struct SandboxDetail { pub state: String, + #[serde(default, rename = "envdAccessToken")] + pub envd_access_token: Option, } #[derive(Debug, Deserialize, Serialize)] @@ -65,14 +73,20 @@ pub struct ListedSandbox { } impl Client { - pub fn create_sandbox(&self, template_id: &str, timeout: Option) -> Result { + pub fn create_sandbox( + &self, + template_id: &str, + timeout: Option, + secure: bool, + ) -> Result { let body = NewSandbox { template_id, timeout, + secure: secure.then_some(true), }; let resp = handle_status(self.post("/sandboxes").send_json(&body))?; let sandbox: Sandbox = resp.into_json()?; - Ok(sandbox.sandbox_id) + Ok(sandbox) } pub fn create_cold_sandbox( @@ -82,17 +96,19 @@ impl Client { cpu_count: Option, memory_mb: Option, disk_size_mb: Option, - ) -> Result { + secure: bool, + ) -> Result { let body = NewColdSandbox { image, timeout, cpu_count, memory_mb, disk_size_mb, + secure: secure.then_some(true), }; let resp = handle_status(self.post("/sandboxes-cold").send_json(&body))?; let sandbox: Sandbox = resp.into_json()?; - Ok(sandbox.sandbox_id) + Ok(sandbox) } pub fn list_sandboxes(&self) -> Result> { @@ -128,6 +144,11 @@ impl Client { Ok(Some(detail.state)) } + pub fn get_sandbox(&self, id: &str) -> Result { + let resp = handle_status(self.get(&format!("/sandboxes/{id}")).call())?; + Ok(resp.into_json()?) + } + /// `connect` resumes a paused sandbox or extends the TTL of a running one. pub fn connect_sandbox(&self, id: &str, timeout: u32) -> Result { let resp = handle_status( @@ -153,18 +174,6 @@ impl Client { )?; Ok(()) } - - pub async fn envd_ready_with_timeout( - &self, - sandbox_id: &str, - timeout: Duration, - ) -> Result { - let transport = self.transport(sandbox_id)?; - match tokio::time::timeout(timeout, transport.ready()).await { - Ok(Ok(())) => Ok(true), - Ok(Err(_)) | Err(_) => Ok(false), - } - } } #[cfg(test)] @@ -176,11 +185,13 @@ mod tests { let body = NewSandbox { template_id: "base-template", timeout: Some(300), + secure: Some(true), }; let value = serde_json::to_value(body).unwrap(); assert_eq!(value["templateID"], "base-template"); assert_eq!(value["timeout"], 300); + assert_eq!(value["secure"], true); assert!(value.get("cpuCount").is_none()); assert!(value.get("memoryMB").is_none()); } @@ -193,6 +204,7 @@ mod tests { cpu_count: Some(2), memory_mb: Some(1024), disk_size_mb: Some(8192), + secure: Some(true), }; let value = serde_json::to_value(body).unwrap(); @@ -201,6 +213,7 @@ mod tests { assert_eq!(value["cpuCount"], 2); assert_eq!(value["memoryMB"], 1024); assert_eq!(value["diskSizeMB"], 8192); + assert_eq!(value["secure"], true); assert!(value.get("templateID").is_none()); } diff --git a/crates/aenv/src/commands/connect.rs b/crates/aenv/src/commands/connect.rs index a6a64c42..ee95859d 100644 --- a/crates/aenv/src/commands/connect.rs +++ b/crates/aenv/src/commands/connect.rs @@ -51,10 +51,10 @@ pub fn run(args: Args) -> Result<()> { } pub(crate) async fn attach(client: &Client, sandbox_id: &str) -> Result { - client.connect_sandbox(sandbox_id, DEFAULT_TIMEOUT_SECS)?; + let sandbox = client.connect_sandbox(sandbox_id, DEFAULT_TIMEOUT_SECS)?; let (cols, rows) = terminal_size(); - let transport = Arc::new(client.transport(sandbox_id)?); + let transport = Arc::new(client.transport(sandbox_id, sandbox.envd_access_token.as_deref())?); let mut last_error = None; let mut started = None; for shell in ["/bin/bash", "/bin/sh"] { @@ -116,10 +116,11 @@ pub(crate) async fn attach(client: &Client, sandbox_id: &str) -> Result { reconnect_rx, reconnect_ack_tx, ); - let resize_task = resize_loop(transport, selector, state.clone()); + let resize_task = resize_loop(transport.clone(), selector, state.clone()); let watchdog_task = watchdog_loop( client.clone(), sandbox_id.to_string(), + transport, state.clone(), reconnect_tx, reconnect_ack_rx, @@ -681,6 +682,7 @@ async fn resize_loop( async fn watchdog_loop( client: Client, sandbox_id: String, + transport: Arc, state: SessionState, reconnect_tx: mpsc::UnboundedSender<()>, mut reconnect_ack_rx: mpsc::UnboundedReceiver<()>, @@ -722,7 +724,7 @@ async fn watchdog_loop( }); } - match envd_ready_probe(client.clone(), sandbox_id.clone()).await { + match envd_ready_probe(Arc::clone(&transport)).await { Ok(true) => { unhealthy_since = None; if matches!(previous_health, ProbeStatus::Unhealthy) { @@ -809,10 +811,11 @@ fn request_reconnect_once( *recovered_health = false; } -async fn envd_ready_probe(client: Client, sandbox_id: String) -> Result { - client - .envd_ready_with_timeout(&sandbox_id, RECONNECT_PROBE_TIMEOUT) - .await +async fn envd_ready_probe(transport: Arc) -> Result { + match tokio::time::timeout(RECONNECT_PROBE_TIMEOUT, transport.ready()).await { + Ok(Ok(())) => Ok(true), + Ok(Err(_)) | Err(_) => Ok(false), + } } fn should_refresh_session_keepalive(state: &SessionState, last_activity: &mut u64) -> bool { diff --git a/crates/aenv/src/commands/exec.rs b/crates/aenv/src/commands/exec.rs index ac02dfec..ecb32cac 100644 --- a/crates/aenv/src/commands/exec.rs +++ b/crates/aenv/src/commands/exec.rs @@ -27,7 +27,8 @@ async fn run_async(client: Client, args: Args) -> Result { .ok_or_else(|| anyhow::anyhow!("missing command"))?; let rest: Vec = cmd_iter.collect(); - let transport = client.transport(&args.sandbox_id)?; + let sandbox = client.get_sandbox(&args.sandbox_id)?; + let transport = client.transport(&args.sandbox_id, sandbox.envd_access_token.as_deref())?; let req = build_start_request(StartOpts { cmd: &cmd, args: rest, diff --git a/crates/aenv/src/commands/start.rs b/crates/aenv/src/commands/start.rs index 5cc5b92e..5fc71b5f 100644 --- a/crates/aenv/src/commands/start.rs +++ b/crates/aenv/src/commands/start.rs @@ -21,6 +21,9 @@ pub struct Args { /// Start directly from an external OCI image instead of a template/snapshot #[arg(long)] cold: bool, + /// Require an envd access token for sandbox control communication + #[arg(long)] + secure: bool, /// Sandbox TTL in seconds #[arg(long, default_value_t = super::DEFAULT_TIMEOUT_SECS)] timeout: u32, @@ -46,20 +49,22 @@ fn parse_disk_size_mb(value: &str) -> std::result::Result { pub fn run(args: Args) -> Result<()> { let client = Client::from_env()?; - let sandbox_id = if args.cold { + let sandbox = if args.cold { client.create_cold_sandbox( &args.target, Some(args.timeout), args.resources.cpu_count, args.resources.memory_mb, args.disk_size_mb, + args.secure, )? } else { if args.resources.is_set() || args.disk_size_mb.is_some() { anyhow::bail!("--cpu-count, --memory-mb, and --disk-size-mb require --cold"); } - client.create_sandbox(&args.target, Some(args.timeout))? + client.create_sandbox(&args.target, Some(args.timeout), args.secure)? }; + let sandbox_id = sandbox.sandbox_id; if args.detach { println!("{}", sandbox_id); @@ -68,18 +73,29 @@ pub fn run(args: Args) -> Result<()> { println!("Started sandbox {}", sandbox_id); let rt = super::tokio_rt()?; - rt.block_on(wait_for_envd(&client, &sandbox_id))?; + rt.block_on(wait_for_envd( + &client, + &sandbox_id, + sandbox.envd_access_token.as_deref(), + ))?; let code = rt.block_on(super::connect::attach(&client, &sandbox_id))?; std::process::exit(code); } -async fn wait_for_envd(client: &Client, sandbox_id: &str) -> Result<()> { +async fn wait_for_envd( + client: &Client, + sandbox_id: &str, + envd_access_token: Option<&str>, +) -> Result<()> { let deadline = Instant::now() + ENVD_READY_TIMEOUT; while Instant::now() < deadline { if matches!( - client - .envd_ready_with_timeout(sandbox_id, ENVD_READY_PROBE_TIMEOUT) - .await, + tokio::time::timeout( + ENVD_READY_PROBE_TIMEOUT, + client.transport(sandbox_id, envd_access_token)?.ready(), + ) + .await + .map(|result| result.is_ok()), Ok(true) ) { return Ok(()); diff --git a/crates/aenv/src/grpc/mod.rs b/crates/aenv/src/grpc/mod.rs index f5e12b98..64336c33 100644 --- a/crates/aenv/src/grpc/mod.rs +++ b/crates/aenv/src/grpc/mod.rs @@ -56,16 +56,23 @@ pub struct Transport { base_url: String, api_key: String, sandbox_id: String, + envd_access_token: Option, } impl Transport { - pub fn new(base_url: &str, api_key: &str, sandbox_id: &str) -> Result { + pub fn new( + base_url: &str, + api_key: &str, + sandbox_id: &str, + envd_access_token: Option<&str>, + ) -> Result { let http = Self::http_client(base_url).context("building Connect-RPC HTTP client")?; Ok(Self { http, base_url: base_url.trim_end_matches('/').to_string(), api_key: api_key.to_string(), sandbox_id: sandbox_id.to_string(), + envd_access_token: envd_access_token.map(str::to_owned), }) } @@ -94,11 +101,15 @@ impl Transport { } fn auth(&self, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - builder + let builder = builder .header("X-API-Key", &self.api_key) .header("x-agentenv-sandbox-id", &self.sandbox_id) .header("x-agentenv-target-port", ENVD_PORT_STR) - .header("Connect-Protocol-Version", "1") + .header("Connect-Protocol-Version", "1"); + match &self.envd_access_token { + Some(token) => builder.header("X-Access-Token", token), + None => builder, + } } fn unary_request( @@ -497,7 +508,7 @@ mod tests { #[test] fn unary_user_is_sent_as_basic_auth() { - let transport = Transport::new("http://127.0.0.1", "api-key", "sandbox-id").unwrap(); + let transport = Transport::new("http://127.0.0.1", "api-key", "sandbox-id", None).unwrap(); let request = transport .unary_request("filesystem.Filesystem", "Stat", Some("app")) .build() @@ -514,4 +525,25 @@ mod tests { .unwrap(); assert!(!request.headers().contains_key(AUTHORIZATION)); } + + #[test] + fn envd_access_token_is_sent_on_connect_requests() { + let transport = Transport::new( + "http://127.0.0.1", + "api-key", + "sandbox-id", + Some("envd-token"), + ) + .unwrap(); + + let request = transport + .unary_request("process.Process", "List", None) + .build() + .unwrap(); + + assert_eq!( + request.headers().get("x-access-token").unwrap(), + "envd-token" + ); + } } diff --git a/docs/src/concepts/sandboxes.md b/docs/src/concepts/sandboxes.md index d6783830..1df4cb8f 100644 --- a/docs/src/concepts/sandboxes.md +++ b/docs/src/concepts/sandboxes.md @@ -51,6 +51,8 @@ aenv start --cold ubuntu:24.04 The cold-start API accepts an optional `diskSizeMB` field to set the root filesystem's virtual size in MiB. Explicit values must be at least 1024 MiB and divisible by 1024 because the current resize tool operates at 1 GiB granularity. Growth is allowed by default; shrinking below the source image size requires `ublk.overlaybd.allow_shrink = true`. If omitted, the image's built-in virtual size is used. Resizing applies only when creating a fresh writable root filesystem, not to read-only images, images with an existing upper, or snapshot resume. Sandbox responses also report disk size as `diskSizeMB`. +Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port only; it does not add authentication to application ports. Each fork derives a distinct envd token from the child sandbox ID. + --- ## Working with Sandboxes diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index 92278dde..aed5ef3a 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -23,6 +23,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | required for normal server startup | Cluster-wide secret used to derive secure sandbox envd access tokens. Use the same value on every node. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index 5fa3da7d..9fef1513 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -251,6 +251,16 @@ In-guest `envd` daemon settings. | `init_timeout_secs` | integer | `60` | Max seconds to wait for envd to become ready after VM start | | `poll_ms` | integer | `3` | Poll interval (ms) for envd health check retries | +## `[sandbox]` + +Sandbox control communication settings. + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `access_token_hash_seed` | string | required | Cluster-wide secret used to derive secure sandbox envd access tokens. All nodes must use the same value. Prefer the `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` environment variable. Normal server startup fails when it is missing or empty; setup-only and host setup do not require it. | + +Changing the seed invalidates access tokens for existing secure sandboxes. Rotate it with a coordinated restart of every node. + ## `[orchestrator]` Sandbox lifecycle management. diff --git a/docs/src/getting-started/aenv-cli.md b/docs/src/getting-started/aenv-cli.md index cfdfebbc..7d1227ac 100644 --- a/docs/src/getting-started/aenv-cli.md +++ b/docs/src/getting-started/aenv-cli.md @@ -103,12 +103,14 @@ Start a sandbox and attach an interactive shell. `` is a template name o ```bash aenv start my-ubuntu +aenv start --secure my-ubuntu # require token-authenticated envd access aenv start --cold ubuntu:24.04 # start directly from an OCI image ``` | Flag | Description | |------|-------------| | `--cold` | Start directly from an external OCI image instead of a template | +| `--secure` | Require token authentication for envd control communication | | `--timeout ` | Sandbox TTL in seconds (default: 300) | | `--cpu-count ` / `--cpu` | CPU cores — only valid with `--cold` | | `--memory-mb ` / `--mem` | Memory in MiB — only valid with `--cold` | diff --git a/src/api/generated/src/models.rs b/src/api/generated/src/models.rs index 76c8704b..434da344 100644 --- a/src/api/generated/src/models.rs +++ b/src/api/generated/src/models.rs @@ -2519,6 +2519,11 @@ pub struct NewColdSandbox { #[serde(skip_serializing_if = "Option::is_none")] pub auto_resume: Option, + /// Secure all system communication with sandbox + #[serde(rename = "secure")] + #[serde(skip_serializing_if = "Option::is_none")] + pub secure: Option, + /// Allow sandbox to access the internet. When set to false, it behaves the same as specifying denyOut to 0.0.0.0/0 in the network config. #[serde(rename = "allowInternetAccess")] #[serde(skip_serializing_if = "Option::is_none")] @@ -2584,6 +2589,7 @@ impl NewColdSandbox { timeout: Some(15), auto_pause: Some(true), auto_resume: None, + secure: None, allow_internet_access: None, network: None, metadata: None, @@ -2613,6 +2619,9 @@ impl std::fmt::Display for NewColdSandbox { .as_ref() .map(|auto_pause| ["autoPause".to_string(), auto_pause.to_string()].join(",")), // Skipping autoResume in query parameter serialization + self.secure + .as_ref() + .map(|secure| ["secure".to_string(), secure.to_string()].join(",")), self.allow_internet_access .as_ref() .map(|allow_internet_access| { @@ -2668,6 +2677,7 @@ impl std::str::FromStr for NewColdSandbox { pub timeout: Vec, pub auto_pause: Vec, pub auto_resume: Vec, + pub secure: Vec, pub allow_internet_access: Vec, pub network: Vec, pub metadata: Vec>, @@ -2718,6 +2728,10 @@ impl std::str::FromStr for NewColdSandbox { .map_err(|x| x.to_string())?, ), #[allow(clippy::redundant_clone)] + "secure" => intermediate_rep.secure.push( + ::from_str(val).map_err(|x| x.to_string())?, + ), + #[allow(clippy::redundant_clone)] "allowInternetAccess" => intermediate_rep.allow_internet_access.push( ::from_str(val).map_err(|x| x.to_string())?, ), @@ -2788,6 +2802,7 @@ impl std::str::FromStr for NewColdSandbox { timeout: intermediate_rep.timeout.into_iter().next(), auto_pause: intermediate_rep.auto_pause.into_iter().next(), auto_resume: intermediate_rep.auto_resume.into_iter().next(), + secure: intermediate_rep.secure.into_iter().next(), allow_internet_access: intermediate_rep.allow_internet_access.into_iter().next(), network: intermediate_rep.network.into_iter().next(), metadata: intermediate_rep.metadata.into_iter().next(), diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 5e8812a9..5382dfb6 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -214,7 +214,12 @@ impl From for models::SandboxDetail { impl ApiImpl { fn sandbox_model(&self, metadata: SandboxMetadata) -> models::Sandbox { + let envd_access_token = self + .orchestrator + .get_envd_access_token(&metadata) + .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); + sandbox.envd_access_token = envd_access_token; sandbox.domain = self .sandbox_proxy_domains() .first() @@ -223,7 +228,12 @@ impl ApiImpl { } fn sandbox_detail_model(&self, metadata: SandboxMetadata) -> models::SandboxDetail { + let envd_access_token = self + .orchestrator + .get_envd_access_token(&metadata) + .map(|token| token.expose().to_owned()); let mut sandbox = models::SandboxDetail::from(metadata); + sandbox.envd_access_token = envd_access_token; sandbox.domain = self .sandbox_proxy_domains() .first() @@ -493,6 +503,7 @@ impl Sandboxes<()> for ApiImpl { .clone() .filter(|env_vars| !env_vars.is_empty()), network_policy, + secure: body.secure == Some(true), custom_extension_params: custom_params, }; @@ -625,6 +636,7 @@ impl Sandboxes<()> for ApiImpl { .clone() .filter(|env_vars| !env_vars.is_empty()), network_policy, + secure: body.secure == Some(true), custom_extension_params: custom_params, }; diff --git a/src/api/openapi.yml b/src/api/openapi.yml index 3b146348..32a4c74b 100644 --- a/src/api/openapi.yml +++ b/src/api/openapi.yml @@ -585,6 +585,9 @@ components: description: Automatically pauses the sandbox after the timeout autoResume: $ref: "#/components/schemas/SandboxAutoResumeConfig" + secure: + type: boolean + description: Secure all system communication with sandbox allowInternetAccess: type: boolean description: diff --git a/src/api/proxy.rs b/src/api/proxy.rs index b331c189..326c847f 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -83,6 +83,7 @@ const E2B_SANDBOX_ID_HEADER: &str = "e2b-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; /// E2B-compatible alias for the target port header. const E2B_TARGET_PORT_HEADER: &str = "e2b-sandbox-port"; +const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; #[cfg(test)] const PROXY_CONNECT_TIMEOUT: Duration = Duration::from_millis(100); @@ -679,6 +680,13 @@ async fn resolve_proxy_request( sandbox_id, ))); } + authorize_secure_envd_auto_resume( + api_impl, + sandbox_id, + target_port, + &parts.headers, + ) + .await?; try_auto_resume(api_impl, sandbox_id).await?; auto_resume_attempted = true; continue; @@ -741,6 +749,45 @@ async fn resolve_proxy_request( }) } +async fn authorize_secure_envd_auto_resume( + api_impl: &ApiImpl, + sandbox_id: SandboxId, + target_port: u16, + headers: &HeaderMap, +) -> Result<(), Response> { + if target_port != ConfigManager::global_config().tools.control_plane_port { + return Ok(()); + } + let metadata = api_impl + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? + .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; + if !metadata.secure { + return Ok(()); + } + let candidate = headers + .get(ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if api_impl + .orchestrator() + .validate_envd_access_token(sandbox_id, candidate) + { + return Ok(()); + } + + Err(Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ) + .body(Body::from("invalid or missing envd access token")) + .expect("static unauthorized proxy response is valid")) +} + async fn try_auto_resume(api_impl: &ApiImpl, sandbox_id: SandboxId) -> Result<(), Response> { match timeout( PROXY_AUTO_RESUME_TIMEOUT, @@ -1342,6 +1389,9 @@ mod tests { "e2b_sandbox_header_seen": headers.get(E2B_SANDBOX_ID_HEADER).is_some(), "target_port_header_seen": headers.get(TARGET_PORT_HEADER).is_some(), "e2b_target_port_header_seen": headers.get(E2B_TARGET_PORT_HEADER).is_some(), + "envd_access_token": headers + .get(ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()), "forwarded_host": headers .get("x-forwarded-host") .and_then(|value| value.to_str().ok()), @@ -1822,6 +1872,84 @@ mod tests { assert_eq!(body, Bytes::from_static(b"sandbox auto-resume failed")); } + #[tokio::test] + async fn paused_secure_envd_auto_resume_requires_valid_access_token() { + let sandbox_id = SandboxId::new(); + let api = build_api().await; + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Paused, + ) + .await; + api.orchestrator() + .set_auto_resume_for_test(&sandbox_id, true) + .await + .unwrap(); + api.orchestrator() + .set_secure_for_test(&sandbox_id, true) + .await + .unwrap(); + let metadata = api + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .unwrap() + .expect("paused sandbox metadata"); + let valid_token = api + .orchestrator() + .get_envd_access_token(&metadata) + .expect("secure paused sandbox token"); + let app = server::new(api); + + for token in [None, Some("incorrect")] { + let mut request = Request::builder() + .method(Method::GET) + .uri("/proxy/health") + .header("x-api-key", "test-key") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header( + TARGET_PORT_HEADER, + ConfigManager::global_config() + .tools + .control_plane_port + .to_string(), + ); + if let Some(token) = token { + request = request.header(ENVD_ACCESS_TOKEN_HEADER, token); + } + let response = app + .clone() + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + let response = app + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/proxy/health") + .header("x-api-key", "test-key") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header( + TARGET_PORT_HEADER, + ConfigManager::global_config() + .tools + .control_plane_port + .to_string(), + ) + .header(ENVD_ACCESS_TOKEN_HEADER, valid_token.expose()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + } + #[tokio::test] async fn proxy_returns_bad_gateway_for_running_sandbox_without_runtime_route() { let upstream_addr = start_upstream_server().await; @@ -1920,6 +2048,7 @@ mod tests { .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) + .header(ENVD_ACCESS_TOKEN_HEADER, "envd-token") .body(Body::empty()) .unwrap(), ) @@ -1936,6 +2065,7 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); + assert_eq!(payload["envd_access_token"], "envd-token"); assert_eq!(payload["forwarded_host"], "client.example"); } diff --git a/src/bin/server.rs b/src/bin/server.rs index 0140c8a1..5dca7ce7 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -66,6 +66,8 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } + config.sandbox.validate_access_token_hash_seed()?; + agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; diff --git a/src/cfg.rs b/src/cfg.rs index 51fd42de..d61f57a7 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -17,6 +17,9 @@ use crate::virtualization::VirtualizationMode; const ENV_CONFIG_PATH: &str = "AENV_CONFIG_PATH"; +#[cfg(test)] +const TEST_ACCESS_TOKEN_HASH_SEED: &str = "agentenv-unit-test-access-token-seed"; + #[derive(Debug, Deserialize)] struct SetupDependencyManifest { firecracker: ManifestVirtualizationDownloads, @@ -107,6 +110,8 @@ pub struct AppConfig { #[config(nested)] pub envd: EnvdConfig, #[config(nested)] + pub sandbox: SandboxConfig, + #[config(nested)] pub orchestrator: OrchestratorConfig, #[config(nested)] pub snapshot: SnapshotConfig, @@ -265,6 +270,42 @@ pub struct EnvdConfig { pub poll_ms: u64, } +#[derive(Clone, Config)] +pub struct SandboxConfig { + #[config( + env = "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED", + parse_env = parse_trimmed_string + )] + pub access_token_hash_seed: Option, +} + +impl std::fmt::Debug for SandboxConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SandboxConfig") + .field( + "access_token_hash_seed", + &self.access_token_hash_seed.as_ref().map(|_| ""), + ) + .finish() + } +} + +impl SandboxConfig { + pub(crate) fn access_token_hash_seed(&self) -> Result<&str> { + let seed = self.access_token_hash_seed.as_deref().ok_or_else(|| { + anyhow!("[sandbox].access_token_hash_seed must be configured and non-empty") + })?; + if seed.trim().is_empty() { + bail!("[sandbox].access_token_hash_seed must be configured and non-empty"); + } + Ok(seed) + } + + pub fn validate_access_token_hash_seed(&self) -> Result<()> { + self.access_token_hash_seed().map(|_| ()) + } +} + #[derive(Debug, Config, Clone)] pub struct MachineConfig { #[config(default = 2u32)] @@ -1113,6 +1154,16 @@ impl ConfigManager { } fn set_global(manager: Self) -> Result<&'static Self> { + #[cfg(test)] + let manager = { + let mut manager = manager; + manager + .config + .sandbox + .access_token_hash_seed + .get_or_insert_with(|| TEST_ACCESS_TOKEN_HASH_SEED.to_string()); + manager + }; let _ = GLOBAL_CONFIG_MANAGER.set(manager); GLOBAL_CONFIG_MANAGER .get() @@ -1293,6 +1344,21 @@ mod tests { Ok(()) } + #[test] + fn sandbox_access_token_seed_is_required_for_serving() { + let mut config = SandboxConfig { + access_token_hash_seed: None, + }; + assert!(config.validate_access_token_hash_seed().is_err()); + + config.access_token_hash_seed = Some(" ".to_string()); + assert!(config.validate_access_token_hash_seed().is_err()); + + config.access_token_hash_seed = Some("cluster-secret".to_string()); + assert!(config.validate_access_token_hash_seed().is_ok()); + assert!(!format!("{config:?}").contains("cluster-secret")); + } + #[test] fn validate_memory_snapshot_options_enforces_dirty_page_requirements() { let cases = [ diff --git a/src/orchestrator/launch_plan.rs b/src/orchestrator/launch_plan.rs index a6624a9b..aad66049 100644 --- a/src/orchestrator/launch_plan.rs +++ b/src/orchestrator/launch_plan.rs @@ -2,7 +2,9 @@ use std::sync::Arc; use super::store::{NewTimeout, SandboxMetadata}; use super::types::SandboxState; -use crate::sandbox::{FreshSandboxBuildSpec, PausedSandboxState, SandboxLaunchConfig}; +use crate::sandbox::{ + EnvdAccessToken, FreshSandboxBuildSpec, PausedSandboxState, SandboxLaunchConfig, +}; use crate::snapshot::RunnableSnapshot; use crate::types::{SandboxId, SandboxResources}; @@ -28,6 +30,7 @@ pub(super) struct ResumeLaunchPlan { pub paused_state: Arc, pub timeout: NewTimeout, pub resources: SandboxResources, + pub envd_access_token: Option, } pub(super) enum LaunchPlan { @@ -75,12 +78,14 @@ impl LaunchPlan { paused_state: Arc, timeout: NewTimeout, resources: SandboxResources, + envd_access_token: Option, ) -> Self { Self::Resume(Box::new(ResumeLaunchPlan { sandbox_id, paused_state, timeout, resources, + envd_access_token, })) } diff --git a/src/orchestrator/persistence/file_backed.rs b/src/orchestrator/persistence/file_backed.rs index 53082f66..0a2d2840 100644 --- a/src/orchestrator/persistence/file_backed.rs +++ b/src/orchestrator/persistence/file_backed.rs @@ -449,6 +449,7 @@ mod tests { &self, _sandbox_id: SandboxId, _state: &dyn PausedSandboxState, + _envd_access_token: Option, ) -> Result> { unreachable!("persister tests only decode state") } diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index d8d7c1cc..8b89e566 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -14,9 +14,10 @@ use crate::image::cache::{ local_image_services_from_global_config, RuntimeImageOwner, RuntimeImageRefs, }; use crate::sandbox::{ - CustomExtensionClient, CustomExtensionParams, FirecrackerSandboxFactory, FreshSandboxBuildSpec, - PausedSandboxState, RuntimeArtifactSet, SandboxBackend, SandboxBackendFactory, - SandboxLaunchConfig, SandboxNetworkPolicy, SandboxRuntimeInfo, + CustomExtensionClient, CustomExtensionParams, EnvdAccessToken, FirecrackerSandboxFactory, + FreshSandboxBuildSpec, PausedSandboxState, RuntimeArtifactSet, SandboxAccessTokenGenerator, + SandboxBackend, SandboxBackendFactory, SandboxForkSpec, SandboxLaunchConfig, + SandboxNetworkPolicy, SandboxRuntimeInfo, }; use crate::snapshot::SnapshotRuntimeVersions; use crate::types::{bytes_to_mib_ceil, SandboxId, SandboxResources}; @@ -104,6 +105,7 @@ pub struct Orchestrator< shutdown_tx: watch::Sender, shutdown_outcome: OnceCell, image_refs: Arc, + access_tokens: Arc, } impl Orchestrator { @@ -151,6 +153,9 @@ where image_refs: Arc, ) -> Result> { let app_config = ConfigManager::global_config(); + let access_tokens = Arc::new(SandboxAccessTokenGenerator::new( + app_config.sandbox.access_token_hash_seed()?, + )?); let config = &app_config.orchestrator; let (shutdown_tx, shutdown_rx) = watch::channel(false); let (sandbox_event_tx, _sandbox_event_rx) = @@ -187,6 +192,7 @@ where shutdown_tx, shutdown_outcome: OnceCell::new(), image_refs, + access_tokens, }); // Start the auto-evict task. @@ -353,7 +359,9 @@ where auto_resume, network_policy, custom_extension_params, + secure, } = request; + let envd_access_token = secure.then(|| self.access_tokens.generate(sandbox_id)); info!(timeout = ?timeout, "creating sandbox"); let result = match source { @@ -370,6 +378,10 @@ where }); } let launch_image_configs = committed.image_configs.clone(); + let mut extra_mmds = serde_json::Map::new(); + if !launch_image_configs.is_empty() { + extra_mmds.insert("imageConfigs".to_string(), launch_image_configs.to_value()); + }; // Effective custom config: a launch-provided value overrides the // one persisted in the source snapshot; otherwise inherit it. // Store the effective value so publishing a snapshot from this @@ -377,11 +389,15 @@ where let effective_custom_extension_params = custom_extension_params .clone() .or_else(|| committed.custom_extension_params.clone()); - let mut launch_config = SandboxLaunchConfig::new(sandbox_id, record.id.to_string()); - launch_config.env_vars = env_vars; - launch_config.network = network_policy.runtime_policy(); - launch_config.custom_extension_params = effective_custom_extension_params.clone(); - let launch_config = launch_config.with_image_configs(&launch_image_configs); + let launch_config = SandboxLaunchConfig { + sandbox_id, + snapshot_id: record.id.to_string(), + env_vars, + network: network_policy.runtime_policy(), + extra_mmds, + custom_extension_params: effective_custom_extension_params.clone(), + envd_access_token: envd_access_token.clone(), + }; let transitional_metadata = SandboxMetadata { id: sandbox_id, @@ -398,6 +414,7 @@ where user_metadata, network_policy, custom_extension_params: effective_custom_extension_params, + secure, ..Default::default() }; @@ -422,11 +439,19 @@ where let context = *context; let resources = resources.unwrap_or_else(default_fresh_sandbox_resources); let launch_image_configs = *image_configs; - let mut launch_config = SandboxLaunchConfig::new(sandbox_id, image_ref.clone()); - launch_config.env_vars = env_vars; - launch_config.network = network_policy.runtime_policy(); - launch_config.custom_extension_params = custom_extension_params.clone(); - let launch_config = launch_config.with_image_configs(&launch_image_configs); + let mut extra_mmds = serde_json::Map::new(); + if !launch_image_configs.is_empty() { + extra_mmds.insert("imageConfigs".to_string(), launch_image_configs.to_value()); + }; + let launch_config = SandboxLaunchConfig { + sandbox_id, + snapshot_id: image_ref.clone(), + env_vars, + network: network_policy.runtime_policy(), + extra_mmds, + custom_extension_params: custom_extension_params.clone(), + envd_access_token, + }; let build_spec = FreshSandboxBuildSpec { image_config_path: overlaybd_config_path, context: context.clone(), @@ -449,6 +474,7 @@ where user_metadata, network_policy, custom_extension_params, + secure, ..Default::default() }; @@ -534,13 +560,23 @@ where })? .previous; - let child_ids = (0..count).map(|_| SandboxId::new()).collect::>(); + let children_spec = (0..count) + .map(|_| { + let sandbox_id = SandboxId::new(); + SandboxForkSpec { + sandbox_id, + envd_access_token: source_metadata + .secure + .then(|| self.access_tokens.generate(sandbox_id)), + } + }) + .collect::>(); // Start to fork the sandbox. // This is a single operation that will return a list of results for each child sandbox. let fork_result = { let mut sandbox = source_handle.lock().await; - sandbox.fork(&child_ids).await + sandbox.fork(&children_spec).await }; let forked_backends = match fork_result { Ok(forked_backends) => forked_backends, @@ -587,10 +623,11 @@ where } // Register each forked sandbox in the store and runtime, and publish events. - let mut outcomes = Vec::with_capacity(child_ids.len()); + let mut outcomes = Vec::with_capacity(children_spec.len()); let mut successes = 0u64; let now = SystemTime::now(); - for (sandbox_id, backend) in child_ids.into_iter().zip(forked_backends) { + for (child, backend) in children_spec.into_iter().zip(forked_backends) { + let sandbox_id = child.sandbox_id; let backend = match backend { Ok(backend) => backend, Err(err) => { @@ -690,6 +727,16 @@ where Ok(self.store.list_filtered(filter).await?) } + pub fn get_envd_access_token(&self, metadata: &SandboxMetadata) -> Option { + metadata + .secure + .then(|| self.access_tokens.generate(metadata.id)) + } + + pub fn validate_envd_access_token(&self, sandbox_id: SandboxId, candidate: &str) -> bool { + self.access_tokens.matches(sandbox_id, candidate) + } + /// Resolves the current proxyability of a sandbox without touching the sandbox mutex. #[tracing::instrument(skip(self), fields(sandbox_id = %sandbox_id))] pub async fn proxy_lookup_for(&self, sandbox_id: &SandboxId) -> Result { @@ -1331,6 +1378,9 @@ where Arc::clone(paused_state), timeout, metadata.resources, + metadata + .secure + .then(|| self.access_tokens.generate(metadata.id)), )) .await; if let Ok(metadata) = resumed.as_ref() { @@ -2088,9 +2138,11 @@ where .factory .build((**build_spec).clone(), plan.launch_config.clone()), }, - LaunchPlan::Resume(plan) => self - .factory - .build_from_paused_state(plan.sandbox_id, plan.paused_state.as_ref()), + LaunchPlan::Resume(plan) => self.factory.build_from_paused_state( + plan.sandbox_id, + plan.paused_state.as_ref(), + plan.envd_access_token.clone(), + ), }; build_result.map_err(|source| { warn!(error = %format_args!("{source:#}"), "failed to build sandbox"); @@ -2457,6 +2509,19 @@ where Ok(()) } + pub(crate) async fn set_secure_for_test( + &self, + sandbox_id: &SandboxId, + secure: bool, + ) -> Result<()> { + let Some(mut metadata) = self.store.get(sandbox_id).await? else { + return Err(OrchestratorError::SandboxNotFound(*sandbox_id)); + }; + metadata.secure = secure; + self.store.update(metadata).await?; + Ok(()) + } + pub(crate) async fn remove_proxy_route_for_test(&self, sandbox_id: &SandboxId) { let _ = self.proxy_routes.write().await.remove(sandbox_id); } diff --git a/src/orchestrator/store/metadata.rs b/src/orchestrator/store/metadata.rs index 891335da..1384cb7a 100644 --- a/src/orchestrator/store/metadata.rs +++ b/src/orchestrator/store/metadata.rs @@ -54,6 +54,10 @@ pub struct SandboxMetadata { /// unless overridden at create time. #[serde(default, skip_serializing_if = "Option::is_none")] pub custom_extension_params: Option, + /// Whether envd requires the access token derived from this sandbox's ID. + /// Older records deserialize as non-secure sandboxes. + #[serde(default)] + pub secure: bool, /// Paused state produced by the sandbox backend during `pause`. /// Passed back to the backend factory when `resume_sandbox` is called. #[serde(skip)] @@ -86,6 +90,7 @@ impl Default for SandboxMetadata { user_metadata: None, network_policy: SandboxNetworkPolicy::default(), custom_extension_params: None, + secure: false, paused_state: None, } } diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index 9de5ad4c..e933525a 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -50,6 +50,18 @@ fn test_runtime_image_refs() -> Arc { local_image_services_from_global_config().runtime_refs } +fn test_access_tokens() -> Arc { + Arc::new( + SandboxAccessTokenGenerator::new( + ConfigManager::global_config() + .sandbox + .access_token_hash_seed() + .unwrap(), + ) + .unwrap(), + ) +} + async fn make_orchestrator() -> Arc { Orchestrator::new_inner( InMemoryMetadataStore::new(), @@ -117,6 +129,7 @@ fn make_orchestrator_without_background_with_factory_and_persister< shutdown_tx: tokio::sync::watch::channel(false).0, shutdown_outcome: tokio::sync::OnceCell::new(), image_refs: test_runtime_image_refs(), + access_tokens: test_access_tokens(), }) } @@ -731,6 +744,7 @@ fn resume_launch_plan(sandbox_id: SandboxId) -> LaunchPlan { Arc::clone(test_paused_state()), NewTimeout::None, SandboxResources::default(), + None, ) } @@ -1075,6 +1089,7 @@ fn create_request( network_policy: SandboxNetworkPolicy::default(), custom_extension_params: None, auto_resume: false, + secure: false, } } @@ -1136,6 +1151,7 @@ async fn create_sandbox_from_image_uses_fresh_launch_metadata() -> Result<()> { network_policy: SandboxNetworkPolicy::default(), custom_extension_params: None, auto_resume: false, + secure: false, }) .await?; @@ -4435,9 +4451,13 @@ async fn fork_sandbox_creates_running_children_from_one_source() -> Result<()> { let orchestrator = make_orchestrator_with_factory(MockBackendFactory::with_behavior(Arc::clone(&behavior))) .await; - let source = orchestrator - .create_sandbox(create_request(Some(60), &[("team", "batch-fork-source")])) - .await?; + let mut request = create_request(Some(60), &[("team", "batch-fork-source")]); + request.secure = true; + let source = orchestrator.create_sandbox(request).await?; + assert!(source.secure); + let source_token = orchestrator + .get_envd_access_token(&source) + .expect("secure source has a token"); behavior.push_action( MockOperation::Build, MockAction::Fail { @@ -4451,12 +4471,20 @@ async fn fork_sandbox_creates_running_children_from_one_source() -> Result<()> { let children = outcomes.into_iter().collect::, _>>()?; assert_eq!(children.len(), 3); + let mut child_tokens = Vec::with_capacity(children.len()); for child in &children { assert_ne!(child.id, source.id); assert_eq!(child.state, SandboxState::Running); assert_eq!(child.snapshot_id, source.snapshot_id); assert_eq!(child.user_metadata, source.user_metadata); assert_eq!(child.timeout, source.timeout); + assert!(child.secure); + let child_token = orchestrator + .get_envd_access_token(child) + .expect("secure child has a token"); + assert_ne!(child_token, source_token); + assert!(!child_tokens.contains(&child_token)); + child_tokens.push(child_token); assert_proxy_ready(&orchestrator, &child.id).await?; } let source_after = orchestrator diff --git a/src/orchestrator/types.rs b/src/orchestrator/types.rs index bbfbc060..8dcaafd6 100644 --- a/src/orchestrator/types.rs +++ b/src/orchestrator/types.rs @@ -33,6 +33,7 @@ pub struct CreateSandboxRequest { pub user_metadata: Option>, pub env_vars: Option>, pub network_policy: crate::sandbox::SandboxNetworkPolicy, + pub secure: bool, /// Opaque user-provided JSON passed through to the custom extension hooks. pub custom_extension_params: Option, } diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs new file mode 100644 index 00000000..1e39926d --- /dev/null +++ b/src/sandbox/access.rs @@ -0,0 +1,98 @@ +use std::fmt; + +use anyhow::{bail, Result}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use crate::types::SandboxId; + +type HmacSha256 = Hmac; + +#[derive(Clone, PartialEq, Eq)] +pub struct EnvdAccessToken(String); + +impl EnvdAccessToken { + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for EnvdAccessToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("EnvdAccessToken()") + } +} + +#[derive(Clone)] +pub struct SandboxAccessTokenGenerator { + seed: Vec, +} + +impl SandboxAccessTokenGenerator { + pub fn new(seed: &str) -> Result { + let seed = seed.trim(); + if seed.is_empty() { + bail!("[sandbox].access_token_hash_seed must be configured and non-empty"); + } + Ok(Self { + seed: seed.as_bytes().to_vec(), + }) + } + + pub fn generate(&self, subject: SandboxId) -> EnvdAccessToken { + let mut mac = + HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); + mac.update(subject.to_string().as_bytes()); + EnvdAccessToken(hex::encode(mac.finalize().into_bytes())) + } + + pub fn matches(&self, subject: SandboxId, candidate: &str) -> bool { + let mut candidate_bytes = [0_u8; 32]; + let decoded = hex::decode_to_slice(candidate, &mut candidate_bytes).is_ok(); + let mut mac = + HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); + mac.update(subject.to_string().as_bytes()); + mac.verify_slice(&candidate_bytes).is_ok() & decoded + } +} + +impl fmt::Debug for SandboxAccessTokenGenerator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SandboxAccessTokenGenerator()") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generates_lowercase_hex_hmac_sha256() { + let generator = SandboxAccessTokenGenerator::new("test-seed").unwrap(); + let subject = SandboxId::try_from("01936f8e-72f5-7000-8000-000000000001").unwrap(); + + let token = generator.generate(subject); + + assert_eq!(token.expose().len(), 64); + assert_eq!( + token.expose(), + "4f00f2a93a87c37161ae01c59b6d4f84506668113441277e9f6272dd4bfae1a7" + ); + assert!(token.expose().bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_eq!(token.expose(), token.expose().to_ascii_lowercase()); + assert!(generator.matches(subject, token.expose())); + assert!(!generator.matches(subject, "not-a-token")); + assert!(!generator.matches(subject, &"0".repeat(64))); + } + + #[test] + fn rejects_empty_seed_and_redacts_secrets() { + assert!(SandboxAccessTokenGenerator::new(" ").is_err()); + let generator = SandboxAccessTokenGenerator::new("super-secret").unwrap(); + let subject = SandboxId::default(); + let token = generator.generate(subject); + + assert!(!format!("{generator:?}").contains("super-secret")); + assert!(!format!("{token:?}").contains(token.expose())); + } +} diff --git a/src/sandbox/backend.rs b/src/sandbox/backend.rs index 37430872..26702917 100644 --- a/src/sandbox/backend.rs +++ b/src/sandbox/backend.rs @@ -14,7 +14,7 @@ use async_trait::async_trait; use serde_json::Value; use super::{ - Executor, FreshSandboxBuildSpec, ProcessHandle, ProcessOpts, ProcessOutput, + EnvdAccessToken, Executor, FreshSandboxBuildSpec, ProcessHandle, ProcessOpts, ProcessOutput, SandboxLaunchConfig, SandboxNetworkPolicy, }; use crate::sandbox::CustomExtensionParams; @@ -80,6 +80,12 @@ impl From for SandboxCaptureError { pub type SandboxCaptureResult = std::result::Result; pub type SandboxForkResult = anyhow::Result>; +#[derive(Clone, Debug)] +pub struct SandboxForkSpec { + pub sandbox_id: SandboxId, + pub envd_access_token: Option, +} + /// Opaque set of local runtime artifacts a sandbox needs while it is alive. /// /// Sandbox backends construct this from their runtime config, the orchestrator @@ -218,7 +224,7 @@ pub trait SandboxBackend: Send + 'static { /// /// The outer error is reserved for failures before child startup begins. /// After the source has been restored, implementations must attempt every - /// child concurrently and return one result per `child_ids` entry in the + /// child concurrently and return one result per `spec` entry in the /// same order. Successful children stay running when a sibling fails. /// /// [`SandboxCaptureError::Terminal`] indicates the fork attempt mutated the @@ -227,7 +233,7 @@ pub trait SandboxBackend: Send + 'static { /// recovery belong in the corresponding [`SandboxForkResult`]. async fn fork( &mut self, - child_ids: &[SandboxId], + spec: &[SandboxForkSpec], ) -> SandboxCaptureResult>; /// Stop the sandbox and release all associated system resources. @@ -287,6 +293,7 @@ pub trait SandboxBackendFactory: Send + Sync + 'static { &self, sandbox_id: crate::types::SandboxId, state: &dyn PausedSandboxState, + envd_access_token: Option, ) -> Result>; } diff --git a/src/sandbox/envd.rs b/src/sandbox/envd.rs index 0fecec65..ccc4af76 100644 --- a/src/sandbox/envd.rs +++ b/src/sandbox/envd.rs @@ -5,8 +5,12 @@ use anyhow::{anyhow, Context, Result}; use tokio::time::{sleep, Duration}; use tracing::{debug, trace}; +use crate::sandbox::EnvdAccessToken; use envd::filesystem::FilesystemClient; -use envd::http_client::apis::{configuration::Configuration, default_api}; +use envd::http_client::apis::{ + configuration::{ApiKey, Configuration}, + default_api, +}; use envd::http_client::models::InitPostRequest; use envd::process::ProcessClient; use envd::reqwest::Client; @@ -25,10 +29,11 @@ static ENVD_BOOTSTRAP_HTTP_CLIENT: LazyLock = LazyLock::new(|| { pub(crate) struct EnvdInstance { config: Configuration, grpc_address: String, + access_token: Option, } impl EnvdInstance { - pub(crate) fn new(base_path: String) -> Self { + pub(crate) fn new(base_path: String, access_token: Option) -> Self { let grpc_address = base_path.clone(); Self { // Share client configuration without retaining bootstrap TCP @@ -40,9 +45,13 @@ impl EnvdInstance { basic_auth: None, oauth_access_token: None, bearer_access_token: None, - api_key: None, + api_key: access_token.as_ref().map(|token| ApiKey { + prefix: None, + key: token.expose().to_owned(), + }), }, grpc_address, + access_token, } } @@ -50,9 +59,12 @@ impl EnvdInstance { #[tracing::instrument(skip(self), fields(grpc_address = %self.grpc_address))] pub(crate) async fn process_client(&self) -> Result { trace!(grpc_address = %self.grpc_address, "connecting envd process client"); - let client = ProcessClient::connect(&self.grpc_address) - .await - .context("failed to connect process client")?; + let client = ProcessClient::connect( + &self.grpc_address, + self.access_token.as_ref().map(EnvdAccessToken::expose), + ) + .await + .context("failed to connect process client")?; trace!("connected to envd process client"); Ok(client) } @@ -61,9 +73,12 @@ impl EnvdInstance { #[tracing::instrument(skip(self), fields(grpc_address = %self.grpc_address))] pub(crate) async fn filesystem_client(&self) -> Result { trace!(grpc_address = %self.grpc_address, "connecting envd filesystem client"); - let client = FilesystemClient::connect(&self.grpc_address) - .await - .context("failed to connect filesystem client")?; + let client = FilesystemClient::connect( + &self.grpc_address, + self.access_token.as_ref().map(EnvdAccessToken::expose), + ) + .await + .context("failed to connect filesystem client")?; trace!("connected to envd filesystem client"); Ok(client) } @@ -124,6 +139,10 @@ impl EnvdInstance { debug!(has_env_vars = env_vars.is_some(), "initializing envd"); let now = chrono::Utc::now().fixed_offset(); let init_post_request = InitPostRequest { + access_token: self + .access_token + .as_ref() + .map(|token| token.expose().to_owned()), env_vars, default_workdir, default_user, @@ -140,10 +159,47 @@ impl EnvdInstance { mod tests { use std::time::Instant; + use axum::extract::State; + use axum::http::{HeaderMap, StatusCode}; + use axum::routing::post; + use axum::{Json, Router}; + use serde_json::Value; use tokio::net::TcpListener; + use tokio::sync::mpsc; use super::*; + async fn capture_init_request( + State(sender): State>, + headers: HeaderMap, + Json(body): Json, + ) -> StatusCode { + sender.send((headers, body)).await.unwrap(); + StatusCode::NO_CONTENT + } + + #[tokio::test] + async fn init_sends_access_token_in_header_and_body() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let (sender, mut receiver) = mpsc::channel(1); + let app = Router::new() + .route("/init", post(capture_init_request)) + .with_state(sender); + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + let token = crate::sandbox::SandboxAccessTokenGenerator::new("envd-init-test-seed")? + .generate(crate::types::SandboxId::new()); + let envd = EnvdInstance::new(format!("http://{address}"), Some(token.clone())); + + envd.init(None, None, None).await?; + + let (headers, body) = receiver.recv().await.expect("captured init request"); + assert_eq!(headers["x-access-token"], token.expose()); + assert_eq!(body["accessToken"], token.expose()); + server.abort(); + Ok(()) + } + #[tokio::test] async fn readiness_deadline_bounds_a_hung_health_probe() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0").await?; @@ -154,7 +210,7 @@ mod tests { #[allow(unreachable_code)] Ok::<_, anyhow::Error>(()) }); - let envd = EnvdInstance::new(format!("http://{address}")); + let envd = EnvdInstance::new(format!("http://{address}"), None); let deadline = Duration::from_millis(50); let started = Instant::now(); diff --git a/src/sandbox/firecracker/config.rs b/src/sandbox/firecracker/config.rs index 66185307..28320caa 100644 --- a/src/sandbox/firecracker/config.rs +++ b/src/sandbox/firecracker/config.rs @@ -14,7 +14,9 @@ use crate::cfg::{AppConfig, ConfigManager, EnvdConfig, ToolsConfig}; use crate::sandbox::ublk::UblkConfig; use crate::sandbox::SandboxNetworkPolicy; use crate::sandbox::UblkBackend; -use crate::sandbox::{validate_drive_id, ExtraDrive, OverlaybdConfig, SandboxLaunchConfig}; +use crate::sandbox::{ + validate_drive_id, EnvdAccessToken, ExtraDrive, OverlaybdConfig, SandboxLaunchConfig, +}; use crate::snapshot::RunnableSnapshot; use anyhow::{bail, Context, Result}; use overlaybd::config::UpperMode; @@ -160,6 +162,10 @@ pub struct FirecrackerCommonConfig { /// the process-global config. #[serde(default)] pub disk_rate_limit: crate::cfg::DiskRateLimitConfig, + /// Runtime-only envd credential. It is re-derived from sandbox metadata on + /// resume and is intentionally excluded from persisted snapshot configs. + #[serde(skip)] + pub envd_access_token: Option, } impl FirecrackerCommonConfig { @@ -193,6 +199,7 @@ impl FirecrackerCommonConfig { network_policy: None, custom_extension_params: None, disk_rate_limit: crate::cfg::DiskRateLimitConfig::default(), + envd_access_token: None, } } @@ -407,10 +414,12 @@ impl FirecrackerSandboxConfig { } self.common.mmds_metadata = Some( MmdsMetadata::new(launch_config.sandbox_id, launch_config.snapshot_id.clone()) + .with_access_token(launch_config.envd_access_token.as_ref()) .with_extra(launch_config.extra_mmds.clone()), ); self.common.network_policy = launch_config.network.clone(); self.common.custom_extension_params = launch_config.custom_extension_params.clone(); + self.common.envd_access_token = launch_config.envd_access_token.clone(); self } diff --git a/src/sandbox/firecracker/factory.rs b/src/sandbox/firecracker/factory.rs index 5aff5508..4e1cca2f 100644 --- a/src/sandbox/firecracker/factory.rs +++ b/src/sandbox/firecracker/factory.rs @@ -14,7 +14,9 @@ use super::config::FirecrackerSandboxConfig; use super::sandbox::{FirecrackerPausedState, FirecrackerSandbox}; use crate::cfg::ConfigManager; use crate::sandbox::backend::{PausedSandboxState, SandboxBackend, SandboxBackendFactory}; -use crate::sandbox::{FreshSandboxBuildSpec, OverlaybdConfig, SandboxLaunchConfig, UblkConfig}; +use crate::sandbox::{ + EnvdAccessToken, FreshSandboxBuildSpec, OverlaybdConfig, SandboxLaunchConfig, UblkConfig, +}; use crate::snapshot::RunnableSnapshot; use crate::types::SandboxId; @@ -165,12 +167,16 @@ impl SandboxBackendFactory for FirecrackerSandboxFactory { &self, sandbox_id: SandboxId, state: &dyn PausedSandboxState, + envd_access_token: Option, ) -> Result> { - let config = state + let paused_state = state .downcast_ref::() .context("The provided PausedSandboxState is not a Firecracker paused state")?; - let sandbox = - FirecrackerSandbox::from_snapshot_config_with_id(config.snapshot_config(), sandbox_id)?; + let sandbox = FirecrackerSandbox::from_snapshot_config_with_override( + paused_state.snapshot_config().clone(), + sandbox_id, + envd_access_token, + )?; Ok(Box::new(sandbox)) } } @@ -245,7 +251,7 @@ mod tests { let factory = FirecrackerSandboxFactory::new(); let state: Arc = Arc::new(WrongPausedState); - match factory.build_from_paused_state(SandboxId::new(), state.as_ref()) { + match factory.build_from_paused_state(SandboxId::new(), state.as_ref(), None) { Ok(_) => panic!("wrong snapshot type should fail"), Err(err) => assert!(err.to_string().contains("not a Firecracker paused state")), } diff --git a/src/sandbox/firecracker/mmds.rs b/src/sandbox/firecracker/mmds.rs index 68013aa2..a6b556b0 100644 --- a/src/sandbox/firecracker/mmds.rs +++ b/src/sandbox/firecracker/mmds.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha512}; +use crate::sandbox::EnvdAccessToken; use crate::types::SandboxId; const EMPTY_ACCESS_TOKEN: &str = ""; @@ -28,7 +29,6 @@ impl MmdsMetadata { sandbox_id: sandbox_id.to_string(), snapshot_id: snapshot_id.into(), logs_collector_address: String::new(), - // Access token is currently not supported. access_token_hash: hash_access_token(EMPTY_ACCESS_TOKEN), extra: serde_json::Map::new(), } @@ -39,6 +39,15 @@ impl MmdsMetadata { self.extra.extend(extra); self } + + pub(crate) fn with_access_token(mut self, token: Option<&EnvdAccessToken>) -> Self { + self.set_access_token(token); + self + } + + pub(crate) fn set_access_token(&mut self, token: Option<&EnvdAccessToken>) { + self.access_token_hash = hash_access_token(token.map_or("", EnvdAccessToken::expose)); + } } fn hash_access_token(token: &str) -> String { @@ -70,6 +79,23 @@ mod tests { ); } + #[test] + fn hashes_secure_access_token_with_sha512() { + let sandbox_id = SandboxId::from_uuid(Uuid::nil()); + let token = crate::sandbox::SandboxAccessTokenGenerator::new("mmds-test-seed") + .unwrap() + .generate(sandbox_id); + + let metadata = + MmdsMetadata::new(sandbox_id, "snapshot-123").with_access_token(Some(&token)); + + assert_eq!( + metadata.access_token_hash, + hash_access_token(token.expose()) + ); + assert!(!metadata.access_token_hash.contains(token.expose())); + } + #[test] fn extra_fields_are_flattened_into_top_level() { let sandbox_id = SandboxId::from_uuid(Uuid::nil()); diff --git a/src/sandbox/firecracker/sandbox.rs b/src/sandbox/firecracker/sandbox.rs index d5464328..0c5ef1e7 100644 --- a/src/sandbox/firecracker/sandbox.rs +++ b/src/sandbox/firecracker/sandbox.rs @@ -30,9 +30,10 @@ use crate::sandbox::custom_extension::{ }; use crate::cfg::ConfigManager; +use crate::sandbox::access::EnvdAccessToken; use crate::sandbox::backend::{ CapturedSandboxSnapshot, PausedSandboxState, RuntimeArtifactSet, SandboxBackend, - SandboxCaptureError, SandboxCaptureResult, SandboxExecutor, SandboxForkResult, + SandboxCaptureError, SandboxCaptureResult, SandboxExecutor, SandboxForkResult, SandboxForkSpec, SandboxRuntimeInfo, }; use crate::sandbox::envd::EnvdInstance; @@ -334,7 +335,7 @@ impl SandboxBackend for FirecrackerSandbox { async fn fork( &mut self, - child_ids: &[SandboxId], + spec: &[SandboxForkSpec], ) -> SandboxCaptureResult> { let snapshot_config = match FirecrackerSandbox::pause(self).await { Ok(snapshot_config) => snapshot_config, @@ -356,12 +357,16 @@ impl SandboxBackend for FirecrackerSandbox { .await .map_err(SandboxCaptureError::terminal)?; - let children = child_ids + let children = spec .iter() - .map(|&child_id| { - Self::from_snapshot_config_with_id(&snapshot_config, child_id) - .map(|child| Box::new(child) as Box) - .context("build forked sandbox") + .map(|child| { + Self::from_snapshot_config_with_override( + snapshot_config.clone(), + child.sandbox_id, + child.envd_access_token.clone(), + ) + .map(|child| Box::new(child) as Box) + .context("build forked sandbox") }) .collect::>(); @@ -479,21 +484,28 @@ impl FirecrackerSandbox { /// Call [`FirecrackerSandbox::start`] or [`FirecrackerSandbox::start_nowait`] to boot it. #[tracing::instrument(skip(snapshot))] pub fn from_snapshot_config(snapshot: &FirecrackerSnapshotConfig) -> Result { - Self::from_snapshot_config_with_id(snapshot, SandboxId::new()) + Self::from_snapshot_config_with_override( + snapshot.clone(), + SandboxId::new(), + snapshot.common.envd_access_token.clone(), + ) } - pub(crate) fn from_snapshot_config_with_id( - snapshot: &FirecrackerSnapshotConfig, + pub(crate) fn from_snapshot_config_with_override( + mut snapshot: FirecrackerSnapshotConfig, id: SandboxId, + envd_access_token: Option, ) -> Result { - let mut snapshot = snapshot.clone(); - - // Ensure the snapshot's MMDS metadata has the correct sandbox ID. - snapshot - .common - .mmds_metadata - .get_or_insert_with(|| MmdsMetadata::new(id, "unknown")) - .sandbox_id = id.to_string(); + // Runtime identity and auth override their values in the source snapshot. + snapshot.common.envd_access_token = envd_access_token; + let FirecrackerCommonConfig { + mmds_metadata, + envd_access_token, + .. + } = &mut snapshot.common; + let metadata = mmds_metadata.get_or_insert_with(|| MmdsMetadata::new(id, "unknown")); + metadata.sandbox_id = id.to_string(); + metadata.set_access_token(envd_access_token.as_ref()); debug!( vm_state_path = %snapshot.vm_state_path.display(), @@ -528,8 +540,10 @@ impl FirecrackerSandbox { let mut snapshot_config = FirecrackerSnapshotConfig::from_runnable_snapshot(snapshot)?; snapshot_config.common.mmds_metadata = Some( MmdsMetadata::new(launch_config.sandbox_id, launch_config.snapshot_id.clone()) + .with_access_token(launch_config.envd_access_token.as_ref()) .with_extra(launch_config.extra_mmds.clone()), ); + snapshot_config.common.envd_access_token = launch_config.envd_access_token.clone(); snapshot_config.common.network_policy = launch_config.network.clone(); // Launch-provided custom config overrides the value persisted in the @@ -1353,7 +1367,10 @@ impl FirecrackerSandbox { "http://{}:{}", interaction_ip, config.common.control_plane_port ); - self.envd_instance = Some(EnvdInstance::new(envd_base_url)); + self.envd_instance = Some(EnvdInstance::new( + envd_base_url, + config.common.envd_access_token.clone(), + )); // ── Configure microVM: tools drive as rootfs + user image + extras ── self.fc_instance @@ -1534,7 +1551,10 @@ impl FirecrackerSandbox { "http://{}:{}", interaction_ip, config.common.control_plane_port ); - self.envd_instance = Some(EnvdInstance::new(envd_base_url)); + self.envd_instance = Some(EnvdInstance::new( + envd_base_url, + config.common.envd_access_token.clone(), + )); let mem_global_config = global_config .memory_snapshot @@ -1904,7 +1924,7 @@ async fn copy_cow(src: &Path, dst: &Path) -> Result<()> { mod tests { use super::*; use crate::cfg::ToolsConfig; - use crate::sandbox::SandboxExecutor; + use crate::sandbox::{SandboxAccessTokenGenerator, SandboxExecutor}; use crate::snapshot::{CommittedSnapshot, RunnableSnapshot, SnapshotRecord}; use std::collections::HashMap; @@ -2079,6 +2099,52 @@ mod tests { ); } + #[test] + fn snapshot_config_runtime_identity_replaces_source_auth() -> Result<()> { + let source_id = SandboxId::new(); + let child_id = SandboxId::new(); + let generator = SandboxAccessTokenGenerator::new("fork-test-seed")?; + let source_token = generator.generate(source_id); + let child_token = generator.generate(child_id); + let mut common = fresh_config().common; + common.mmds_metadata = + Some(MmdsMetadata::new(source_id, "snapshot").with_access_token(Some(&source_token))); + common.envd_access_token = Some(source_token.clone()); + let snapshot = FirecrackerSnapshotConfig { + common, + vm_state_path: "snapshot/vm_state.bin".into(), + mem_overlaybd_config: OverlaybdConfig { + image_config_path: "snapshot/mem_image.json".into(), + read_only: true, + runtime_upper_mode: overlaybd::config::UpperMode::LogStructured, + }, + mem_virtual_size: 4096, + managed_snapshot_root: None, + }; + + let child = FirecrackerSandbox::from_snapshot_config_with_override( + snapshot, + child_id, + Some(child_token.clone()), + )?; + let common = child.launch.common(); + let metadata = common.mmds_metadata.as_ref().expect("child MMDS metadata"); + let expected = + MmdsMetadata::new(child_id, "snapshot").with_access_token(Some(&child_token)); + + assert_eq!(child.id, child_id); + assert_eq!(common.envd_access_token.as_ref(), Some(&child_token)); + assert_eq!(metadata.sandbox_id, child_id.to_string()); + assert_eq!(metadata.access_token_hash, expected.access_token_hash); + assert_ne!( + metadata.access_token_hash, + MmdsMetadata::new(source_id, "snapshot") + .with_access_token(Some(&source_token)) + .access_token_hash + ); + Ok(()) + } + #[test] fn paused_state_without_tools_drive_version_remains_readable_but_not_resumable() -> Result<()> { let temp = TempDir::new()?; @@ -2158,10 +2224,13 @@ mod tests { #[tokio::test] async fn stop_without_process_clears_envd_instance() -> Result<()> { let mut sandbox = FirecrackerSandbox::new(fresh_config())?; - sandbox.envd_instance = Some(EnvdInstance::new(format!( - "http://127.0.0.1:{}", - ToolsConfig::default().control_plane_port - ))); + sandbox.envd_instance = Some(EnvdInstance::new( + format!( + "http://127.0.0.1:{}", + ToolsConfig::default().control_plane_port + ), + None, + )); sandbox.stop().await?; @@ -2259,6 +2328,7 @@ mod tests { network: None, extra_mmds: serde_json::Map::new(), custom_extension_params: None, + envd_access_token: None, }; let snapshot_config = diff --git a/src/sandbox/mock.rs b/src/sandbox/mock.rs index 2c4bde1c..9a4d0f87 100644 --- a/src/sandbox/mock.rs +++ b/src/sandbox/mock.rs @@ -16,15 +16,14 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use tokio::time::sleep; -use crate::snapshot::RunnableSnapshot; -use crate::types::SandboxId; - use super::backend::{ CapturedSandboxSnapshot, PausedSandboxState, RuntimeArtifactSet, SandboxBackend, - SandboxBackendFactory, SandboxCaptureResult, SandboxForkResult, SandboxRuntimeInfo, + SandboxBackendFactory, SandboxCaptureResult, SandboxForkResult, SandboxForkSpec, + SandboxRuntimeInfo, }; use super::{FreshSandboxBuildSpec, SandboxCaptureError, SandboxLaunchConfig}; use crate::sandbox::CustomExtensionParams; +use crate::snapshot::RunnableSnapshot; #[derive(Debug)] pub struct MockSnapshot; @@ -309,12 +308,12 @@ impl SandboxBackend for MockSandboxBackend { async fn fork( &mut self, - child_ids: &[SandboxId], + spec: &[SandboxForkSpec], ) -> SandboxCaptureResult> { self.behavior .apply_capture_result(MockOperation::Fork) .await?; - Ok(child_ids + Ok(spec .iter() .map(|_| { self.behavior @@ -420,6 +419,7 @@ impl SandboxBackendFactory for MockBackendFactory { &self, _sandbox_id: crate::types::SandboxId, _state: &dyn PausedSandboxState, + _envd_access_token: Option, ) -> Result> { self.behavior.apply_sync(MockOperation::BuildFromSnapshot)?; Ok(Box::new(MockSandboxBackend::new_with_host_ip( diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 8e052eb2..5cfceb3b 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -1,3 +1,4 @@ +mod access; mod backend; pub(crate) mod custom_extension; mod envd; @@ -18,10 +19,11 @@ pub(crate) use custom_extension::{ use crate::types::{ImageConfigs, SandboxId}; pub use ::envd::process::Signal; +pub use access::{EnvdAccessToken, SandboxAccessTokenGenerator}; pub use backend::{ CapturedSandboxSnapshot, PausedSandboxState, RuntimeArtifactSet, SandboxBackend, SandboxBackendFactory, SandboxCaptureError, SandboxCaptureResult, SandboxExecutor, - SandboxForkResult, SandboxRuntimeInfo, + SandboxForkResult, SandboxForkSpec, SandboxRuntimeInfo, }; pub use extra_drive::{ normalize_mount_path_for_drive, validate_drive_id, validate_mount_path, validate_sub_path, @@ -67,6 +69,9 @@ pub struct SandboxLaunchConfig { /// Opaque user-provided JSON passed through to the custom extension hooks. /// Takes precedence over any value persisted in the source snapshot. pub custom_extension_params: Option, + /// Runtime-only credential used by envd. The token is never serialized and + /// its Debug representation is redacted. + pub envd_access_token: Option, } impl SandboxLaunchConfig { @@ -78,6 +83,7 @@ impl SandboxLaunchConfig { network: None, extra_mmds: serde_json::Map::new(), custom_extension_params: None, + envd_access_token: None, } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 280460c6..60b0f3df 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -36,6 +36,10 @@ pub async fn setup() { /// registry image. pub async fn setup_runtime_only() -> &'static agentenv::cfg::AppConfig { agentenv::logging::init_for_tests(); + std::env::set_var( + "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED", + "integration-test-seed", + ); let config = ConfigManager::init_global() .expect("config manager") .config(); diff --git a/tests/integration/orchestrator.rs b/tests/integration/orchestrator.rs index 9df80a8f..fd906625 100644 --- a/tests/integration/orchestrator.rs +++ b/tests/integration/orchestrator.rs @@ -12,13 +12,39 @@ use agentenv::snapshot::{ }; use anyhow::Result; +use envd::process::{ListRequest, ProcessClient}; use std::path::PathBuf; use tempfile::tempdir; use tokio::time::{timeout, Duration}; +use tonic::Request; use uuid::Uuid; const TEST_TIMEOUT: Duration = Duration::from_secs(120); +async fn envd_process_list_status( + target: &agentenv::orchestrator::ProxyTarget, + access_token: Option<&str>, +) -> Result { + let port = ConfigManager::global_config().tools.control_plane_port; + let mut request = envd::reqwest::Client::new() + .post(format!("http://{}:{port}/process.Process/List", target.ip)); + if let Some(access_token) = access_token { + request = request.header("X-Access-Token", access_token); + } + Ok(request.send().await?.status()) +} + +async fn assert_envd_process_list_succeeds( + target: &agentenv::orchestrator::ProxyTarget, + access_token: &str, +) -> Result<()> { + let port = ConfigManager::global_config().tools.control_plane_port; + let mut client = + ProcessClient::connect(&format!("http://{}:{port}", target.ip), Some(access_token)).await?; + client.list(Request::new(ListRequest {})).await?; + Ok(()) +} + fn host_file_persister(root: PathBuf) -> FileBackedSandboxPersister { FileBackedSandboxPersister::new(root, ConfigManager::global_config().virtualization_mode) } @@ -65,10 +91,14 @@ async fn orchestrator_lifecycle() -> Result<()> { network_policy: SandboxNetworkPolicy::default(), auto_resume: false, custom_extension_params: None, + secure: true, }; let created = orchestrator.create_sandbox(request).await?; assert_eq!(created.state, SandboxState::Running); + let access_token = orchestrator + .get_envd_access_token(&created) + .expect("secure sandbox access token"); let sandbox_id = created.id; let lookup = orchestrator.proxy_lookup_for(&sandbox_id).await?; @@ -76,6 +106,39 @@ async fn orchestrator_lifecycle() -> Result<()> { matches!(lookup, ProxyLookupResult::Ready(_)), "expected proxy lookup Ready for sandbox {sandbox_id}, got {lookup:?}" ); + let ProxyLookupResult::Ready(target) = lookup else { + unreachable!(); + }; + assert_eq!( + envd_process_list_status(&target, None).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_eq!( + envd_process_list_status(&target, Some("wrong-token")).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_envd_process_list_succeeds(&target, access_token.expose()).await?; + + let child = orchestrator + .fork_sandbox(sandbox_id, 1, NewTimeout::UseExisting) + .await? + .pop() + .expect("one fork result")?; + let child_token = orchestrator + .get_envd_access_token(&child) + .expect("secure fork access token"); + assert_ne!(child_token, access_token); + let ProxyLookupResult::Ready(child_target) = + orchestrator.proxy_lookup_for(&child.id).await? + else { + panic!("secure fork should have a proxy route"); + }; + assert_eq!( + envd_process_list_status(&child_target, Some(access_token.expose())).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_envd_process_list_succeeds(&child_target, child_token.expose()).await?; + orchestrator.delete_sandbox(child.id).await?; let fetched = orchestrator .get_sandbox(&sandbox_id) @@ -124,6 +187,22 @@ async fn orchestrator_lifecycle() -> Result<()> { matches!(lookup, ProxyLookupResult::Ready(_)), "expected proxy lookup Ready after resume for sandbox {sandbox_id}, got {lookup:?}" ); + let resumed_token = restarted + .get_envd_access_token(&resumed) + .expect("resumed secure sandbox access token"); + assert_eq!(resumed_token, access_token); + let ProxyLookupResult::Ready(target) = lookup else { + unreachable!(); + }; + assert_eq!( + envd_process_list_status(&target, None).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_eq!( + envd_process_list_status(&target, Some("wrong-token")).await?, + envd::reqwest::StatusCode::UNAUTHORIZED + ); + assert_envd_process_list_succeeds(&target, resumed_token.expose()).await?; restarted.delete_sandbox(sandbox_id).await?; let deleted = restarted.get_sandbox(&sandbox_id).await?; @@ -170,6 +249,7 @@ async fn orchestrator_capture_snapshot_can_be_published_and_relaunched() -> Resu network_policy: SandboxNetworkPolicy::default(), auto_resume: false, custom_extension_params: None, + secure: false, }) .await?; let sandbox_id = created.id; @@ -263,6 +343,7 @@ async fn orchestrator_capture_snapshot_can_be_published_and_relaunched() -> Resu network_policy: SandboxNetworkPolicy::default(), auto_resume: false, custom_extension_params: None, + secure: false, }) .await?; assert_eq!(relaunched.state, SandboxState::Running); diff --git a/tests/integration/snapshot.rs b/tests/integration/snapshot.rs index 950e1e85..de24f7b5 100644 --- a/tests/integration/snapshot.rs +++ b/tests/integration/snapshot.rs @@ -374,6 +374,7 @@ async fn persistent_snapshot_lifecycle_preserves_original_pause_resume_state() - network: None, extra_mmds: serde_json::Map::new(), custom_extension_params: None, + envd_access_token: None, }; let mut child = FirecrackerSandbox::from_snapshot(runnable, &launch_config)?; child.start().await?; @@ -574,6 +575,7 @@ async fn randomized_snapshot_lifecycle_operations_preserve_artifact_ownership() network: None, extra_mmds: serde_json::Map::new(), custom_extension_params: None, + envd_access_token: None, }; let mut sandbox = FirecrackerSandbox::from_snapshot(&runnable, &launch_config)?; sandbox.start().await?; diff --git a/thirdparty/envd/src/lib.rs b/thirdparty/envd/src/lib.rs index c7996d4f..0a2671e6 100644 --- a/thirdparty/envd/src/lib.rs +++ b/thirdparty/envd/src/lib.rs @@ -17,8 +17,11 @@ macro_rules! impl_envd_client { } impl $client_struct { - pub async fn connect(dst: &str) -> Result { - let chan = crate::transport::new_channel(dst)?; + pub async fn connect( + dst: &str, + access_token: Option<&str>, + ) -> Result { + let chan = crate::transport::new_channel(dst, access_token)?; Ok(Self { inner: GeneratedClient::new(chan), }) diff --git a/thirdparty/envd/src/transport.rs b/thirdparty/envd/src/transport.rs index df484aa3..036b40ea 100644 --- a/thirdparty/envd/src/transport.rs +++ b/thirdparty/envd/src/transport.rs @@ -23,6 +23,7 @@ struct DualClient { h2: Client, protocol: Arc>>, uri: Uri, + access_token: Option, } #[derive(Clone, Copy, Debug)] @@ -50,6 +51,7 @@ impl Service> for DualClient { let h2 = self.h2.clone(); let protocol = self.protocol.clone(); let uri = self.uri.clone(); + let access_token = self.access_token.clone(); Box::pin(async move { // Determine protocol if unknown @@ -79,6 +81,10 @@ impl Service> for DualClient { *protocol.lock().await = proto; } + if let Some(access_token) = access_token { + req.headers_mut().insert("x-access-token", access_token); + } + // Prepare the actual request match proto.unwrap() { Protocol::H2 => { @@ -116,8 +122,16 @@ fn nodelay_connector() -> HttpConnector { /// Creates a channel compatible with both HTTP/1.1 and HTTP/2. /// /// The channel automatically probes the server to determine supported protocol (H2 or H1). -pub fn new_channel(addr: &str) -> anyhow::Result { +pub fn new_channel(addr: &str, access_token: Option<&str>) -> anyhow::Result { let uri: Uri = addr.parse().context("Invalid URI")?; + let access_token = access_token + .map(|token| { + let mut token = http::HeaderValue::from_str(token)?; + token.set_sensitive(true); + Ok::<_, http::header::InvalidHeaderValue>(token) + }) + .transpose() + .context("Invalid envd access token")?; let h1 = Client::builder(TokioExecutor::new()) .http2_only(false) @@ -133,6 +147,7 @@ pub fn new_channel(addr: &str) -> anyhow::Result { h2, protocol: Arc::new(Mutex::new(None)), uri, + access_token, }; Ok(tower::util::BoxCloneService::new(service)) diff --git a/thirdparty/envd/tests/filesystem_test.rs b/thirdparty/envd/tests/filesystem_test.rs index eba36712..9a4bf502 100644 --- a/thirdparty/envd/tests/filesystem_test.rs +++ b/thirdparty/envd/tests/filesystem_test.rs @@ -11,7 +11,7 @@ use common::TEST_ENVD_ADDR; /// Verifies filesystem operations: MakeDir, Stat, Move, ListDir, Remove. #[tokio::test] async fn test_filesystem_lifecycle() -> Result<()> { - let mut client = FilesystemClient::connect(TEST_ENVD_ADDR).await?; + let mut client = FilesystemClient::connect(TEST_ENVD_ADDR, None).await?; let tmp = tempfile::TempDir::new()?; let base_path = tmp diff --git a/thirdparty/envd/tests/process_test.rs b/thirdparty/envd/tests/process_test.rs index b9fe43f7..5b44888d 100644 --- a/thirdparty/envd/tests/process_test.rs +++ b/thirdparty/envd/tests/process_test.rs @@ -14,7 +14,7 @@ const TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); /// Verifies that the process list is initially empty. #[tokio::test] async fn test_list_process() -> Result<()> { - let mut client = ProcessClient::connect(TEST_ENVD_ADDR).await?; + let mut client = ProcessClient::connect(TEST_ENVD_ADDR, None).await?; let request = Request::new(ListRequest {}); let response = client.list(request).await?; @@ -32,7 +32,7 @@ async fn test_list_process() -> Result<()> { async fn test_start_process() -> Result<()> { tokio::time::timeout(TEST_TIMEOUT, async { // Connect to the server - let mut client = ProcessClient::connect(TEST_ENVD_ADDR).await?; + let mut client = ProcessClient::connect(TEST_ENVD_ADDR, None).await?; // Configure the process request: run /bin/bash validation script let request = Request::new(StartRequest { @@ -110,7 +110,7 @@ async fn test_start_process() -> Result<()> { #[tokio::test] async fn test_process_interactive() -> Result<()> { tokio::time::timeout(TEST_TIMEOUT, async { - let mut client = ProcessClient::connect(TEST_ENVD_ADDR).await?; + let mut client = ProcessClient::connect(TEST_ENVD_ADDR, None).await?; // Start `cat` let request = Request::new(StartRequest { From dd45ddd5a8e6298fbbbf3ba399a3255126f8e0e6 Mon Sep 17 00:00:00 2001 From: Sixing Lin <2682191498@qq.com> Date: Wed, 5 Aug 2026 16:26:58 +0800 Subject: [PATCH 2/4] feat(sandbox): generate envd access token seed when unconfigured --- config/default.toml | 5 +- deploy/k8s/base/agentenv-daemonset.yaml | 5 + .../k8s/overlays/local-dev/kustomization.yaml | 10 + docs/src/configuration/env-vars.md | 2 +- docs/src/configuration/reference.md | 6 +- docs/src/deployment/docker-compose.md | 31 +- docs/src/deployment/kubernetes.md | 39 ++ docs/src/deployment/static-multi-node.md | 8 +- .../persistence-artifact-inventory.md | 2 + scripts/tests/e2e/code_interpreter_compat.py | 3 + scripts/tests/e2e/e2b_python_sdk_compat.py | 2 + scripts/tests/e2e/e2b_ts_sdk_compat.ts | 2 + src/bin/server.rs | 2 - src/cfg.rs | 30 +- src/orchestrator/service.rs | 12 +- src/orchestrator/tests.rs | 14 +- src/sandbox/access.rs | 489 +++++++++++++++++- 17 files changed, 603 insertions(+), 59 deletions(-) diff --git a/config/default.toml b/config/default.toml index 795e48e8..5d28f511 100644 --- a/config/default.toml +++ b/config/default.toml @@ -188,8 +188,9 @@ init_timeout_secs = 60 poll_ms = 3 [sandbox] -# Shared secret used to derive per-sandbox envd access tokens. All nodes in a -# cluster must use the same value. Prefer AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED. +# Optional secret used to derive per-sandbox envd access tokens. When unset, +# AgentENV creates a node-local seed under $AENV_HOME/secrets. Configure the +# same explicit value on every node before enabling cross-node sandbox recovery. # access_token_hash_seed = "replace-with-a-secret" [orchestrator] diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index fd608752..cd10a6c9 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -23,6 +23,11 @@ spec: env: - name: AENV_CONFIG_PATH value: /workspace/config/agentenv.toml + - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED + valueFrom: + secretKeyRef: + name: agentenv-runtime-secrets + key: sandbox-access-token-hash-seed - name: AENV_VIRTUALIZATION_MODE value: "kvm" - name: API_ADDR diff --git a/deploy/k8s/overlays/local-dev/kustomization.yaml b/deploy/k8s/overlays/local-dev/kustomization.yaml index 0a2ae1d2..2a71de5e 100644 --- a/deploy/k8s/overlays/local-dev/kustomization.yaml +++ b/deploy/k8s/overlays/local-dev/kustomization.yaml @@ -1,9 +1,19 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: agentenv-system + resources: - ../../base +secretGenerator: + - name: agentenv-runtime-secrets + literals: + - sandbox-access-token-hash-seed=agentenv-local-dev-access-token-hash-seed + +generatorOptions: + disableNameSuffixHash: true + patches: - target: kind: DaemonSet diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index aed5ef3a..1253556f 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -23,7 +23,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | required for normal server startup | Cluster-wide secret used to derive secure sandbox envd access tokens. Use the same value on every node. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional override for the secret used to derive secure sandbox envd access tokens. Use the same explicit value on every node before enabling cross-node recovery of the same sandbox ID. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index 9fef1513..12fb2720 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -257,9 +257,11 @@ Sandbox control communication settings. | Key | Type | Default | Description | |-----|------|---------|-------------| -| `access_token_hash_seed` | string | required | Cluster-wide secret used to derive secure sandbox envd access tokens. All nodes must use the same value. Prefer the `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` environment variable. Normal server startup fails when it is missing or empty; setup-only and host setup do not require it. | +| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive secure sandbox envd access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. | -Changing the seed invalidates access tokens for existing secure sandboxes. Rotate it with a coordinated restart of every node. +The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. + +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every node to enable cross-node recovery of the same sandbox. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 3c606cd5..5d29b467 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -32,11 +32,38 @@ git clone https://github.com/kvcache-ai/AgentENV.git cd AgentENV ``` +## Configure the Shared Access-Token Seed + +Multi-node deployments should explicitly configure one envd access-token seed +and use the same value on every **runtime node**. Do not rely on the node-local +auto-generated seed in a scheduler-managed deployment. + +Generate the value once and store it in your secret manager: + +```bash +openssl rand -hex 32 +``` + +For the checked-in Compose stack, create a private configuration copy outside +the repository and set the generated value in it: + +```bash +install -d -m 0700 "$HOME/.config/agentenv" +install -m 0600 config/default.toml "$HOME/.config/agentenv/cluster.toml" +``` + +```toml +[sandbox] +access_token_hash_seed = "" +``` + +Changing this value rotates the access tokens for existing secure sandboxes. + ## Start the Cluster ```bash sudo bash scripts/docker-setup.sh -make deploy-up +CONFIG_PATH="$HOME/.config/agentenv/cluster.toml" make deploy-up ``` To enable host-based sandbox data-plane URLs, set the shared sandbox proxy @@ -44,7 +71,7 @@ domain variable when starting the stack: ```bash SANDBOX_PROXY_DOMAINS=sandbox.example.com \ -make deploy-up +CONFIG_PATH="$HOME/.config/agentenv/cluster.toml" make deploy-up ``` Compose passes this value to both the gateway routing allowlist and runtime diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 99ef1245..bd77a962 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -43,6 +43,41 @@ make k8s-build This builds three images: `agentenv-runtime:latest`, `agentenv-gateway:latest`, and `agentenv-scheduler:latest`. +## Configure the Shared Access-Token Seed + +Before starting runtime Pods, create the namespace and generate one envd +access-token seed. Store it in the Secret required by the checked-in DaemonSet: + +```bash +kubectl apply -f deploy/k8s/base/namespace.yaml + +AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ + --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ + --dry-run=client -o yaml | kubectl apply -f - +unset AENV_ACCESS_TOKEN_HASH_SEED +``` + +Run this once for a new cluster. During upgrades, preserve the existing Secret +instead of generating another value. Production deployments may replace this +command with an external secret manager, but must provide the same Secret name +and key: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: agentenv-runtime-secrets + namespace: agentenv-system +stringData: + sandbox-access-token-hash-seed: +``` + +The DaemonSet injects this value into every runtime Pod. The Gateway and +Scheduler do not consume it. Changing the Secret rotates the access tokens for +existing secure sandboxes, so update it only as a coordinated credential +rotation. + ## Deploy ```bash @@ -96,6 +131,10 @@ make k8s-delete A dedicated `local-dev` overlay mounts the repository's `env/` directory directly into the DaemonSet at `/workspace/env`, avoiding runtime asset copies: +This overlay also generates `agentenv-runtime-secrets` with a fixed test-only +seed so local and E2E deployments do not require production secret management. +Do not reuse that value outside local development. + ```bash make k8s-build make k8s-load-dev # Import images into k3s/containerd diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 2ab8e573..8d5efe84 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -62,11 +62,13 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in ``` Edit `/etc/default/aenv` on each machine without removing the paths written by -the installer. Node A uses: +the installer. Generate one access-token seed with `openssl rand -hex 32`, store +it securely, and set the same value on every runtime node. Node A uses: ```bash API_ADDR="0.0.0.0:8000" AENV_NODE_ID="node-a" +AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="" AENV_OBSERVABILITY_SCHEDULER_REPORT_ENABLED="true" AENV_OBSERVABILITY_SCHEDULER_ENDPOINT="http://10.0.0.10:9090" ``` @@ -76,12 +78,14 @@ Node B uses the same values except for its unique node ID: ```bash API_ADDR="0.0.0.0:8000" AENV_NODE_ID="node-b" +AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="" AENV_OBSERVABILITY_SCHEDULER_REPORT_ENABLED="true" AENV_OBSERVABILITY_SCHEDULER_ENDPOINT="http://10.0.0.10:9090" ``` The `AENV_NODE_ID` values must exactly match the corresponding IDs in the -Scheduler configuration below. Restart and verify each runtime: +Scheduler configuration below. Changing the shared seed rotates the access +tokens for existing secure sandboxes. Restart and verify each runtime: ```bash sudo systemctl restart aenv diff --git a/docs/src/internals/persistence-artifact-inventory.md b/docs/src/internals/persistence-artifact-inventory.md index e577d49a..750b8d0a 100644 --- a/docs/src/internals/persistence-artifact-inventory.md +++ b/docs/src/internals/persistence-artifact-inventory.md @@ -9,6 +9,7 @@ This document lists AgentENV artifacts that can remain on disk or in object stor | `home_path` | `/var/lib/aenv` | `src/cfg.rs` | Base for paths containing the literal `$AENV_HOME` placeholder. `AENV_HOME_PATH` overrides it before placeholder expansion. | | `runtime_path` | `/run/aenv` | `src/cfg.rs`, `src/sandbox/network/*` | Base for transient namespace mount points and daemon sockets. `AENV_RUNTIME_PATH` overrides it. | | `deps_path` | `$AENV_HOME/deps` | `src/cfg.rs`, `src/setup/*` | Base for downloaded runtime dependencies. `AENV_DEPS_PATH` can place these rebuildable assets outside `home_path`. | +| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure sandboxes. | | Firecracker sandbox work dirs | `$AENV_HOME/firecracker-work` with `agentenv-fc-` children | `src/sandbox/firecracker/*` | Per-sandbox runtime directories for sockets, symlinks, ublk runtime dirs, local logs, and writable OverlayBD upper layer data (`overlaybd/upper.data`, `overlaybd/upper.index`). An explicit `[firecracker].work_dir` overrides the root. | | `firecracker.serial_dir` | `$AENV_HOME/logs/serial` | `src/sandbox/firecracker/*` | Durable Firecracker stdout/stderr root, grouped by sandbox ID. An explicit `[firecracker].serial_dir` overrides the root. | | `managed_snapshot_root` | `/managed-snapshots` | `src/sandbox/firecracker/*` | In-process live snapshot artifact root used to keep captured snapshots alive until publish or drop. | @@ -36,6 +37,7 @@ Owned by `src/setup/*` and `src/cfg.rs`. | Overlaybd package downloads | `/overlaybd/downloads/*` | Temporary downloaded package archives | Setup staging for overlaybd release packages | Removed after a successful install. | | Generated overlaybd config | `$AENV_HOME/overlaybd/overlaybd-global.json`, `$AENV_HOME/overlaybd/mem-overlaybd-global.json`, `$AENV_HOME/overlaybd/convert-overlaybd-global.json`, `$AENV_HOME/overlaybd/resize-overlaybd-global.json` | Runtime global config, cache path, credentials config | Configures overlaybd runtime, memory snapshot overlaybd access, and the offline C++ tools (`overlaybd-apply`, `overlaybd-resize`), which get dedicated configs with isolated cacheDirs (`convert-blocks`, `resize-blocks`) and download disabled | Rewritten during setup/startup. | | Overlaybd runtime log | `$AENV_HOME/overlaybd/overlaybd.log` | Overlaybd runtime logs | Debugging | Appended by overlaybd runtime; no automatic GC. | +| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure sandboxes are persisted. | ## Firecracker Sandbox diff --git a/scripts/tests/e2e/code_interpreter_compat.py b/scripts/tests/e2e/code_interpreter_compat.py index 6ac26a0f..ccfd9cd5 100644 --- a/scripts/tests/e2e/code_interpreter_compat.py +++ b/scripts/tests/e2e/code_interpreter_compat.py @@ -105,6 +105,7 @@ def correctness_tests( sandbox_url=sandbox_url, api_key=api_key, request_timeout=60, + secure=True, ) require(correct_sandbox.sandbox_id, "sandbox create returned empty sandbox_id") log(f"sandbox created: {correct_sandbox.sandbox_id}") @@ -254,6 +255,7 @@ def performance_tests( sandbox_url=sandbox_url, api_key=api_key, request_timeout=60, + secure=True, ) retry(lambda: perf_sandbox.run_code("1 + 1"), "perf first cmd", attempts=10, delay=2.0) cold_to_first_cmd_s = time.perf_counter() - t0 @@ -301,6 +303,7 @@ def _create(idx: int) -> tuple[int, float, CodeSandbox | None]: sandbox_url=sandbox_url, api_key=api_key, request_timeout=60, + secure=True, ) return idx, time.perf_counter() - t0, sb except Exception as err: diff --git a/scripts/tests/e2e/e2b_python_sdk_compat.py b/scripts/tests/e2e/e2b_python_sdk_compat.py index 2845e432..9300e81f 100644 --- a/scripts/tests/e2e/e2b_python_sdk_compat.py +++ b/scripts/tests/e2e/e2b_python_sdk_compat.py @@ -109,6 +109,7 @@ def main() -> int: api_key=api_key, sandbox_url=sandbox_url, request_timeout=60, + secure=True, ) require(sandbox.sandbox_id, "sandbox create returned an empty sandbox_id") log(f"sandbox created: {sandbox.sandbox_id}") @@ -228,6 +229,7 @@ def read_build_artifacts(): api_key=api_key, sandbox_url=sandbox_url, request_timeout=60, + secure=True, ) require( derived_sandbox.sandbox_id, diff --git a/scripts/tests/e2e/e2b_ts_sdk_compat.ts b/scripts/tests/e2e/e2b_ts_sdk_compat.ts index 44ad8155..7912e2cc 100644 --- a/scripts/tests/e2e/e2b_ts_sdk_compat.ts +++ b/scripts/tests/e2e/e2b_ts_sdk_compat.ts @@ -95,6 +95,7 @@ async function main(): Promise { template: templateName, }, timeoutMs: 90_000, + secure: true, ...connOpts, }); check(!!sandbox.sandboxId, "sandbox create returned an empty sandboxId"); @@ -189,6 +190,7 @@ async function main(): Promise { template: derivedTemplateName, baseTemplate: publicTemplate, }, + secure: true, ...connOpts, }); check(!!derivedSandbox.sandboxId, "from_template sandbox create returned an empty sandboxId"); diff --git a/src/bin/server.rs b/src/bin/server.rs index 5dca7ce7..0140c8a1 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -66,8 +66,6 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } - config.sandbox.validate_access_token_hash_seed()?; - agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; diff --git a/src/cfg.rs b/src/cfg.rs index d61f57a7..9828d864 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -290,22 +290,6 @@ impl std::fmt::Debug for SandboxConfig { } } -impl SandboxConfig { - pub(crate) fn access_token_hash_seed(&self) -> Result<&str> { - let seed = self.access_token_hash_seed.as_deref().ok_or_else(|| { - anyhow!("[sandbox].access_token_hash_seed must be configured and non-empty") - })?; - if seed.trim().is_empty() { - bail!("[sandbox].access_token_hash_seed must be configured and non-empty"); - } - Ok(seed) - } - - pub fn validate_access_token_hash_seed(&self) -> Result<()> { - self.access_token_hash_seed().map(|_| ()) - } -} - #[derive(Debug, Config, Clone)] pub struct MachineConfig { #[config(default = 2u32)] @@ -626,6 +610,7 @@ impl_config_default!( ToolsConfig, SandboxProxyConfig, EnvdConfig, + SandboxConfig, MachineConfig, SnapshotConfig, SnapshotImagePublishConfig, @@ -1345,17 +1330,10 @@ mod tests { } #[test] - fn sandbox_access_token_seed_is_required_for_serving() { - let mut config = SandboxConfig { - access_token_hash_seed: None, + fn sandbox_access_token_seed_is_redacted() { + let config = SandboxConfig { + access_token_hash_seed: Some("cluster-secret".to_string()), }; - assert!(config.validate_access_token_hash_seed().is_err()); - - config.access_token_hash_seed = Some(" ".to_string()); - assert!(config.validate_access_token_hash_seed().is_err()); - - config.access_token_hash_seed = Some("cluster-secret".to_string()); - assert!(config.validate_access_token_hash_seed().is_ok()); assert!(!format!("{config:?}").contains("cluster-secret")); } diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index 8b89e566..079362ed 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -5,6 +5,7 @@ use std::sync::{ }; use std::time::{Duration, SystemTime}; +use anyhow::Context; use tokio::sync::{broadcast, oneshot, watch, Mutex, OnceCell, RwLock}; use tokio::time::MissedTickBehavior; use tracing::{debug, info, trace, warn}; @@ -105,7 +106,7 @@ pub struct Orchestrator< shutdown_tx: watch::Sender, shutdown_outcome: OnceCell, image_refs: Arc, - access_tokens: Arc, + access_tokens: SandboxAccessTokenGenerator, } impl Orchestrator { @@ -153,9 +154,6 @@ where image_refs: Arc, ) -> Result> { let app_config = ConfigManager::global_config(); - let access_tokens = Arc::new(SandboxAccessTokenGenerator::new( - app_config.sandbox.access_token_hash_seed()?, - )?); let config = &app_config.orchestrator; let (shutdown_tx, shutdown_rx) = watch::channel(false); let (sandbox_event_tx, _sandbox_event_rx) = @@ -164,6 +162,12 @@ where // Restore persisted sandboxes from the previous run, keeping the paused // ones (with their state) for the paused-protection reconcile below. let persisted = persister.load_all(&factory).await?; + let managed_seed_must_exist = persisted.iter().any(|metadata| metadata.secure); + let access_tokens = tokio::task::spawn_blocking(move || { + SandboxAccessTokenGenerator::load_or_create(app_config, managed_seed_must_exist) + }) + .await + .context("join envd access-token seed loader")??; let restored_paused: Vec<(SandboxId, Arc)> = persisted .iter() .filter(|metadata| metadata.state == SandboxState::Paused) diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index e933525a..af4d24ae 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -50,18 +50,6 @@ fn test_runtime_image_refs() -> Arc { local_image_services_from_global_config().runtime_refs } -fn test_access_tokens() -> Arc { - Arc::new( - SandboxAccessTokenGenerator::new( - ConfigManager::global_config() - .sandbox - .access_token_hash_seed() - .unwrap(), - ) - .unwrap(), - ) -} - async fn make_orchestrator() -> Arc { Orchestrator::new_inner( InMemoryMetadataStore::new(), @@ -129,7 +117,7 @@ fn make_orchestrator_without_background_with_factory_and_persister< shutdown_tx: tokio::sync::watch::channel(false).0, shutdown_outcome: tokio::sync::OnceCell::new(), image_refs: test_runtime_image_refs(), - access_tokens: test_access_tokens(), + access_tokens: SandboxAccessTokenGenerator::new("orchestrator-test-seed").unwrap(), }) } diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index 1e39926d..3a620430 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -1,13 +1,24 @@ use std::fmt; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::Path; -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use hmac::{Hmac, Mac}; +use rand::{rngs::SysRng, TryRng}; use sha2::Sha256; +use tracing::{info, warn}; +use crate::cfg::AppConfig; use crate::types::SandboxId; type HmacSha256 = Hmac; +const MANAGED_SEED_RELATIVE_PATH: &str = "secrets/sandbox-access-token-hash-seed"; +const MANAGED_SEED_BYTES: usize = 32; +const SEED_HEX_LEN: usize = MANAGED_SEED_BYTES * 2; +const MANAGED_SEED_FILE_MAX_LEN: usize = SEED_HEX_LEN + 1; + #[derive(Clone, PartialEq, Eq)] pub struct EnvdAccessToken(String); @@ -30,15 +41,35 @@ pub struct SandboxAccessTokenGenerator { impl SandboxAccessTokenGenerator { pub fn new(seed: &str) -> Result { - let seed = seed.trim(); - if seed.is_empty() { - bail!("[sandbox].access_token_hash_seed must be configured and non-empty"); - } + let seed = validate_explicit_seed(seed)?; Ok(Self { seed: seed.as_bytes().to_vec(), }) } + pub(crate) fn load_or_create( + config: &AppConfig, + managed_seed_must_exist: bool, + ) -> Result { + if let Some(seed) = config.sandbox.access_token_hash_seed.as_deref() { + return Self::new(seed); + } + + let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); + let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; + + if config.sandbox.access_token_hash_seed.is_none() + && config.cluster.scheduler_endpoint.is_some() + { + warn!( + path = %managed_seed_path.display(), + "using a node-local managed envd access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node before enabling cross-node sandbox recovery" + ); + } + + Self::new(&seed) + } + pub fn generate(&self, subject: SandboxId) -> EnvdAccessToken { let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); @@ -56,6 +87,250 @@ impl SandboxAccessTokenGenerator { } } +fn validate_explicit_seed(seed: &str) -> Result<&str> { + let seed = seed.trim(); + if seed.is_empty() { + bail!("[sandbox].access_token_hash_seed must be non-empty when configured"); + } + Ok(seed) +} + +fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { + let parent = managed_path + .parent() + .context("managed envd access-token seed path has no parent")?; + match validate_managed_seed_directory(parent) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("validate managed secret directory {}", parent.display()) + }); + } + } + + match open_managed_seed(managed_path) { + Ok(file) => return read_managed_seed(managed_path, file), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "open managed envd access-token seed {}", + managed_path.display() + ) + }); + } + } + + if managed_seed_must_exist { + bail!( + "managed envd access-token seed {} is missing while persisted secure sandboxes exist; restore the file or configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED", + managed_path.display() + ); + } + + create_managed_seed(managed_path) +} + +fn open_managed_seed(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + + options.open(path) +} + +fn validate_managed_seed_file(path: &Path, file: &File) -> Result { + let metadata = file + .metadata() + .with_context(|| format!("inspect managed envd access-token seed {}", path.display()))?; + if !metadata.is_file() { + bail!( + "managed envd access-token seed {} must be a regular file", + path.display() + ); + } + + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let mode = metadata.permissions().mode() & 0o777; + if mode != 0o600 { + bail!( + "managed envd access-token seed {} must have permissions 0600, found {mode:04o}", + path.display() + ); + } + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + bail!( + "managed envd access-token seed {} must be owned by uid {expected_uid}, found uid {}", + path.display(), + metadata.uid() + ); + } + } + + Ok(metadata) +} + +fn read_managed_seed(path: &Path, mut file: File) -> Result { + let metadata = validate_managed_seed_file(path, &file)?; + + if metadata.len() > MANAGED_SEED_FILE_MAX_LEN as u64 { + bail!( + "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", + path.display() + ); + } + + let mut contents = String::with_capacity(MANAGED_SEED_FILE_MAX_LEN); + Read::by_ref(&mut file) + .take((MANAGED_SEED_FILE_MAX_LEN + 1) as u64) + .read_to_string(&mut contents) + .with_context(|| format!("read managed envd access-token seed {}", path.display()))?; + if contents.len() > MANAGED_SEED_FILE_MAX_LEN { + bail!( + "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", + path.display() + ); + } + let seed = contents.strip_suffix('\n').unwrap_or(&contents); + if !is_valid_managed_seed(seed) { + bail!( + "managed envd access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", + path.display() + ); + } + + Ok(seed.to_owned()) +} + +fn create_managed_seed(path: &Path) -> Result { + let parent = path + .parent() + .context("managed envd access-token seed path has no parent")?; + fs::create_dir_all(parent) + .with_context(|| format!("create managed secret directory {}", parent.display()))?; + validate_managed_seed_directory_identity(parent).with_context(|| { + format!( + "validate managed secret directory ownership {}", + parent.display() + ) + })?; + set_permissions(parent, 0o700)?; + validate_managed_seed_directory(parent) + .with_context(|| format!("validate managed secret directory {}", parent.display()))?; + + let mut random = [0_u8; MANAGED_SEED_BYTES]; + SysRng + .try_fill_bytes(&mut random) + .context("generate managed envd access-token seed")?; + let seed = hex::encode(random); + + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temporary seed file in {}", parent.display()))?; + set_permissions(temporary.path(), 0o600)?; + writeln!(temporary, "{seed}") + .with_context(|| format!("write temporary seed file in {}", parent.display()))?; + temporary + .as_file() + .sync_all() + .with_context(|| format!("sync temporary seed file in {}", parent.display()))?; + + match temporary.persist_noclobber(path) { + Ok(_) => { + fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("sync managed secret directory {}", parent.display()))?; + info!(path = %path.display(), "generated managed envd access-token seed"); + Ok(seed) + } + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { + let file = open_managed_seed(path).with_context(|| { + format!("open managed envd access-token seed {}", path.display()) + })?; + read_managed_seed(path, file) + } + Err(error) => Err(error.error) + .with_context(|| format!("persist managed envd access-token seed {}", path.display())), + } +} + +fn is_valid_managed_seed(seed: &str) -> bool { + seed.len() == SEED_HEX_LEN + && seed + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn validate_managed_seed_directory(path: &Path) -> io::Result<()> { + let metadata = validate_managed_seed_directory_identity(path)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = metadata.permissions().mode() & 0o777; + if mode != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("must have permissions 0700, found {mode:04o}"), + )); + } + } + + Ok(()) +} + +fn validate_managed_seed_directory_identity(path: &Path) -> io::Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "must be a directory and not a symbolic link", + )); + } + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "must be owned by uid {expected_uid}, found uid {}", + metadata.uid() + ), + )); + } + } + + Ok(metadata) +} + +#[cfg(unix)] +fn set_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} + impl fmt::Debug for SandboxAccessTokenGenerator { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("SandboxAccessTokenGenerator()") @@ -65,6 +340,14 @@ impl fmt::Debug for SandboxAccessTokenGenerator { #[cfg(test)] mod tests { use super::*; + use std::sync::{Arc, Barrier}; + + use tempfile::TempDir; + + fn create_private_managed_seed_directory(path: &Path) -> Result<()> { + fs::create_dir_all(path)?; + set_permissions(path, 0o700) + } #[test] fn generates_lowercase_hex_hmac_sha256() { @@ -95,4 +378,200 @@ mod tests { assert!(!format!("{generator:?}").contains("super-secret")); assert!(!format!("{token:?}").contains(token.expose())); } + + #[test] + fn explicit_seed_takes_precedence_without_creating_managed_state() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + let config = AppConfig { + home_path: temp.path().to_owned(), + sandbox: crate::cfg::SandboxConfig { + access_token_hash_seed: Some("configured-seed".to_owned()), + }, + ..Default::default() + }; + + let generator = SandboxAccessTokenGenerator::load_or_create(&config, false)?; + + assert_eq!(generator.seed, "configured-seed".as_bytes()); + assert!(!managed_path.exists()); + Ok(()) + } + + #[test] + fn managed_seed_is_private_and_stable() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + + let first = resolve_seed(&managed_path, false)?; + let second = resolve_seed(&managed_path, false)?; + + assert_eq!(first, second); + assert_eq!(first.len(), 64); + assert!(first + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let directory_mode = fs::metadata(managed_path.parent().unwrap())? + .permissions() + .mode() + & 0o777; + let file_mode = fs::metadata(&managed_path)?.permissions().mode() & 0o777; + assert_eq!(directory_mode, 0o700); + assert_eq!(file_mode, 0o600); + } + + let subject = SandboxId::new(); + let first_generator = SandboxAccessTokenGenerator::new(&first)?; + let second_generator = SandboxAccessTokenGenerator::new(&second)?; + assert_eq!( + first_generator.generate(subject), + second_generator.generate(subject) + ); + Ok(()) + } + + #[test] + fn concurrent_managed_seed_creation_converges() -> Result<()> { + const THREADS: usize = 8; + + let temp = TempDir::new()?; + let managed_path = Arc::new(temp.path().join(MANAGED_SEED_RELATIVE_PATH)); + let barrier = Arc::new(Barrier::new(THREADS)); + let handles = (0..THREADS) + .map(|_| { + let managed_path = Arc::clone(&managed_path); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + resolve_seed(&managed_path, false) + }) + }) + .collect::>(); + + let seeds = handles + .into_iter() + .map(|handle| handle.join().expect("seed creation thread panicked")) + .collect::>>()?; + assert!(seeds.iter().all(|seed| seed == &seeds[0])); + Ok(()) + } + + #[test] + fn empty_or_invalid_managed_seed_is_not_replaced() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + create_private_managed_seed_directory(managed_path.parent().unwrap())?; + + for contents in ["", "invalid\n"] { + fs::write(&managed_path, contents)?; + set_permissions(&managed_path, 0o600)?; + + let error = resolve_seed(&managed_path, false).unwrap_err(); + + assert!(error.to_string().contains("64 lowercase hexadecimal")); + assert_eq!(fs::read_to_string(&managed_path)?, contents); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn permissive_managed_seed_is_rejected() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + create_private_managed_seed_directory(managed_path.parent().unwrap())?; + fs::write(&managed_path, format!("{}\n", "a".repeat(64)))?; + set_permissions(&managed_path, 0o640)?; + + let error = resolve_seed(&managed_path, false).unwrap_err(); + + assert!(error.to_string().contains("permissions 0600")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn managed_seed_symlink_is_rejected() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + let target_path = temp.path().join("seed-target"); + create_private_managed_seed_directory(managed_path.parent().unwrap())?; + fs::write(&target_path, format!("{}\n", "a".repeat(64)))?; + set_permissions(&target_path, 0o600)?; + symlink(&target_path, &managed_path)?; + + let error = resolve_seed(&managed_path, false).unwrap_err(); + + assert!(error + .to_string() + .contains("open managed envd access-token seed")); + Ok(()) + } + + #[test] + fn oversized_managed_seed_is_rejected_before_reading_contents() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + create_private_managed_seed_directory(managed_path.parent().unwrap())?; + let file = File::create(&managed_path)?; + file.set_len(1024 * 1024)?; + set_permissions(&managed_path, 0o600)?; + + let error = resolve_seed(&managed_path, false).unwrap_err(); + + assert!(error.to_string().contains("must be at most 65 bytes")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn permissive_managed_seed_directory_is_rejected() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + fs::create_dir_all(managed_path.parent().unwrap())?; + set_permissions(managed_path.parent().unwrap(), 0o770)?; + + let error = resolve_seed(&managed_path, false).unwrap_err(); + + assert!(format!("{error:#}").contains("permissions 0700")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn managed_seed_directory_symlink_is_rejected() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + let target_directory = temp.path().join("target-secrets"); + create_private_managed_seed_directory(&target_directory)?; + symlink(&target_directory, managed_path.parent().unwrap())?; + + let error = resolve_seed(&managed_path, false).unwrap_err(); + + assert!(format!("{error:#}").contains("not a symbolic link")); + Ok(()) + } + + #[test] + fn missing_managed_seed_is_not_recreated_for_secure_state() -> Result<()> { + let temp = TempDir::new()?; + let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); + + let error = resolve_seed(&managed_path, true).unwrap_err(); + + assert!(error + .to_string() + .contains("persisted secure sandboxes exist")); + assert!(!managed_path.exists()); + Ok(()) + } } From 244e2a5abb9903b18b496b2ae46910cc85a1209d Mon Sep 17 00:00:00 2001 From: Sixing Lin <2682191498@qq.com> Date: Tue, 11 Aug 2026 23:18:54 +0800 Subject: [PATCH 3/4] fix(sandbox): reject overriding reserved MMDS fields --- src/sandbox/firecracker/mmds.rs | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/sandbox/firecracker/mmds.rs b/src/sandbox/firecracker/mmds.rs index a6b556b0..b39409d9 100644 --- a/src/sandbox/firecracker/mmds.rs +++ b/src/sandbox/firecracker/mmds.rs @@ -5,6 +5,7 @@ use crate::sandbox::EnvdAccessToken; use crate::types::SandboxId; const EMPTY_ACCESS_TOKEN: &str = ""; +const RESERVED_FIELDS: [&str; 4] = ["instanceID", "envID", "address", "accessTokenHash"]; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct MmdsMetadata { @@ -18,9 +19,10 @@ pub struct MmdsMetadata { pub access_token_hash: String, /// Extra key-value pairs merged into the top-level MMDS JSON object. /// API layers use this to pass opaque data (e.g. image configs) - /// through to the VM without the sandbox layer interpreting it. + /// through to the VM without the sandbox layer interpreting it. Reserved + /// MMDS fields are ignored so extras cannot override runtime identity or auth. #[serde(flatten)] - pub extra: serde_json::Map, + extra: serde_json::Map, } impl MmdsMetadata { @@ -35,7 +37,8 @@ impl MmdsMetadata { } /// Merge additional key-value pairs into the extra MMDS metadata. - pub fn with_extra(mut self, extra: serde_json::Map) -> Self { + pub fn with_extra(mut self, mut extra: serde_json::Map) -> Self { + extra.retain(|key, _| !RESERVED_FIELDS.contains(&key.as_str())); self.extra.extend(extra); self } @@ -118,6 +121,29 @@ mod tests { ); } + #[test] + fn extra_fields_cannot_override_reserved_fields() { + let sandbox_id = SandboxId::from_uuid(Uuid::nil()); + let token = crate::sandbox::SandboxAccessTokenGenerator::new("mmds-test-seed") + .unwrap() + .generate(sandbox_id); + let expected = + MmdsMetadata::new(sandbox_id, "snapshot-123").with_access_token(Some(&token)); + let mut extra = serde_json::Map::new(); + for field in RESERVED_FIELDS { + extra.insert(field.to_string(), json!("attacker-controlled")); + } + + let metadata = expected.clone().with_extra(extra); + let value = serde_json::to_value(&metadata).expect("serialize"); + + assert_eq!(metadata, expected); + assert_eq!(value["instanceID"], json!(sandbox_id.to_string())); + assert_eq!(value["envID"], json!("snapshot-123")); + assert_eq!(value["address"], json!("")); + assert_eq!(value["accessTokenHash"], json!(expected.access_token_hash)); + } + #[test] fn round_trips_with_extra_fields() { let sandbox_id = SandboxId::from_uuid(Uuid::nil()); From 30340ae353d17956ee3f03c017cbb0576004415a Mon Sep 17 00:00:00 2001 From: Sixing Lin <2682191498@qq.com> Date: Thu, 13 Aug 2026 22:10:55 +0800 Subject: [PATCH 4/4] feat(sandbox): make shared access token seed optional --- config/default.toml | 2 +- deploy/k8s/base/agentenv-daemonset.yaml | 1 + docs/src/SUMMARY.md | 4 ++ docs/src/configuration/env-vars.md | 2 +- docs/src/configuration/reference.md | 4 +- docs/src/deployment/docker-compose.md | 32 ++----------- docs/src/deployment/kubernetes.md | 35 +------------- docs/src/deployment/pvm.md | 3 ++ docs/src/deployment/static-multi-node.md | 12 ++--- docs/src/security/secure-sandboxes.md | 58 ++++++++++++++++++++++++ 10 files changed, 82 insertions(+), 71 deletions(-) create mode 100644 docs/src/security/secure-sandboxes.md diff --git a/config/default.toml b/config/default.toml index 5d28f511..fcaae5f8 100644 --- a/config/default.toml +++ b/config/default.toml @@ -190,7 +190,7 @@ poll_ms = 3 [sandbox] # Optional secret used to derive per-sandbox envd access tokens. When unset, # AgentENV creates a node-local seed under $AENV_HOME/secrets. Configure the -# same explicit value on every node before enabling cross-node sandbox recovery. +# same explicit value on every node when cross-node sandbox recovery is required. # access_token_hash_seed = "replace-with-a-secret" [orchestrator] diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index cd10a6c9..2f9c43f4 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -28,6 +28,7 @@ spec: secretKeyRef: name: agentenv-runtime-secrets key: sandbox-access-token-hash-seed + optional: true - name: AENV_VIRTUALIZATION_MODE value: "kvm" - name: API_ADDR diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 7d02dab9..f1d8d120 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -21,6 +21,10 @@ - [Configuration Reference](./configuration/reference.md) - [Environment Variables](./configuration/env-vars.md) +# Security + +- [Secure Sandboxes](./security/secure-sandboxes.md) + # Core Concepts - [How AgentENV Works](./concepts/overview.md) diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index 1253556f..8581458d 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -23,7 +23,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional override for the secret used to derive secure sandbox envd access tokens. Use the same explicit value on every node before enabling cross-node recovery of the same sandbox ID. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional override for the secret used to derive secure sandbox envd access tokens. Configure the same value on every node when cross-node recovery of the same sandbox ID is required; otherwise each node uses its own managed seed. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index 12fb2720..49cda48a 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -257,11 +257,11 @@ Sandbox control communication settings. | Key | Type | Default | Description | |-----|------|---------|-------------| -| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive secure sandbox envd access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. | +| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive secure sandbox envd access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value when the deployment needs to recover the same sandbox ID on another node. | The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. -Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every node to enable cross-node recovery of the same sandbox. +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every node when cross-node recovery of the same sandbox is required. Nodes use their own managed seed when it is unset. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 5d29b467..e39636a0 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -32,38 +32,16 @@ git clone https://github.com/kvcache-ai/AgentENV.git cd AgentENV ``` -## Configure the Shared Access-Token Seed +## Configure the Access-Token Seed (Optional) -Multi-node deployments should explicitly configure one envd access-token seed -and use the same value on every **runtime node**. Do not rely on the node-local -auto-generated seed in a scheduler-managed deployment. - -Generate the value once and store it in your secret manager: - -```bash -openssl rand -hex 32 -``` - -For the checked-in Compose stack, create a private configuration copy outside -the repository and set the generated value in it: - -```bash -install -d -m 0700 "$HOME/.config/agentenv" -install -m 0600 config/default.toml "$HOME/.config/agentenv/cluster.toml" -``` - -```toml -[sandbox] -access_token_hash_seed = "" -``` - -Changing this value rotates the access tokens for existing secure sandboxes. +See [Secure Sandboxes](../security/secure-sandboxes.md) +if the deployment needs future cross-node sandbox recovery. ## Start the Cluster ```bash sudo bash scripts/docker-setup.sh -CONFIG_PATH="$HOME/.config/agentenv/cluster.toml" make deploy-up +make deploy-up ``` To enable host-based sandbox data-plane URLs, set the shared sandbox proxy @@ -71,7 +49,7 @@ domain variable when starting the stack: ```bash SANDBOX_PROXY_DOMAINS=sandbox.example.com \ -CONFIG_PATH="$HOME/.config/agentenv/cluster.toml" make deploy-up +make deploy-up ``` Compose passes this value to both the gateway routing allowlist and runtime diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index bd77a962..4e72591a 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -43,40 +43,9 @@ make k8s-build This builds three images: `agentenv-runtime:latest`, `agentenv-gateway:latest`, and `agentenv-scheduler:latest`. -## Configure the Shared Access-Token Seed +## Configure the Access-Token Seed (Optional) -Before starting runtime Pods, create the namespace and generate one envd -access-token seed. Store it in the Secret required by the checked-in DaemonSet: - -```bash -kubectl apply -f deploy/k8s/base/namespace.yaml - -AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" -kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ - --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ - --dry-run=client -o yaml | kubectl apply -f - -unset AENV_ACCESS_TOKEN_HASH_SEED -``` - -Run this once for a new cluster. During upgrades, preserve the existing Secret -instead of generating another value. Production deployments may replace this -command with an external secret manager, but must provide the same Secret name -and key: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: agentenv-runtime-secrets - namespace: agentenv-system -stringData: - sandbox-access-token-hash-seed: -``` - -The DaemonSet injects this value into every runtime Pod. The Gateway and -Scheduler do not consume it. Changing the Secret rotates the access tokens for -existing secure sandboxes, so update it only as a coordinated credential -rotation. +See [Secure Sandboxes](../security/secure-sandboxes.md) for the optional shared seed configuration and Kubernetes Secret example. ## Deploy diff --git a/docs/src/deployment/pvm.md b/docs/src/deployment/pvm.md index fc91b8b7..47781d78 100644 --- a/docs/src/deployment/pvm.md +++ b/docs/src/deployment/pvm.md @@ -1,7 +1,9 @@ # PVM Deployment +> [!NOTE] > **Use this guide when standard KVM is unavailable**, which commonly happens on cloud VMs where nested virtualization is not exposed. If standard KVM already works, use the [Quick Start](../getting-started/quickstart.md) instead. +> [!WARNING] > This feature is **EXPERIMENTAL**. The PVM feature has not yet been merged into the mainline Linux kernel, and the forked kernel may not receive the same level of testing and security updates as the mainline kernel. PVM, originally proposed in the paper [*PVM: Efficient Shadow Paging for Deploying Secure Containers in Cloud-native Environment*](https://dl.acm.org/doi/10.1145/3600006.3613158), is an alternative virtualization mode that can provide the KVM-compatible interface required by AgentENV without relying on conventional nested virtualization. After the PVM host environment is installed, AgentENV still uses `/dev/kvm` to create Firecracker microVMs. @@ -27,6 +29,7 @@ You need: AgentENV does **not** replace the running host kernel automatically. Prebuilt PVM host-kernel packages are published separately in the [`kvcache-ai/linux` releases](https://github.com/kvcache-ai/linux/releases). You must install the appropriate package, reboot into that kernel, and verify the PVM module before installing AgentENV. +> [!WARNING] > Before changing kernels on a production server, confirm that you have console access or another recovery path in case the new kernel does not boot. ## Host and Guest Kernel Compatibility diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 8d5efe84..80533fff 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -61,14 +61,14 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in | sudo bash ``` -Edit `/etc/default/aenv` on each machine without removing the paths written by -the installer. Generate one access-token seed with `openssl rand -hex 32`, store -it securely, and set the same value on every runtime node. Node A uses: +Edit `/etc/default/aenv` on each machine without removing the paths written by the installer. +See [Secure Sandboxes](../security/secure-sandboxes.md) if the deployment needs future cross-node sandbox recovery. + +Node A uses: ```bash API_ADDR="0.0.0.0:8000" AENV_NODE_ID="node-a" -AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="" AENV_OBSERVABILITY_SCHEDULER_REPORT_ENABLED="true" AENV_OBSERVABILITY_SCHEDULER_ENDPOINT="http://10.0.0.10:9090" ``` @@ -78,14 +78,12 @@ Node B uses the same values except for its unique node ID: ```bash API_ADDR="0.0.0.0:8000" AENV_NODE_ID="node-b" -AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="" AENV_OBSERVABILITY_SCHEDULER_REPORT_ENABLED="true" AENV_OBSERVABILITY_SCHEDULER_ENDPOINT="http://10.0.0.10:9090" ``` The `AENV_NODE_ID` values must exactly match the corresponding IDs in the -Scheduler configuration below. Changing the shared seed rotates the access -tokens for existing secure sandboxes. Restart and verify each runtime: +Scheduler configuration below. Restart and verify each runtime: ```bash sudo systemctl restart aenv diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md new file mode 100644 index 00000000..9b763a4a --- /dev/null +++ b/docs/src/security/secure-sandboxes.md @@ -0,0 +1,58 @@ +# Secure Sandboxes + +Secure sandboxes use an envd access token for control-plane communication. This protects envd operations such as command execution and file access. + +> [!WARNING] +> This feature does not add authentication to application ports exposed by the sandbox. + +Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to enable secure mode. Or use the CLI: + +```bash +aenv start --secure +``` + +The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Forked sandboxes get independent tokens. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. + +## Access-Token Seed + +A seed is a random value used to derive the access token for each sandbox. This seed is optional. When it is unset, each runtime node automatically creates and persists a node-local seed under `$AENV_HOME/secrets`. +This is sufficient for normal single-node operation and does not require additional setup. + +Configure the same explicit seed on every runtime node when the deployment needs to recover the same sandbox ID on another node in the future. Generate it once and store it in the deployment's secret manager: + +```bash +openssl rand -hex 32 +``` + +Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on every runtime node. +For TOML configuration, use `[sandbox].access_token_hash_seed` instead. + +Preserve the seed across upgrades; changing it rotates access tokens for existing secure sandboxes. + +### Kubernetes + +The runtime DaemonSet reads the optional `agentenv-runtime-secrets` Secret. To configure a shared seed for all runtime Pods, create it before applying the runtime manifests: + +```bash +kubectl apply -f deploy/k8s/base/namespace.yaml + +AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ + --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ + --dry-run=client -o yaml | kubectl apply -f - +unset AENV_ACCESS_TOKEN_HASH_SEED +``` + +Run this once for a new cluster and preserve the existing Secret during upgrades. An external secret manager may be used instead, provided it creates the same Secret name and key: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: agentenv-runtime-secrets + namespace: agentenv-system +stringData: + sandbox-access-token-hash-seed: +``` + +If the Secret is not created, the DaemonSet still starts and each runtime Pod uses its automatically managed node-local seed.