diff --git a/Cargo.lock b/Cargo.lock index 94520db6..ab005734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,7 @@ dependencies = [ "sha2 0.10.9", "shell-util", "storage-util", + "subtle", "tar", "tempfile", "thiserror 2.0.18", @@ -8311,7 +8312,6 @@ dependencies = [ "serde_json", "storage-util", "tempfile", - "tikv-jemallocator", "tokio", "toml", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 62083f75..b596b9aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ io-uring = "0.7.9" tokio-tungstenite = "0.28" sha2 = "0.10" hmac = "0.12" +subtle = "2.6" hex = "0.4" semver = "1" iroh = { version = "=1.0.0-rc.0" } diff --git a/README.md b/README.md index 4c649223..0c162746 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ If your server does not support standard KVM, see the [PVM deployment guide](htt ## ⚡ Quick Start (Single Node) > [!WARNING] -> **AgentENV currently does not support authorization.** Do not expose the AgentENV -> API to the public network. Run it only on a trusted network or behind an -> authorization proxy with appropriate network controls. +> AgentENV authenticates API requests but does not encrypt traffic. Do not send +> the API key over an untrusted plaintext network. Run AgentENV on a trusted +> network or terminate HTTPS at a reverse proxy or load balancer. **1. Install and start the server** @@ -66,7 +66,7 @@ Set up the server: ```bash curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash docker pull ghcr.io/kvcache-ai/aenv-server:latest -docker run -d --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-server:latest +docker run -d --name aenv-server --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-server:latest ``` The server is accessible at `http://127.0.0.1:8000` by default. @@ -83,10 +83,23 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in **3. Authenticate** +The server generates an API key on its first startup. Retrieve it for the +installation method used in step 1: + +```bash +# Native install +sudo cat /var/lib/aenv/secrets/api-key + +# Docker +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Then run `aenv auth` and paste that key: + ```bash aenv auth # AENV server URL [http://localhost:8000]: http://127.0.0.1:8000 -# API key: dummy +# API key: ``` **4. Pull a template and run a sandbox** diff --git a/config/default.toml b/config/default.toml index 00f94bf7..2ec626f9 100644 --- a/config/default.toml +++ b/config/default.toml @@ -188,7 +188,7 @@ init_timeout_secs = 60 poll_ms = 3 [sandbox] -# Optional secret used to derive per-sandbox envd access tokens. When unset, +# Optional secret used to derive per-sandbox envd and traffic access tokens. When unset, # AgentENV creates a node-local seed under $AENV_HOME/secrets. Configure the # same explicit value on every node when cross-node sandbox recovery is required. # access_token_hash_seed = "replace-with-a-secret" diff --git a/crates/aenv/src/client/files.rs b/crates/aenv/src/client/files.rs index ac5173ae..39be8490 100644 --- a/crates/aenv/src/client/files.rs +++ b/crates/aenv/src/client/files.rs @@ -17,7 +17,6 @@ use super::Client; use crate::grpc::{RpcError, Transport, ENVD_PORT_STR}; use crate::progress::TransferProgress; -const API_KEY_HEADER: &str = "X-API-Key"; const SANDBOX_ID_HEADER: &str = "x-agentenv-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; const ACCESS_TOKEN_HEADER: &str = "X-Access-Token"; @@ -32,17 +31,8 @@ pub struct EnvdFilesClient { } impl EnvdFilesClient { - fn new( - base_url: &str, - api_key: &str, - sandbox_id: &str, - envd_access_token: Option<&str>, - ) -> Result { + fn new(base_url: &str, sandbox_id: &str, envd_access_token: Option<&str>) -> Result { let mut headers = HeaderMap::new(); - headers.insert( - API_KEY_HEADER, - HeaderValue::from_str(api_key).context("invalid API key header value")?, - ); headers.insert( SANDBOX_ID_HEADER, HeaderValue::from_str(sandbox_id).context("invalid sandbox ID header value")?, @@ -72,7 +62,7 @@ impl EnvdFilesClient { Ok(Self { base_url: base_url.trim_end_matches('/').to_string(), http: client, - transport: Transport::new(base_url, api_key, sandbox_id, envd_access_token)?, + transport: Transport::new(base_url, sandbox_id, envd_access_token)?, }) } @@ -311,12 +301,7 @@ fn format_envd_response_error(status: reqwest::StatusCode, content: &str) -> any impl Client { pub fn files(&self, sandbox_id: &str) -> Result { let sandbox = self.get_sandbox(sandbox_id)?; - EnvdFilesClient::new( - &self.base, - &self.api_key, - sandbox_id, - sandbox.envd_access_token.as_deref(), - ) + EnvdFilesClient::new(&self.base, sandbox_id, sandbox.envd_access_token.as_deref()) } } diff --git a/crates/aenv/src/client/mod.rs b/crates/aenv/src/client/mod.rs index 55dba491..c384e964 100644 --- a/crates/aenv/src/client/mod.rs +++ b/crates/aenv/src/client/mod.rs @@ -40,7 +40,7 @@ impl Client { sandbox_id: &str, envd_access_token: Option<&str>, ) -> Result { - Transport::new(&self.base, &self.api_key, sandbox_id, envd_access_token) + Transport::new(&self.base, sandbox_id, envd_access_token) } fn url(&self, path: &str) -> String { diff --git a/crates/aenv/src/grpc/mod.rs b/crates/aenv/src/grpc/mod.rs index 64336c33..cdc24e83 100644 --- a/crates/aenv/src/grpc/mod.rs +++ b/crates/aenv/src/grpc/mod.rs @@ -54,23 +54,16 @@ impl std::error::Error for RpcError {} pub struct Transport { http: HttpClient, base_url: String, - api_key: String, sandbox_id: String, envd_access_token: Option, } impl Transport { - pub fn new( - base_url: &str, - api_key: &str, - sandbox_id: &str, - envd_access_token: Option<&str>, - ) -> Result { + pub fn new(base_url: &str, sandbox_id: &str, envd_access_token: Option<&str>) -> Result { let http = Self::http_client(base_url).context("building Connect-RPC HTTP client")?; Ok(Self { http, base_url: base_url.trim_end_matches('/').to_string(), - api_key: api_key.to_string(), sandbox_id: sandbox_id.to_string(), envd_access_token: envd_access_token.map(str::to_owned), }) @@ -102,7 +95,6 @@ impl Transport { fn auth(&self, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { let builder = builder - .header("X-API-Key", &self.api_key) .header("x-agentenv-sandbox-id", &self.sandbox_id) .header("x-agentenv-target-port", ENVD_PORT_STR) .header("Connect-Protocol-Version", "1"); @@ -508,7 +500,7 @@ mod tests { #[test] fn unary_user_is_sent_as_basic_auth() { - let transport = Transport::new("http://127.0.0.1", "api-key", "sandbox-id", None).unwrap(); + let transport = Transport::new("http://127.0.0.1", "sandbox-id", None).unwrap(); let request = transport .unary_request("filesystem.Filesystem", "Stat", Some("app")) .build() @@ -518,6 +510,7 @@ mod tests { request.headers().get(AUTHORIZATION).unwrap(), "Basic YXBwOg==" ); + assert!(!request.headers().contains_key("x-api-key")); let request = transport .unary_request("filesystem.Filesystem", "Stat", None) @@ -528,13 +521,8 @@ mod tests { #[test] fn envd_access_token_is_sent_on_connect_requests() { - let transport = Transport::new( - "http://127.0.0.1", - "api-key", - "sandbox-id", - Some("envd-token"), - ) - .unwrap(); + let transport = + Transport::new("http://127.0.0.1", "sandbox-id", Some("envd-token")).unwrap(); let request = transport .unary_request("process.Process", "List", None) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 52130b7c..7f06bc38 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -3,6 +3,7 @@ x-agentenv-base: &agentenv-base init: true working_dir: /workspace environment: &agentenv-environment + AENV_API_KEY: AENV_CONFIG_PATH: /workspace/config/default.toml AENV_VIRTUALIZATION_MODE: ${AENV_VIRTUALIZATION_MODE:-kvm} API_ADDR: 0.0.0.0:8000 @@ -16,8 +17,9 @@ x-agentenv-base: &agentenv-base - /dev:/dev - ${CONFIG_PATH:-../config/default.toml}:/workspace/config/default.toml:ro # Runtime assets are baked into the image by `server --setup-only`; compose - # persists only committed snapshots across container restarts. + # persists committed snapshots and deployment secrets across restarts. - agentenv-snapshot-store:/workspace/env/snapshot-store + - agentenv-auth:/workspace/env/secrets devices: - /dev/kvm:/dev/kvm privileged: true @@ -67,8 +69,11 @@ services: depends_on: scheduler: condition: service_healthy - volumes: *control-plane-config-volume + volumes: + - ./docker/config/default.json:/config/default.json:ro + - agentenv-auth:/run/secrets:ro environment: + AENV_API_KEY: GATEWAY_HTTP_LISTEN_ADDR: :8080 GATEWAY_SCHEDULER_ADDR: scheduler:9090 GATEWAY_SANDBOX_PROXY_DOMAINS: ${SANDBOX_PROXY_DOMAINS:-} @@ -96,4 +101,5 @@ services: AENV_NODE_ID: node-b volumes: + agentenv-auth: agentenv-snapshot-store: diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index 2f9c43f4..470b2a9c 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -21,6 +21,11 @@ spec: image: agentenv-runtime:latest imagePullPolicy: IfNotPresent env: + - name: AENV_API_KEY + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_API_KEY - name: AENV_CONFIG_PATH value: /workspace/config/agentenv.toml - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED @@ -76,7 +81,7 @@ spec: - | echo "preStop: waiting for sandboxes to drain..." while true; do - count=$(curl -sf -H 'X-API-Key: preStop' http://localhost:8000/sandboxes | jq 'length') || count="" + count=$(curl -sf -H "X-API-Key: ${AENV_API_KEY}" http://localhost:8000/sandboxes | jq 'length') || count="" if [ -z "$count" ]; then echo "preStop: failed to query sandbox count, retrying..." sleep 3 diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 81f222c8..08536fc0 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -27,6 +27,11 @@ spec: - name: http containerPort: 8080 env: + - name: AENV_API_KEY + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_API_KEY - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 797a3bb7..556f0d70 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -30,6 +30,11 @@ configMapGenerator: literals: - SANDBOX_PROXY_DOMAINS= +secretGenerator: + - name: agentenv-auth + literals: + - AENV_API_KEY= + images: - name: agentenv-gateway newName: agentenv-gateway diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index d0322c01..e891b2f0 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -8,8 +8,48 @@ fi MODE="$1" shift +case "${MODE}" in + render|apply|delete) ;; + *) + echo "unsupported mode: ${MODE}" >&2 + exit 1 + ;; +esac + KUBECTL_BIN="${KUBECTL:-kubectl}" OVERLAY_NAME="${K8S_OVERLAY:-default}" +NAMESPACE="${K8S_NAMESPACE:-agentenv-system}" +if [[ ${#NAMESPACE} -gt 63 || ! "${NAMESPACE}" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "K8S_NAMESPACE must be a valid Kubernetes namespace name" >&2 + exit 1 +fi + +KUBECTL_TARGET_ARGS=() +DRY_RUN=0 +ARGS=("$@") +for ((i = 0; i < ${#ARGS[@]}; i++)); do + arg="${ARGS[i]}" + case "${arg}" in + --context|--kubeconfig) + if ((i + 1 >= ${#ARGS[@]})); then + echo "${arg} requires a value" >&2 + exit 1 + fi + KUBECTL_TARGET_ARGS+=("${arg}" "${ARGS[i + 1]}") + i=$((i + 1)) + ;; + --context=*|--kubeconfig=*) KUBECTL_TARGET_ARGS+=("${arg}") ;; + --dry-run) + if ((i + 1 < ${#ARGS[@]})) && [[ "${ARGS[i + 1]}" == "none" ]]; then + DRY_RUN=0 + else + DRY_RUN=1 + fi + ;; + --dry-run=client|--dry-run=server) DRY_RUN=1 ;; + --dry-run=none) DRY_RUN=0 ;; + esac +done SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" @@ -26,8 +66,121 @@ sed_in_place() { fi } +render_api_key() { + local file="$1" + local rendered_file + + rendered_file="$(mktemp "${TEMP_DIR}/api-key.XXXXXX")" + if ! { + printf '%s\n' "${API_KEY_VALUE}" + cat "${file}" + } | awk ' + NR == 1 { api_key = $0; next } + /^ - AENV_API_KEY=/ { print " - AENV_API_KEY=" api_key; replaced = 1; next } + { print } + END { if (!replaced) exit 1 } + ' >"${rendered_file}"; then + return 1 + fi + mv "${rendered_file}" "${file}" +} + cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" +OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" +if [[ ! -d "${OVERLAY_PATH}" ]]; then + echo "unknown overlay: ${OVERLAY_NAME}" >&2 + exit 1 +fi +sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/kustomization.yaml" +sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${OVERLAY_PATH}/kustomization.yaml" +sed_in_place "s# name: agentenv-system# name: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/namespace.yaml" +sed_in_place "s#\"namespace\": \"agentenv-system\"#\"namespace\": \"${NAMESPACE}\"#" "${TEMP_DIR}/k8s/base/config/scheduler.json" + +read_existing_api_key() { + local value="" + + if ! value="$("${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY" | base64decode}}')"; then + echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 + return 1 + fi + printf '%s' "${value}" +} + +ensure_namespace() { + if ! "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" create namespace "${NAMESPACE}" \ + --dry-run=client -o yaml | "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" apply -f - >/dev/null; then + echo "failed to create or verify namespace ${NAMESPACE}" >&2 + return 1 + fi +} + +generate_api_key() { + printf 'e2b_%s' "$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" +} + +bootstrap_api_key() { + local create_error secret_file + + if ! API_KEY_VALUE="$(read_existing_api_key)"; then + return 1 + fi + if [[ -n "${API_KEY_VALUE}" ]]; then + return 0 + fi + + secret_file="${TEMP_DIR}/bootstrap-api-key" + create_error="${TEMP_DIR}/bootstrap-api-key.err" + generate_api_key >"${secret_file}" + chmod 0600 "${secret_file}" + + # A concurrent apply can win this create. The persisted reread below is + # authoritative whether this command succeeds or reports AlreadyExists. + "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" -n "${NAMESPACE}" create secret generic agentenv-auth \ + --from-file="AENV_API_KEY=${secret_file}" >/dev/null 2>"${create_error}" || true + + if ! API_KEY_VALUE="$(read_existing_api_key)"; then + return 1 + fi + if [[ -z "${API_KEY_VALUE}" ]]; then + cat "${create_error}" >&2 + echo "failed to bootstrap Secret ${NAMESPACE}/agentenv-auth" >&2 + return 1 + fi +} + +if [[ "${MODE}" != "delete" ]]; then + restore_xtrace=0 + if [[ $- == *x* ]]; then + restore_xtrace=1 + set +x + fi + API_KEY_VALUE="" + if [[ "${MODE}" == "render" ]]; then + API_KEY_VALUE="REDACTED" + else + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + if [[ -z "${AENV_API_KEY}" ]]; then + echo "AENV_API_KEY must not be empty" >&2 + exit 1 + fi + API_KEY_VALUE="${AENV_API_KEY}" + elif [[ "${DRY_RUN}" == "1" ]]; then + API_KEY_VALUE="$(generate_api_key)" + else + ensure_namespace || exit 1 + bootstrap_api_key || exit 1 + fi + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,256}$ ]]; then + echo "AENV_API_KEY must contain between 32 and 256 URL-safe characters" >&2 + exit 1 + fi + fi + + render_api_key "${TEMP_DIR}/k8s/base/kustomization.yaml" + [[ "${restore_xtrace}" == "0" ]] || set -x +fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then ESCAPED_SANDBOX_PROXY_DOMAINS="${SANDBOX_PROXY_DOMAINS//\\/\\\\}" @@ -36,12 +189,6 @@ if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then sed_in_place "s#- SANDBOX_PROXY_DOMAINS=.*#- SANDBOX_PROXY_DOMAINS=${ESCAPED_SANDBOX_PROXY_DOMAINS}#" "${TEMP_DIR}/k8s/base/kustomization.yaml" fi -OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" -if [[ ! -d "${OVERLAY_PATH}" ]]; then - echo "unknown overlay: ${OVERLAY_NAME}" >&2 - exit 1 -fi - if [[ "${OVERLAY_NAME}" == "local-dev" ]]; then REPO_ENV_PATH="${AENV_LOCAL_REPO_ENV_PATH:-${REPO_ROOT}/env}" if [[ ! -d "${REPO_ENV_PATH}" ]]; then @@ -65,12 +212,18 @@ case "${MODE}" in ;; apply) "${KUBECTL_BIN}" apply -k "${OVERLAY_PATH}" "$@" + if [[ "${DRY_RUN}" == "0" ]]; then + "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" -n "${NAMESPACE}" rollout restart \ + deployment/agentenv-gateway daemonset/agentenv-node + echo "AgentENV API key stored in Secret ${NAMESPACE}/agentenv-auth." >&2 + echo "Read it with:" >&2 + printf ' %q' "${KUBECTL_BIN}" "${KUBECTL_TARGET_ARGS[@]}" >&2 + printf " -n %q get secret agentenv-auth -o go-template='%s'\n" \ + "${NAMESPACE}" \ + '{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' >&2 + fi ;; delete) "${KUBECTL_BIN}" delete --ignore-not-found -k "${OVERLAY_PATH}" "$@" ;; - *) - echo "unsupported mode: ${MODE}" >&2 - exit 1 - ;; esac diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index f1d8d120..a3494dad 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -23,6 +23,7 @@ # Security +- [Authentication](./security/authentication.md) - [Secure Sandboxes](./security/secure-sandboxes.md) # Core Concepts diff --git a/docs/src/concepts/proxy.md b/docs/src/concepts/proxy.md index 20ebff43..aca0824b 100644 --- a/docs/src/concepts/proxy.md +++ b/docs/src/concepts/proxy.md @@ -33,6 +33,23 @@ E2B-compatible aliases are also accepted: These routing headers are stripped before the request is forwarded to the sandbox. +## Access Control + +Proxy authentication is independent from AgentENV API authentication: + +- Public application ingress (`allowPublicTraffic: true`, the default) requires + no AgentENV credential. +- Private application ingress (`allowPublicTraffic: false`) requires the + sandbox's `trafficAccessToken` in `e2b-traffic-access-token`. +- Secure envd traffic requires the sandbox's `envdAccessToken` in + `X-Access-Token`. Insecure envd traffic has no envd token. + +`X-API-Key` authenticates AgentENV control-plane APIs only. It does not grant +access to private application ingress or secure envd. A matching platform key +is stripped on proxy requests; other `X-API-Key` values remain available to +sandbox applications. AgentENV also strips the traffic token, and forwards +`X-Access-Token` only to the matching secure envd port. + Host-based proxy requests derive both values from `Host`, for example `http://8080-.sandbox.example.com/health` targets port `8080`. The configured domain must route to the AgentENV server in single-node mode or diff --git a/docs/src/concepts/sandboxes.md b/docs/src/concepts/sandboxes.md index ab6be6ea..31f91e5b 100644 --- a/docs/src/concepts/sandboxes.md +++ b/docs/src/concepts/sandboxes.md @@ -59,7 +59,7 @@ aenv start --cold ubuntu:24.04 The cold-start API accepts an optional `diskSizeMB` field to set the root filesystem's virtual size in MiB. Explicit values must be at least 1024 MiB and divisible by 1024 because the current resize tool operates at 1 GiB granularity. Growth is allowed by default; shrinking below the source image size requires `ublk.overlaybd.allow_shrink = true`. If omitted, the image's built-in virtual size is used. Resizing applies only when creating a fresh writable root filesystem, not to read-only images, images with an existing upper, or snapshot resume. Sandbox responses also report disk size as `diskSizeMB`. -Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port only; it does not add authentication to application ports. Each fork derives a distinct envd token from the child sandbox ID. +Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port, while application proxy requests use the independent sandbox-scoped traffic token. Each fork derives distinct envd and traffic credentials from the child sandbox ID. --- diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index 8581458d..755f5a08 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -12,6 +12,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | Variable | Default | Description | |----------|---------|-------------| +| `AENV_API_KEY` | generated under `$AENV_HOME/secrets/api-key` | Optional API-key override. Runtime nodes also check `/run/secrets/api-key` before creating a managed key. Use one shared value or secret in multi-node deployments. | | `API_ADDR` | `0.0.0.0:8000` | Address and port the API server listens on | | `AENV_CONFIG_PATH` | `config/default.toml` | Path to the TOML configuration file | | `AENV_LOG_FORMAT` | `compact` | Server log output format: `compact`, `pretty`, or `json` | @@ -23,7 +24,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional override for the secret used to derive secure sandbox envd access tokens. Configure the same value on every node when cross-node recovery of the same sandbox ID is required; otherwise each node uses its own managed seed. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Configure the same value on every runtime node in clustered deployments. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | @@ -46,8 +47,7 @@ These variables configure the E2B SDK and CLI to point at an AgentENV server. Va |----------|-------------| | `E2B_API_URL` | AgentENV server API base URL | | `E2B_SANDBOX_URL` | Sandbox proxy URL (for WebSocket and process interaction) | -| `E2B_API_KEY` | API key for authentication | -| `E2B_ACCESS_TOKEN` | Access token (used by `e2b template` commands) | +| `E2B_API_KEY` | Set to the deployment's `AENV_API_KEY` | ### Values by Deployment Mode @@ -56,8 +56,7 @@ These variables configure the E2B SDK and CLI to point at an AgentENV server. Va ```bash export E2B_API_URL=http://127.0.0.1:8000 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` **Docker Compose / Kubernetes (multi-node)**: @@ -65,15 +64,14 @@ export E2B_ACCESS_TOKEN=dummy ```bash export E2B_API_URL=http://127.0.0.1:8080 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` > In both modes, sandbox data-plane requests can use routing headers with > `E2B_SANDBOX_URL=${E2B_API_URL}`. The explicit `/proxy` prefix > (`${E2B_API_URL}/proxy`) is still accepted for back-compat. -> For local development, any non-empty value works for `E2B_API_KEY` and `E2B_ACCESS_TOKEN` because the server only checks that the auth header is present. +See [Authentication](../security/authentication.md) for key generation and storage. ## Gateway and Scheduler @@ -88,6 +86,7 @@ These variables apply to both the gateway and scheduler processes. | Variable | Default | Description | |----------|---------|-------------| +| `AENV_API_KEY` | unset | Shared single-tenant API key. The gateway uses the environment value when set, otherwise it reads `/run/secrets/api-key`. | | `GATEWAY_HTTP_LISTEN_ADDR` | `:8080` | HTTP listen address | | `GATEWAY_METRICS_LISTEN_ADDR` | `:9102` | Prometheus metrics listen address | | `GATEWAY_SCHEDULER_ADDR` | `127.0.0.1:9090` | Scheduler gRPC address for routing and node lookup | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index d2ad8049..7c4b609d 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -257,11 +257,11 @@ Sandbox control communication settings. | Key | Type | Default | Description | |-----|------|---------|-------------| -| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive secure sandbox envd access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value when the deployment needs to recover the same sandbox ID on another node. | +| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | -The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. +The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure or private-ingress sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. -Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every node when cross-node recovery of the same sandbox is required. Nodes use their own managed seed when it is unset. +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 5b0bf38b..22648ae9 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -23,11 +23,6 @@ git clone https://github.com/kvcache-ai/AgentENV.git cd AgentENV ``` -## Configure the Access-Token Seed (Optional) - -See [Secure Sandboxes](../security/secure-sandboxes.md) -if the deployment needs future cross-node sandbox recovery. - ## Start the Cluster ```bash @@ -35,8 +30,10 @@ sudo bash scripts/docker-setup.sh make deploy-up ``` -The Gateway is available at `http://127.0.0.1:8000` and forwards requests to -the backend nodes. +On first startup, the runtime nodes atomically generate one API key and sandbox +access-token seed in the shared `agentenv-auth` volume. The gateway mounts that +volume read-only and reads the API key; sandbox tokens are validated by the +runtime nodes. Normal `make deploy-down` calls preserve both values. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: @@ -56,8 +53,13 @@ usually through wildcard DNS for `*.sandbox.example.com`. # Health check via gateway curl http://127.0.0.1:8000/health -# Cluster node snapshots via gateway -curl http://127.0.0.1:8000/nodes +# Authenticated cluster node snapshots via gateway +export AENV_API_KEY="$(docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ + cat /workspace/env/secrets/api-key)" +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes + +# Direct health check on a backend node +curl http://127.0.0.1:8000/health ``` ## Management Commands @@ -68,6 +70,32 @@ make deploy-logs # Stream logs from all services make deploy-down # Tear down the cluster ``` +Removing Compose volumes with `docker compose down -v` also removes both +secrets. The next startup generates new values, so existing clients and sandbox +access tokens are invalidated. + +To provide an existing key through Docker Compose secrets, add a file-backed +secret in an override file and mount it with `target: api-key` on the gateway +and both runtime nodes. AgentENV automatically reads `/run/secrets/api-key`; +no file-path environment variable is needed. + +```yaml +services: + gateway: + secrets: [api-key] + agentenv-a: + secrets: [api-key] + agentenv-b: + secrets: [api-key] + +secrets: + api-key: + file: ./api-key +``` + +The secret name is also its default target filename, so this mounts the key at +`/run/secrets/api-key` in each service. + ## Configuration Container deployments use `deploy/docker/config/default.json`. Scheduler and backend node endpoints are configured for the Docker network. diff --git a/docs/src/deployment/docker.md b/docs/src/deployment/docker.md index 84d01578..b7c5ffbf 100644 --- a/docs/src/deployment/docker.md +++ b/docs/src/deployment/docker.md @@ -44,6 +44,7 @@ docker build \ ```bash docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -p 8000:8000 \ ghcr.io/kvcache-ai/aenv-server:latest # or aenv:latest if built from source @@ -51,6 +52,17 @@ docker run --rm -it \ The `--privileged` flag is required for Firecracker's network namespace operations (veth pairs, iptables). The server auto-downloads runtime assets on first start and is accessible at `http://127.0.0.1:8000` once ready. +On normal startup, the server generates the API key inside the container at +`/workspace/env/secrets/api-key`. Read it while the container is running with: + +```bash +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Removing the container also removes this generated key. Supply an explicit +`AENV_API_KEY` or a secret at `/run/secrets/api-key` when the key must remain +stable across container replacements. + ## Verify ```bash diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index e622b532..63c05854 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -61,6 +61,21 @@ make k8s-render make k8s-apply ``` +`make k8s-apply` generates a 256-bit API key on the first deployment and stores +it in `Secret/agentenv-auth`. Later applies reuse it. Read the key locally when +configuring clients: + +```bash +kubectl -n agentenv-system get secret agentenv-auth \ + -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' +``` + +Set `AENV_API_KEY` when applying to supply your own value. A standalone +`make k8s-render` uses an invalid `REDACTED` placeholder so preview output never +contains a deployable API key. The optional runtime seed keeps its existing +`agentenv-runtime-secrets` contract described in +[Secure Sandboxes](../security/secure-sandboxes.md). + To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: @@ -101,9 +116,9 @@ make k8s-delete A dedicated `local-dev` overlay mounts the repository's `env/` directory directly into the DaemonSet at `/workspace/env`, avoiding runtime asset copies: -This overlay also generates `agentenv-runtime-secrets` with a fixed test-only -seed so local and E2E deployments do not require production secret management. -Do not reuse that value outside local development. +The apply helper provisions the same generated API key used by the default +overlay. The local development overlay retains its fixed test-only runtime seed; +do not reuse that seed outside local development. ```bash make k8s-build diff --git a/docs/src/deployment/manual-compile.md b/docs/src/deployment/manual-compile.md index 6f1c447e..9b287e0b 100644 --- a/docs/src/deployment/manual-compile.md +++ b/docs/src/deployment/manual-compile.md @@ -33,6 +33,9 @@ make release ## Start the Server +Start the server. On first normal startup it generates an API key under +`$AENV_HOME/secrets/api-key` and reuses it on later starts: + ```bash # Debug build API_ADDR=0.0.0.0:8000 make start-server @@ -41,14 +44,22 @@ API_ADDR=0.0.0.0:8000 make start-server API_ADDR=0.0.0.0:8000 make start-server-release ``` -The server auto-downloads runtime assets (Firecracker binary, kernel, rootfs) on first start. Once ready, it listens at `http://127.0.0.1:8000`. +The server auto-downloads runtime assets (Firecracker binary, kernel, rootfs) on first start. Once ready, it listens at `http://127.0.0.1:8000`. Read the generated key before making authenticated requests: + +```bash +export AENV_API_KEY="$(cat "${AENV_HOME_PATH:-/var/lib/aenv}/secrets/api-key")" +``` ## Verify ```bash curl http://127.0.0.1:8000/health +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8000/sandboxes ``` +HTTP does not protect the key in transit. Use a trusted network, VPN, or +TLS-terminating reverse proxy for remote clients. + ## Configuration The server reads `config/default.toml` by default. Override with: diff --git a/docs/src/deployment/pvm.md b/docs/src/deployment/pvm.md index 47781d78..1b896a65 100644 --- a/docs/src/deployment/pvm.md +++ b/docs/src/deployment/pvm.md @@ -197,7 +197,7 @@ Use the dedicated PVM image: ```bash docker pull ghcr.io/kvcache-ai/aenv-server:latest-pvm -docker run --rm -it \ +docker run --rm -it --name aenv-server \ --device /dev/kvm \ --privileged \ -v /dev:/dev \ @@ -218,6 +218,9 @@ cargo run --bin server -- --setup-only make start-server ``` +The server generates and persists the API key under +`$AENV_HOME/secrets/api-key` on its first normal startup. + You can also set the mode in the TOML configuration: ```toml diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 5ecc6c3d..354cd2f3 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -21,9 +21,9 @@ AgentENV runtime nodes: | Runtime node A | `10.0.0.21:8000` | Runs Firecracker sandboxes as `node-a` | | Runtime node B | `10.0.0.22:8000` | Runs Firecracker sandboxes as `node-b` | -Use private addresses or an otherwise trusted network. AgentENV does not -currently provide an authentication boundary suitable for exposing these -services directly to the public Internet. +AgentENV authenticates HTTP requests but does not encrypt them. Use private +addresses, a VPN, or TLS termination before traffic crosses an untrusted +network. ## Prerequisites @@ -55,15 +55,25 @@ an external metrics collector needs them. ## 1. Install the runtime nodes +Generate one API key and one sandbox access-token seed through your normal +secret-management channel. Use the API key on the gateway and every runtime +node; use the seed only on runtime nodes: + +```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +``` + Run the installation on each runtime node: ```bash -curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh \ - | sudo bash +curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh | sudo bash ``` -Edit `/etc/default/aenv` on each machine without removing the paths written by the installer. -See [Secure Sandboxes](../security/secure-sandboxes.md) if the deployment needs future cross-node sandbox recovery. +Edit `/etc/default/aenv` on each machine without removing the paths written by +the installer, and add both shared values before starting the services. A +multi-node deployment must not let each node generate independent managed +secrets. Node A uses: @@ -107,6 +117,17 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` +Create `/etc/agentenv/auth.env` with the API key used on the runtime nodes: + +```bash +sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env +sudoedit /etc/agentenv/auth.env +``` + +```text +AENV_API_KEY= +``` + If the `agentenv-control` account already exists, the `useradd` command reports that fact and can be skipped. @@ -192,6 +213,7 @@ After=network-online.target agentenv-scheduler.service [Service] User=agentenv-control Group=agentenv-control +EnvironmentFile=/etc/agentenv/auth.env ExecStart=/usr/local/bin/agentenv-gateway -config /etc/agentenv/control-plane.json Restart=on-failure RestartSec=5 @@ -219,7 +241,8 @@ curl http://10.0.0.22:8000/health curl http://127.0.0.1:8080/health # Wait for node heartbeats, then inspect the cluster through the Gateway -curl http://127.0.0.1:8080/nodes +export AENV_API_KEY="$(sudo sed -n 's/^AENV_API_KEY=//p' /etc/agentenv/auth.env)" +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes ``` The node list should contain `node-a` and `node-b`. Point clients at the @@ -228,7 +251,7 @@ Gateway, not directly at a runtime node: ```bash aenv auth # AENV server URL: http://10.0.0.10:8080 -# API key: dummy +# API key: ``` Sandbox create, list, lifecycle, and data-plane requests can then be routed diff --git a/docs/src/getting-started/aenv-cli.md b/docs/src/getting-started/aenv-cli.md index 7d1227ac..79500c7e 100644 --- a/docs/src/getting-started/aenv-cli.md +++ b/docs/src/getting-started/aenv-cli.md @@ -27,7 +27,7 @@ Save the server URL and API key. Credentials are stored at `~/.config/aenv/crede ```bash aenv auth # AENV server URL [http://localhost:8000]: The address of the AgentENV server -# API key: dummy (Any non-empty string works for local development.) +# API key: ``` --- diff --git a/docs/src/getting-started/quickstart.md b/docs/src/getting-started/quickstart.md index 0db8b658..9ebd1757 100644 --- a/docs/src/getting-started/quickstart.md +++ b/docs/src/getting-started/quickstart.md @@ -50,6 +50,7 @@ default. Transient namespace and daemon-socket state lives under `/run/aenv`. curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash docker pull ghcr.io/kvcache-ai/aenv-server:latest docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -p 8000:8000 \ ghcr.io/kvcache-ai/aenv-server:latest @@ -61,6 +62,7 @@ To customize the server configuration, download and edit the configuration file, curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/config/default.toml -o config.toml vim config.toml docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -v "$PWD/config.toml:/workspace/config/default.toml:ro" \ -p 8000:8000 \ @@ -87,14 +89,23 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in ### 3. Authenticate +The server generates the key on its first normal startup. Native installations +reuse the managed key; a normal Docker container keeps it in its writable +container layer: + +```bash +# Native +sudo cat /var/lib/aenv/secrets/api-key +# Docker +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + ```bash aenv auth # AENV server URL [http://localhost:8000]: http://127.0.0.1:8000 -# API key: dummy +# API key: ``` -For local development, any non-empty string works as the API key. - ### 4. Pull a template and run a sandbox ```bash diff --git a/docs/src/integration/e2b.md b/docs/src/integration/e2b.md index 6a6db325..8650d859 100644 --- a/docs/src/integration/e2b.md +++ b/docs/src/integration/e2b.md @@ -12,10 +12,16 @@ Set environment variables to point at your AgentENV server. See [Environment Var # Single-node example export E2B_API_URL=http://127.0.0.1:8000 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` +AgentENV returns `trafficAccessToken` when `network.allowPublicTraffic` is false +and (for secure sandboxes) +`envdAccessToken` for envd control traffic. These credentials have different +headers and trust boundaries: use `e2b-traffic-access-token` for private +application routes and `X-Access-Token` only for envd. Public application +routes require neither token. + ### TypeScript SDK #### Setup diff --git a/docs/src/internals/architecture.md b/docs/src/internals/architecture.md index 1cf4fa21..3f4399f5 100644 --- a/docs/src/internals/architecture.md +++ b/docs/src/internals/architecture.md @@ -220,10 +220,11 @@ Discovery modes: **Deployment**: ```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" # shared by local runtime and gateway processes # local dev (single node) make start-server && make -C services run-scheduler && make -C services run-gateway -# docker compose (multi-node) +# docker compose (multi-node; shared auth volume is provisioned automatically) make deploy-up # gateway + scheduler + 2 backend nodes make deploy-down # teardown diff --git a/docs/src/internals/persistence-artifact-inventory.md b/docs/src/internals/persistence-artifact-inventory.md index eefdb882..e1291be9 100644 --- a/docs/src/internals/persistence-artifact-inventory.md +++ b/docs/src/internals/persistence-artifact-inventory.md @@ -9,7 +9,7 @@ This document lists AgentENV artifacts that can remain on disk or in object stor | `home_path` | `/var/lib/aenv` | `src/cfg.rs` | Base for paths containing the literal `$AENV_HOME` placeholder. `AENV_HOME_PATH` overrides it before placeholder expansion. | | `runtime_path` | `/run/aenv` | `src/cfg.rs`, `src/sandbox/network/*` | Base for transient namespace mount points and daemon sockets. `AENV_RUNTIME_PATH` overrides it. | | `deps_path` | `$AENV_HOME/deps` | `src/cfg.rs`, `src/setup/*` | Base for downloaded runtime dependencies. `AENV_DEPS_PATH` can place these rebuildable assets outside `home_path`. | -| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure sandboxes. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure or private-ingress sandboxes. | | Firecracker sandbox work dirs | `$AENV_HOME/firecracker-work` with `agentenv-fc-` children | `src/sandbox/firecracker/*` | Per-sandbox runtime directories for sockets, symlinks, ublk runtime dirs, local logs, and writable OverlayBD upper layer data (`overlaybd/upper.data`, `overlaybd/upper.index`). An explicit `[firecracker].work_dir` overrides the root. | | `firecracker.serial_dir` | `$AENV_HOME/logs/serial` | `src/sandbox/firecracker/*` | Durable Firecracker stdout/stderr root, grouped by sandbox ID. An explicit `[firecracker].serial_dir` overrides the root. | | `managed_snapshot_root` | `/managed-snapshots` | `src/sandbox/firecracker/*` | In-process live snapshot artifact root used to keep captured snapshots alive until publish or drop. | @@ -37,7 +37,7 @@ Owned by `src/setup/*` and `src/cfg.rs`. | Overlaybd package downloads | `/overlaybd/downloads/*` | Temporary downloaded package archives | Setup staging for overlaybd release packages | Removed after a successful install. | | Generated overlaybd config | `$AENV_HOME/overlaybd/overlaybd-global.json`, `$AENV_HOME/overlaybd/mem-overlaybd-global.json`, `$AENV_HOME/overlaybd/convert-overlaybd-global.json`, `$AENV_HOME/overlaybd/resize-overlaybd-global.json` | Runtime global config, cache path, credentials config | Configures overlaybd runtime, memory snapshot overlaybd access, and the offline C++ tools (`overlaybd-apply`, `overlaybd-resize`), which get dedicated configs with isolated cacheDirs (`convert-blocks`, `resize-blocks`) and download disabled | Rewritten during setup/startup. | | Overlaybd runtime log | `$AENV_HOME/overlaybd/overlaybd.log` | Overlaybd runtime logs | Debugging | Appended by overlaybd runtime; no automatic GC. | -| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure sandboxes are persisted. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure or private-ingress sandboxes are persisted. | ## Firecracker Sandbox diff --git a/docs/src/internals/proxy-design.md b/docs/src/internals/proxy-design.md index 3fff71da..822f9189 100644 --- a/docs/src/internals/proxy-design.md +++ b/docs/src/internals/proxy-design.md @@ -43,6 +43,23 @@ Validation: - Sandbox ID must be a valid UUID format. - Target port must parse as `u16` and be greater than `0`. +Authorization is evaluated by the owning runtime after route parsing: + +- Control-plane routes require the deployment `X-API-Key` and do not accept + sandbox credentials. +- Node Prometheus `/metrics` and health `/health` are public to application + auth and should be protected separately at the deployment boundary. +- Non-envd application routes require `e2b-traffic-access-token` only when the + sandbox has private ingress (`allowPublicTraffic: false`). +- The envd port requires `X-Access-Token` only for secure sandboxes. +- `X-API-Key` is never a data-plane credential. + +The distributed gateway deliberately does not make sandbox authorization +decisions. It routes data-plane requests, including public ingress and insecure +envd requests with no credential, to the owning runtime. The runtime has the +sandbox metadata needed to apply the policy and performs the authoritative +token validation. + Host-based routing derives both fields from `Host`. The configured domain must match exactly after lowercase normalization and optional trailing-dot removal. Sandbox IDs in host routes must be valid UUIDs and the target port must fit in @@ -75,7 +92,7 @@ This keeps hot-path reads lock-light and avoids reading sandbox instance interna `/proxy` can auto-resume paused sandboxes when lifecycle policy enables it. -- Proxy never reads sandbox metadata directly. +- Proxy route resolution does not read sandbox instance internals. - Orchestrator lookup returns `Paused { auto_resume }`, and proxy decides behavior from that signal. - Auto-resume is attempted once per request. - Resume timeout update uses `EnsureMinimum(5 minutes)`: @@ -135,6 +152,14 @@ Control-plane routing headers are stripped before forwarding upstream: - `e2b-sandbox-id` - `e2b-sandbox-port` +Sandbox credential handling: + +- `e2b-traffic-access-token` is stripped before forwarding. +- A successfully validated secure-envd `X-Access-Token` is forwarded to envd; + otherwise that header is stripped. +- A value matching the platform `X-API-Key` is stripped; other values are + forwarded as application headers. + Hop-by-hop headers are stripped on both request and response paths, including: - Standard hop-by-hop headers (`Connection`, `Upgrade`, `TE`, `Trailer`, `Transfer-Encoding`, `Proxy-Authenticate`, `Proxy-Authorization`, `Keep-Alive`) @@ -191,7 +216,7 @@ Handshake failure behavior: ```bash curl -i \ - -H 'X-API-Key: test-key' \ + -H 'e2b-traffic-access-token: ' \ -H 'x-agentenv-sandbox-id: ' \ -H 'x-agentenv-target-port: 8080' \ 'http://127.0.0.1:8000/proxy/health?full=true' diff --git a/docs/src/internals/services.md b/docs/src/internals/services.md index 98dffb02..5343cf08 100644 --- a/docs/src/internals/services.md +++ b/docs/src/internals/services.md @@ -25,7 +25,8 @@ make proto # regenerate protobuf # Start scheduler (default: 127.0.0.1:9090) make -C services run-scheduler -# Start gateway +# Start gateway (use the same key on runtime nodes) +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make -C services run-gateway ``` @@ -41,6 +42,7 @@ The scheduler supports two node discovery modes: ### Docker Compose ```bash +# Run scripts/docker-setup.sh first for host prerequisites. make deploy-up # gateway + scheduler + 2 backend nodes make deploy-ps # status make deploy-logs # logs diff --git a/docs/src/security/authentication.md b/docs/src/security/authentication.md new file mode 100644 index 00000000..48dfc961 --- /dev/null +++ b/docs/src/security/authentication.md @@ -0,0 +1,61 @@ +# Authentication + +AgentENV uses one shared API key for a single-tenant deployment. The gateway +and every runtime node must use the same value. + +| Credential | Scope | Header | +|---|---|---| +| API key | AgentENV lifecycle and management APIs | `X-API-Key` | +| `trafficAccessToken` | Application ingress when `allowPublicTraffic` is `false` | `e2b-traffic-access-token` | +| `envdAccessToken` | Direct envd access for secure sandboxes | `X-Access-Token` | + +The credentials are not interchangeable. Public application ingress and envd +in insecure sandboxes need no AgentENV credential. `Authorization` remains an +application header and does not authenticate AgentENV. On sandbox routes, +`X-API-Key` is also treated as application data unless it exactly matches the +AgentENV API key, in which case it is removed to avoid forwarding the platform +credential. + +`GET /health` and node `GET /metrics` are outside API-key authentication. The +gateway exposes Prometheus metrics on its separate metrics listener. Protect +these endpoints with the network and authentication controls used by your +Prometheus deployment. They are distinct from E2B's authenticated sandbox +metrics API. + +E2B SDK users set `E2B_API_KEY` to the AgentENV API key. Sandbox credentials +are derived from the sandbox ID and +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED`, independently of the API key. + +## Key Resolution + +A runtime node checks these sources in order: + +1. `AENV_API_KEY` +2. `/run/secrets/api-key` +3. `$AENV_HOME/secrets/api-key` + +If none exists, normal server startup generates and atomically stores a key at +the managed path. The gateway checks only the first two sources and never +generates a key. + +Custom keys must contain 32 to 256 URL-safe characters. Generated keys use an +E2B-compatible `e2b_` prefix. For example: + +```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +``` + +Docker Compose shares one managed-secret volume between runtime nodes and +mounts it read-only on the gateway. Kubernetes stores the key in +`Secret/agentenv-auth`. See the corresponding deployment guide for commands to +read or supply those values. Multi-node deployments must also share one +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` across runtime nodes. + +## Security and Rotation + +API-key authentication does not encrypt traffic. Use HTTPS termination, a VPN, +loopback, or a trusted private network. + +Changing `AENV_API_KEY` invalidates API clients without changing sandbox +credentials. Changing `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both +sandbox token types. Apply either change to all relevant processes together. diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 9b763a4a..57807973 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -2,8 +2,10 @@ Secure sandboxes use an envd access token for control-plane communication. This protects envd operations such as command execution and file access. -> [!WARNING] -> This feature does not add authentication to application ports exposed by the sandbox. +> [!NOTE] +> Secure mode protects envd control-plane operations. Application traffic uses +> the sandbox-scoped `trafficAccessToken` described in +> [Authentication](./authentication.md). Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to enable secure mode. Or use the CLI: @@ -11,14 +13,14 @@ Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to aenv start --secure ``` -The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Forked sandboxes get independent tokens. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. +The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Private application ingress has an independent `trafficAccessToken`, sent as `e2b-traffic-access-token`; public application ingress has no AgentENV credential. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. ## Access-Token Seed -A seed is a random value used to derive the access token for each sandbox. This seed is optional. When it is unset, each runtime node automatically creates and persists a node-local seed under `$AENV_HOME/secrets`. +A seed is a random value used to derive each sandbox's envd and traffic access tokens. This seed is optional for a standalone runtime. When it is unset, the runtime automatically creates and persists a seed under `$AENV_HOME/secrets`. This is sufficient for normal single-node operation and does not require additional setup. -Configure the same explicit seed on every runtime node when the deployment needs to recover the same sandbox ID on another node in the future. Generate it once and store it in the deployment's secret manager: +Configure the same explicit seed on every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: ```bash openssl rand -hex 32 @@ -27,11 +29,12 @@ openssl rand -hex 32 Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on every runtime node. For TOML configuration, use `[sandbox].access_token_hash_seed` instead. -Preserve the seed across upgrades; changing it rotates access tokens for existing secure sandboxes. +Preserve the seed across upgrades; changing it rotates both sandbox access tokens. ### Kubernetes -The runtime DaemonSet reads the optional `agentenv-runtime-secrets` Secret. To configure a shared seed for all runtime Pods, create it before applying the runtime manifests: +The runtime DaemonSet retains the existing optional `agentenv-runtime-secrets` +contract. Create one shared seed before applying the runtime manifests: ```bash kubectl apply -f deploy/k8s/base/namespace.yaml @@ -43,7 +46,8 @@ kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ unset AENV_ACCESS_TOKEN_HASH_SEED ``` -Run this once for a new cluster and preserve the existing Secret during upgrades. An external secret manager may be used instead, provided it creates the same Secret name and key: +Preserve this Secret during upgrades. An external secret manager may provide +the same name and key: ```yaml apiVersion: v1 @@ -55,4 +59,4 @@ stringData: sandbox-access-token-hash-seed: ``` -If the Secret is not created, the DaemonSet still starts and each runtime Pod uses its automatically managed node-local seed. +If the Secret is absent, each runtime Pod uses its managed node-local seed. diff --git a/scripts/install.sh b/scripts/install.sh index 1cc31e2d..254bd17c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -256,6 +256,7 @@ sudo chmod 0640 "$CONFIG_PATH" if [[ -d /run/systemd/system ]]; then ENV_FILE_STATUS="exists" if [[ ! -f "$ENV_FILE" ]]; then + sudo install -o root -g "$SERVICE_GROUP" -m 0640 /dev/null "$ENV_FILE" sudo tee "$ENV_FILE" > /dev/null <> "$tmp_env" fi - sudo install -m 0644 "$tmp_env" "$ENV_FILE" + sudo install -o root -g "$SERVICE_GROUP" -m 0640 "$tmp_env" "$ENV_FILE" rm -f "$current_env" "$tmp_env" ENV_FILE_STATUS="updated" fi @@ -370,6 +371,7 @@ echo " CLI : ${INSTALL_DIR}/aenv" echo " Server : ${INSTALL_DIR}/server" echo " Data : ${DATA_DIR}" echo " Config : ${CONFIG_PATH}" +echo " API key: ${DATA_DIR}/secrets/api-key (auto-generated when no API key is configured)" echo " Mode : ${VIRTUALIZATION_MODE}" if [[ -d /run/systemd/system ]]; then if [[ "$ENV_FILE_STATUS" == "written" ]]; then diff --git a/scripts/tests/e2e/lib/helpers.sh b/scripts/tests/e2e/lib/helpers.sh index b84b9aad..e5c088f4 100644 --- a/scripts/tests/e2e/lib/helpers.sh +++ b/scripts/tests/e2e/lib/helpers.sh @@ -5,10 +5,10 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then E2E_HELPERS_SH_LOADED=1 : "${AENV_URL:?AENV_URL must be set}" - : "${AENV_API_KEY:=e2e-test-key}" - : "${AENV_ADMIN_TOKEN:=e2e-admin-token}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${AENV_TEMPLATE_ID:=ubuntu}" : "${AENV_PROXY_URL:=${AENV_URL}/proxy}" + : "${AENV_ENVD_PORT:=49983}" : "${E2E_MODE:=single-node}" : "${E2E_DEFAULT_USER_IMAGE:=ghcr.io/linuxserver/baseimage-ubuntu:noble}" : "${E2E_TEMPLATE_USER_IMAGE:=${E2E_DEFAULT_USER_IMAGE}}" @@ -121,44 +121,6 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then HTTP_HEADERS=$(<"$_E2E_HEADERS") } - api_admin_get() { - local path="$1" - _curl_do -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ - "${AENV_URL}${path}" - } - - api_admin_get_at() { - local base_url="$1" - local path="$2" - _curl_do -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ - "${base_url}${path}" - } - - api_admin_get_with_headers() { - local path="$1" - [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) - curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ - "${AENV_URL}${path}" > "$_E2E_STATUS" 2>/dev/null || true - HTTP_STATUS=$(<"$_E2E_STATUS") - HTTP_BODY=$(<"$_E2E_BODY") - HTTP_HEADERS=$(<"$_E2E_HEADERS") - } - - api_admin_get_with_headers_at() { - local base_url="$1" - local path="$2" - [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) - curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ - "${base_url}${path}" > "$_E2E_STATUS" 2>/dev/null || true - HTTP_STATUS=$(<"$_E2E_STATUS") - HTTP_BODY=$(<"$_E2E_BODY") - HTTP_HEADERS=$(<"$_E2E_HEADERS") - } - api_post() { local path="$1" local body="${2:-}" @@ -209,9 +171,8 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then proxy_get_with_sandbox() { local sandbox_id="$1" local path="${2:-/health}" - local target_port="${3:-49983}" + local target_port="${3:-${AENV_ENVD_PORT}}" _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${sandbox_id}" \ -H "x-agentenv-target-port: ${target_port}" \ "${AENV_PROXY_URL}${path}" diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index 87886efe..f1973f11 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -7,12 +7,20 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then E2E_RUNTIME_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" E2E_REPO_ROOT="$(cd "${E2E_RUNTIME_DIR}/../../../.." && pwd)" + _E2E_API_KEY_FROM_USER=0 + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + if [[ -z "${AENV_API_KEY}" ]]; then + echo "AENV_API_KEY must not be empty" >&2 + return 1 + fi + _E2E_API_KEY_FROM_USER=1 + fi + # shellcheck source=/dev/null source "${E2E_RUNTIME_DIR}/server.sh" : "${E2E_MODE:=single-node}" - : "${AENV_API_KEY:=e2e-test-key}" - : "${AENV_ADMIN_TOKEN:=e2e-admin-token}" + export AENV_API_KEY : "${E2E_COMPOSE_FILE:=deploy/docker-compose.yml}" : "${E2E_COMPOSE_OVERRIDE_FILE:=scripts/tests/e2e/docker-compose.e2e.yml}" : "${E2E_COMPOSE_START_TIMEOUT:=120}" @@ -61,7 +69,11 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then } _deploy_make_cmd() { - make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + if [[ "${_E2E_API_KEY_FROM_USER}" == "1" ]]; then + make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + else + env -u AENV_API_KEY make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + fi } _run_deploy_target() { @@ -156,7 +168,12 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _run_k8s_target() { local target="${1:?usage: _run_k8s_target }" [[ -n "${target}" && "${target}" != "none" ]] || return 0 - make --no-print-directory -C "${E2E_REPO_ROOT}" "${target}" + _deploy_make_cmd "${target}" + } + + _read_k8s_api_key() { + kubectl -n "${E2E_K8S_NAMESPACE}" get secret agentenv-auth \ + -o 'go-template={{index .data "AENV_API_KEY" | base64decode}}' } _resolve_runtime_path() { @@ -237,7 +254,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then log "Waiting for scheduler to observe ${expected_count} ready node(s) via ${AENV_URL}/nodes (timeout ${timeout}s) ..." for ((i = 1; i <= timeout; i++)); do response=$(curl -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ -w $'\n%{http_code}' \ "${AENV_URL}/nodes" 2>/dev/null || true) status="${response##*$'\n'}" @@ -398,6 +415,14 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _wait_for_health_url "agentenv-b" "${AENV_NODE_B_URL}" "${timeout}" || die "agentenv-b failed to become ready within ${timeout}s" + if [[ "${_E2E_API_KEY_FROM_USER}" != "1" ]]; then + AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || + die "Failed to read the Compose deployment API key" + fi + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,256}$ ]] || + die "Compose deployment returned an invalid API key" + export AENV_API_KEY + expected_nodes="$(_runtime_node_count)" [[ "${expected_nodes}" -gt 0 ]] || expected_nodes=1 _wait_for_scheduler_ready_nodes "${timeout}" "${expected_nodes}" || @@ -434,6 +459,14 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then die "${label} failed to become ready within ${timeout}s" done < <(printf '%s\n' "${AENV_NODE_URLS}" | tr ' ' '\n') + if [[ "${_E2E_API_KEY_FROM_USER}" != "1" ]]; then + AENV_API_KEY="$(_read_k8s_api_key)" || + die "Failed to read the Kubernetes deployment API key" + fi + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,256}$ ]] || + die "Kubernetes deployment returned an invalid API key" + export AENV_API_KEY + expected_nodes="$(_runtime_node_count)" [[ "${expected_nodes}" -gt 0 ]] || expected_nodes=1 _wait_for_scheduler_ready_nodes "${timeout}" "${expected_nodes}" || diff --git a/scripts/tests/e2e/lib/server.sh b/scripts/tests/e2e/lib/server.sh index c550b661..bc3997eb 100644 --- a/scripts/tests/e2e/lib/server.sh +++ b/scripts/tests/e2e/lib/server.sh @@ -6,7 +6,7 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then : "${AENV_PORT:=18080}" : "${AENV_URL:=http://127.0.0.1:${AENV_PORT}}" - : "${AENV_API_KEY:=e2e-test-key}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${SERVER_START_TIMEOUT:=30}" _SERVER_PID="" @@ -15,6 +15,7 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then local binary="${1:?usage: start_server [config_path]}" local config="${2:-}" + export AENV_API_KEY local env_vars=( "API_ADDR=127.0.0.1:${AENV_PORT}" "RUST_LOG=agentenv=info,envd=info" diff --git a/scripts/tests/e2e/suites/06_proxy.sh b/scripts/tests/e2e/suites/06_proxy.sh index da9903e8..e500e67c 100755 --- a/scripts/tests/e2e/suites/06_proxy.sh +++ b/scripts/tests/e2e/suites/06_proxy.sh @@ -12,24 +12,23 @@ log "Suite: Proxy Routing" sandbox_id=$(create_sandbox); _sync_http assert_status "$HTTP_STATUS" "201" "create sandbox for proxy test" assert_not_empty "$sandbox_id" "sandboxID present" +assert_json_field "$HTTP_BODY" '.trafficAccessToken' "null" "public sandbox omits traffic token" track_sandbox "$sandbox_id" wait_for_sandbox_state "$sandbox_id" "running" 30 # -- Proxy request using AgentENV headers -- -# The envd health endpoint runs on port 49983 inside the sandbox. +# The envd health endpoint runs on the configured control-plane port. _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${sandbox_id}" \ - -H "x-agentenv-target-port: 49983" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (agentenv headers) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "proxy with agentenv headers" # -- Proxy request using E2B compat headers -- _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "e2b-sandbox-id: ${sandbox_id}" \ - -H "e2b-sandbox-port: 49983" \ + -H "e2b-sandbox-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (e2b headers) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "proxy with e2b headers" @@ -38,7 +37,6 @@ assert_status "$HTTP_STATUS" "204" "proxy with e2b headers" # Use the explicit /proxy path so both single-node and compose reach the # node-local proxy entrypoint before header validation. _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}/proxy/health" log "Proxy (no sandbox header) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "400" "proxy without sandbox header" @@ -59,9 +57,8 @@ else fi _curl_do -s --max-time 10 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${paused_no_resume_id}" \ - -H "x-agentenv-target-port: 49983" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (paused + auto-resume disabled) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "410" "paused sandbox without auto-resume returns 410" @@ -83,9 +80,8 @@ fi # Auto-resume may wait up to 60s in non-test runtime. Keep client timeout above that. _curl_do -s --max-time 75 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "e2b-sandbox-id: ${paused_auto_resume_id}" \ - -H "e2b-sandbox-port: 49983" \ + -H "e2b-sandbox-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (paused + auto-resume enabled) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "paused sandbox auto-resumes on proxy request" diff --git a/scripts/tests/e2e/suites/08_auth.sh b/scripts/tests/e2e/suites/08_auth.sh index 5f0d30da..0c87dab4 100755 --- a/scripts/tests/e2e/suites/08_auth.sh +++ b/scripts/tests/e2e/suites/08_auth.sh @@ -8,16 +8,130 @@ init_suite "08_auth" log "Suite: Authentication" +proxy_envd_health() { + local sandbox_id="$1" + local header_name="${2:-}" + local header_value="${3:-}" + local args=(-s --max-time 5 + -H "x-agentenv-sandbox-id: ${sandbox_id}" + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}") + [[ -z "${header_name}" ]] || args+=(-H "${header_name}: ${header_value}") + _curl_do "${args[@]}" "${AENV_PROXY_URL}/health" +} + # -- Request without auth header returns 401 -- api_get_no_auth "/sandboxes" assert_status "$HTTP_STATUS" "401" "no auth header returns 401" +# -- Alternative and malformed credentials are rejected -- +_curl_do -s -H "X-API-Key: ${AENV_API_KEY}x" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "wrong API key returns 401" + +_curl_do -s -H "Authorization: Bearer ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "Authorization does not authenticate AgentENV" + +_curl_do -s -H "X-Admin-Token: ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "legacy admin token does not authenticate AgentENV" + +_curl_do -s -H "X-Team-ID: ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "legacy team key does not authenticate AgentENV" + +_curl_do -s \ + -H "X-API-Key: ${AENV_API_KEY}" \ + -H "X-API-Key: ${AENV_API_KEY}x" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "valid and invalid API key headers return 401" + +_curl_do -s \ + -H "X-API-Key: ${AENV_API_KEY}x" \ + -H "X-API-Key: ${AENV_API_KEY}y" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "conflicting invalid API key headers return 401" + # -- Request with valid API key succeeds -- api_get "/sandboxes" -assert_not_eq "$HTTP_STATUS" "401" "valid API key does not return 401" +assert_status "$HTTP_STATUS" "200" "valid API key authenticates successfully" + +# -- Sandbox-scoped credentials cannot authenticate the control plane -- +secure_sandbox_id=$(create_sandbox "$AENV_TEMPLATE_ID" 60 \ + '{"secure":true,"network":{"allowPublicTraffic":false}}'); _sync_http +if [[ -n "$secure_sandbox_id" ]]; then + track_sandbox "$secure_sandbox_id" +fi +assert_status "$HTTP_STATUS" "201" "create private secure sandbox" +assert_not_empty "$secure_sandbox_id" "private secure sandbox ID present" + +if [[ "$HTTP_STATUS" != "201" || -z "$secure_sandbox_id" ]]; then + suite_summary "08_auth" || true + exit 1 +fi + +traffic_access_token=$(echo "$HTTP_BODY" | jq -r '.trafficAccessToken // empty') +envd_access_token=$(echo "$HTTP_BODY" | jq -r '.envdAccessToken // empty') +assert_not_empty "$traffic_access_token" "private sandbox returns traffic token" +assert_not_empty "$envd_access_token" "secure sandbox returns envd token" +if [[ -z "$traffic_access_token" || -z "$envd_access_token" ]]; then + suite_summary "08_auth" || true + exit 1 +fi +wait_for_sandbox_state "$secure_sandbox_id" "running" 30 -# -- Health endpoint works without auth -- +_curl_do -s \ + -H "e2b-traffic-access-token: ${traffic_access_token}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "traffic token does not authenticate control plane" + +_curl_do -s \ + -H "X-Access-Token: ${envd_access_token}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "envd token does not authenticate control plane" + +# -- Secure envd accepts only its envd access token -- +proxy_envd_health "${secure_sandbox_id}" +assert_status "$HTTP_STATUS" "401" "secure envd rejects missing token" + +proxy_envd_health "${secure_sandbox_id}" "X-API-Key" "${AENV_API_KEY}" +assert_status "$HTTP_STATUS" "401" "secure envd rejects API key" + +proxy_envd_health \ + "${secure_sandbox_id}" "e2b-traffic-access-token" "${traffic_access_token}" +assert_status "$HTTP_STATUS" "401" "secure envd rejects traffic token" + +proxy_envd_health "${secure_sandbox_id}" "X-Access-Token" "${envd_access_token}" +assert_status "$HTTP_STATUS" "204" "secure envd accepts envd token" + +# -- Envd tokens are scoped to one sandbox -- +other_secure_sandbox_id=$(create_sandbox "$AENV_TEMPLATE_ID" 60 \ + '{"secure":true,"network":{"allowPublicTraffic":false}}'); _sync_http +if [[ -n "$other_secure_sandbox_id" ]]; then + track_sandbox "$other_secure_sandbox_id" +fi +assert_status "$HTTP_STATUS" "201" "create second private secure sandbox" +assert_not_empty "$other_secure_sandbox_id" "second private secure sandbox ID present" +if [[ "$HTTP_STATUS" != "201" || -z "$other_secure_sandbox_id" ]]; then + suite_summary "08_auth" || true + exit 1 +fi +wait_for_sandbox_state "$other_secure_sandbox_id" "running" 30 + +proxy_envd_health "${other_secure_sandbox_id}" "X-Access-Token" "${envd_access_token}" +assert_status "$HTTP_STATUS" "401" "envd token cannot authenticate another sandbox" + +# -- Health and Prometheus metrics work without application auth -- api_get_no_auth "/health" assert_status "$HTTP_STATUS" "204" "/health works without auth" +api_get_no_auth "/metrics" +if e2e_mode_is_clustered; then + assert_status "$HTTP_STATUS" "404" "gateway client listener does not expose /metrics" +else + assert_status "$HTTP_STATUS" "200" "/metrics works without auth" +fi + +while IFS= read -r node_url; do + [[ -z "${node_url}" ]] && continue + api_get_no_auth_at "${node_url}" "/metrics" + assert_status "$HTTP_STATUS" "200" "node /metrics works without auth at ${node_url}" +done < <(printf '%s\n' "${AENV_NODE_URLS:-}" | tr ' ' '\n') + suite_summary "08_auth" diff --git a/scripts/tests/e2e/suites/09_e2b_compat.sh b/scripts/tests/e2e/suites/09_e2b_compat.sh index a5bb8ff6..611b09ad 100755 --- a/scripts/tests/e2e/suites/09_e2b_compat.sh +++ b/scripts/tests/e2e/suites/09_e2b_compat.sh @@ -10,8 +10,7 @@ log "Suite: E2B Compatibility" export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" -export E2B_API_KEY="e2b_000000" -export E2B_ACCESS_TOKEN="${AENV_API_KEY}" +export E2B_API_KEY="${AENV_API_KEY}" export E2B_COMPAT_USER_IMAGE="${E2B_COMPAT_USER_IMAGE:-${E2E_TEMPLATE_USER_IMAGE:-ghcr.io/linuxserver/baseimage-ubuntu:noble}}" cli_available=0 diff --git a/scripts/tests/e2e/suites/11_node_metrics.sh b/scripts/tests/e2e/suites/11_node_metrics.sh index d547ed48..d2df68c7 100644 --- a/scripts/tests/e2e/suites/11_node_metrics.sh +++ b/scripts/tests/e2e/suites/11_node_metrics.sh @@ -11,7 +11,7 @@ log "Suite: Node Metrics" readonly NODE_METRICS_SANDBOX_TIMEOUT_SECONDS=60 readonly SCHEDULER_BINDING_CLEANUP_TIMEOUT_SECONDS=75 readonly PROXY_HEALTH_PATH="/health" -readonly PROXY_HEALTH_PORT=49983 +readonly PROXY_HEALTH_PORT="${AENV_ENVD_PORT}" wait_for_global_sandboxes_quiesced() { local timeout="${1:-20}" @@ -55,7 +55,7 @@ wait_for_node_runtime_allocations_quiesced() { local attempt for ((attempt = 0; attempt < timeout * 2; attempt++)); do - api_admin_get "/nodes" + api_get "/nodes" if [[ "${HTTP_STATUS}" == "200" ]] && echo "${HTTP_BODY}" | jq -e 'all(.[]; (.sandboxCount == 0 and .metrics.allocatedCPU == 0 and .metrics.allocatedMemoryBytes == 0))' >/dev/null 2>&1; then return 0 @@ -97,7 +97,7 @@ quiesce_runtime_state_before_baseline() { fetch_admin_nodes() { local base_url="${1:-${AENV_URL}}" - api_admin_get_at "${base_url}" "/nodes" + api_get_at "${base_url}" "/nodes" [[ "${HTTP_STATUS}" == "200" ]] || return 1 printf '%s\n' "${HTTP_BODY}" } @@ -138,7 +138,7 @@ wait_for_node_snapshot() { local create_successes for ((attempt = 0; attempt < timeout * 2; attempt++)); do - api_admin_get "/nodes" + api_get "/nodes" if [[ "${HTTP_STATUS}" == "200" ]]; then body="${HTTP_BODY}" sandbox_count="$(echo "${body}" | jq -r --arg id "${node_id}" '.[] | select(.id == $id) | .sandboxCount // empty' 2>/dev/null || true)" @@ -164,7 +164,7 @@ wait_for_node_snapshot() { node_detail_sandbox_count() { local base_url="$1" local node_id="$2" - api_admin_get_at "${base_url}" "/nodes/${node_id}" + api_get_at "${base_url}" "/nodes/${node_id}" [[ "${HTTP_STATUS}" == "200" ]] || return 1 echo "${HTTP_BODY}" | jq '.sandboxCount' } @@ -192,7 +192,7 @@ quiesce_runtime_state_before_baseline || wait_for_admin_nodes_count "${AENV_URL}" "${expected_nodes}" 60 || die "Timed out waiting for gateway/admin nodes endpoint" -api_admin_get "/nodes" +api_get "/nodes" assert_status "${HTTP_STATUS}" "200" "admin /nodes returns 200" baseline_nodes_json="${HTTP_BODY}" @@ -206,7 +206,7 @@ while IFS=$'\t' read -r node_id sandbox_count allocated_cpu allocated_memory cre BASELINE_ALLOCATED_MEMORY["${node_id}"]="${allocated_memory}" BASELINE_CREATE_SUCCESSES["${node_id}"]="${create_successes}" - api_admin_get "/nodes/${node_id}" + api_get "/nodes/${node_id}" assert_status "${HTTP_STATUS}" "200" "admin /nodes/${node_id} returns 200" BASELINE_DETAIL_SANDBOX_COUNT["${node_id}"]="$(echo "${HTTP_BODY}" | jq '.sandboxCount')" done < <(echo "${baseline_nodes_json}" | jq -r '.[] | [ @@ -222,7 +222,7 @@ if e2e_mode_is_clustered; then [[ -n "${node_url}" ]] || continue wait_for_admin_nodes_count "${node_url}" 1 45 || die "Timed out waiting for ${node_url}/nodes" - api_admin_get_at "${node_url}" "/nodes" + api_get_at "${node_url}" "/nodes" assert_status "${HTTP_STATUS}" "200" "node-local admin /nodes returns 200 for $(node_label_for_url "${node_url}")" local_node_id="$(echo "${HTTP_BODY}" | jq -r '.[0].id')" assert_not_empty "${local_node_id}" "node-local admin /nodes exposes node id for $(node_label_for_url "${node_url}")" @@ -316,14 +316,14 @@ for node_id in "${!BASELINE_SANDBOX_COUNT[@]}"; do 30; then _pass "gateway /nodes metrics converge for ${node_id}" else - api_admin_get "/nodes" + api_get "/nodes" _fail \ "gateway /nodes metrics converge for ${node_id}" \ "sandboxCount=${expected_sandbox_count}, allocatedCPU=${expected_allocated_cpu}, allocatedMemoryBytes=${expected_allocated_memory}, createSuccesses>=${expected_create_successes_min}" \ "${HTTP_BODY}" fi - api_admin_get "/nodes/${node_id}" + api_get "/nodes/${node_id}" assert_status "${HTTP_STATUS}" "200" "gateway /nodes/${node_id} returns 200 after workload" detail_sandboxes="$(echo "${HTTP_BODY}" | jq '.sandboxCount')" expected_detail_sandboxes=$((BASELINE_DETAIL_SANDBOX_COUNT["${node_id}"] + owned_count)) @@ -370,7 +370,7 @@ for node_id in "${!BASELINE_SANDBOX_COUNT[@]}"; do 30; then _pass "gateway /nodes runtime allocation resets for ${node_id} after cleanup" else - api_admin_get "/nodes" + api_get "/nodes" _fail \ "gateway /nodes runtime allocation resets for ${node_id} after cleanup" \ "sandboxCount=${BASELINE_SANDBOX_COUNT[${node_id}]}, allocatedCPU=${BASELINE_ALLOCATED_CPU[${node_id}]}, allocatedMemoryBytes=${BASELINE_ALLOCATED_MEMORY[${node_id}]}, createSuccesses>=${expected_create_successes_min_after_cleanup}" \ diff --git a/scripts/tests/e2e/suites/14_code_interpreter.sh b/scripts/tests/e2e/suites/14_code_interpreter.sh index e95b2a8b..2b84c70a 100755 --- a/scripts/tests/e2e/suites/14_code_interpreter.sh +++ b/scripts/tests/e2e/suites/14_code_interpreter.sh @@ -18,7 +18,7 @@ exit 0 export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" -export E2B_API_KEY="e2b_000000" +export E2B_API_KEY="${AENV_API_KEY}" if ! python3 -c 'import e2b_code_interpreter' >/dev/null 2>&1; then warn "e2b_code_interpreter Python package not installed; skipping" diff --git a/services/README.md b/services/README.md index ee2880ca..59fe1194 100644 --- a/services/README.md +++ b/services/README.md @@ -69,14 +69,23 @@ Start scheduler: make run-scheduler ``` -Start gateway: +Start gateway with the same API key configured on every AgentENV runtime node: ```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. +The gateway and runtime nodes require the same API key for control-plane APIs. +The gateway reads `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate +a key. The gateway routes data-plane requests without authenticating them +because only the owning runtime has the sandbox policy needed to distinguish +public ingress, private ingress, and secure envd. Private application proxy +requests use the sandbox response's `trafficAccessToken` in the +`e2b-traffic-access-token` header; secure envd requests use `X-Access-Token`. + ## Scheduler configuration Scheduler discovery modes: @@ -176,6 +185,7 @@ LOG_FORMAT=json make run-gateway From **repository root**, start gateway + scheduler + two backend nodes: ```bash +# Run scripts/docker-setup.sh first for host prerequisites. make deploy-up ``` diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index b21738bf..e7bbef26 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -4,10 +4,13 @@ import ( "context" "errors" "flag" + "fmt" + "io" "log" "net/http" "os" "os/signal" + "strings" "syscall" "time" @@ -22,6 +25,13 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +const ( + apiKeyEnv = "AENV_API_KEY" + defaultAPIKeyPath = "/run/secrets/api-key" + maxAPIKeyLen = 256 + maxAPIKeyFileLen = maxAPIKeyLen + 2 +) + func newSchedulerConn(addr string) (*grpc.ClientConn, error) { return grpc.NewClient( addr, @@ -29,6 +39,71 @@ func newSchedulerConn(addr string) (*grpc.ClientConn, error) { ) } +func loadAPIKey() (string, error) { + return loadAPIKeyFrom(os.LookupEnv, defaultAPIKeyPath) +} + +func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { + value, source := "", apiKeyEnv + if explicit, present := lookupEnv(apiKeyEnv); present { + value = explicit + } else { + file, err := openSecretFile(secretPath) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + } + return "", fmt.Errorf("read secret %s: %w", secretPath, err) + } + defer file.Close() + contents, err := io.ReadAll(io.LimitReader(file, maxAPIKeyFileLen+1)) + if err != nil { + return "", fmt.Errorf("read secret %s: %w", secretPath, err) + } + value = strings.TrimSuffix(strings.TrimSuffix(string(contents), "\n"), "\r") + source = secretPath + } + return validateAPIKey(value, source) +} + +func openSecretFile(path string) (*os.File, error) { + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = syscall.Close(fd) + return nil, fmt.Errorf("open returned an invalid file descriptor") + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, fmt.Errorf("must be a regular file") + } + return file, nil +} + +func validateAPIKey(value, source string) (string, error) { + if len(value) < 32 || len(value) > maxAPIKeyLen { + return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) + } + for _, char := range []byte(value) { + if (char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '.' || char == '_' || char == '~' || char == '-' { + continue + } + return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) + } + return value, nil +} + func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -37,7 +112,10 @@ func main() { if err != nil { log.Fatalf("load config failed: %v", err) } - + apiKey, err := loadAPIKey() + if err != nil { + log.Fatalf("load API key failed: %v", err) + } logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { log.Fatalf("init logger failed: %v", err) @@ -65,6 +143,7 @@ func main() { s, err := gateway.NewServer(logger, schedulerClient, gateway.ServerOptions{ RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, + APIKey: apiKey, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go new file mode 100644 index 00000000..f384460e --- /dev/null +++ b/services/gateway/cmd/main_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestValidateAPIKey(t *testing.T) { + t.Parallel() + + got, err := validateAPIKey(testAPIKey, "test") + if err != nil { + t.Fatalf("validateAPIKey() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("validateAPIKey() = %q, want %q", got, testAPIKey) + } + + for _, invalid := range []string{"", "too-short", " " + testAPIKey, testAPIKey + "\n", strings.Repeat("a", 31), strings.Repeat("a", maxAPIKeyLen+1), strings.Repeat("a", 31) + "!"} { + if _, err := validateAPIKey(invalid, "test"); err == nil { + t.Errorf("validateAPIKey(%q) unexpectedly succeeded", invalid) + } + } +} + +func TestLoadAPIKeyFromEnvironment(t *testing.T) { + got, err := loadAPIKeyFrom( + func(name string) (string, bool) { return testAPIKey, name == apiKeyEnv }, + filepath.Join(t.TempDir(), "missing"), + ) + if err != nil { + t.Fatalf("loadAPIKey() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKey() = %q, want %q", got, testAPIKey) + } +} + +func TestLoadAPIKeyRejectsExplicitEmptyEnvironment(t *testing.T) { + if _, err := loadAPIKeyFrom( + func(name string) (string, bool) { return "", name == apiKeyEnv }, + filepath.Join(t.TempDir(), "missing"), + ); err == nil { + t.Fatal("loadAPIKey() unexpectedly accepted an empty environment value") + } +} + +func TestLoadAPIKeyFromFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(path, []byte(testAPIKey+"\n"), 0o444); err != nil { + t.Fatal(err) + } + got, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAPIKeyFrom() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKeyFrom() = %q, want %q", got, testAPIKey) + } +} + +func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missing := filepath.Join(dir, "missing") + if _, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, missing); err == nil { + t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") + } +} + +func TestLoadAPIKeyRejectsNonRegularFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "api-key") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path); err == nil { + t.Fatal("loadAPIKeyFrom() unexpectedly accepted a FIFO") + } +} + +func TestLoadAPIKeyAllowsSymlinkedSecret(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + target := filepath.Join(dir, "..data-api-key") + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(target, []byte(testAPIKey+"\n"), 0o444); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Base(target), path); err != nil { + t.Fatal(err) + } + got, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAPIKeyFrom() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKeyFrom() = %q, want %q", got, testAPIKey) + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 2f6d0829..77516682 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,6 +3,7 @@ package gateway import ( "bytes" "context" + "crypto/subtle" "encoding/json" "errors" "io" @@ -22,6 +23,9 @@ import ( ) const ( + headerAPIKey = "X-API-Key" + headerTrafficToken = "e2b-traffic-access-token" + headerEnvdAccessToken = "X-Access-Token" headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" headerTargetPort = "x-agentenv-target-port" @@ -41,6 +45,7 @@ const ( ) type ServerOptions struct { + APIKey string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -53,6 +58,7 @@ type Server struct { scheduler schedulerv1.SchedulerClient queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client + apiKey []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -63,6 +69,9 @@ type Server struct { } func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, options ServerOptions) (*Server, error) { + if options.APIKey == "" { + return nil, errors.New("API key is required") + } sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { return nil, err @@ -80,6 +89,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, httpClient: &http.Client{}, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, + apiKey: []byte(options.APIKey), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -94,6 +104,15 @@ func (s *Server) Handler() http.Handler { // decoding %2F → / and issuing 301 redirects), which breaks proxy // forwarding of percent-encoded path segments such as /files/%2F. core := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if isExplicitProxyPath(r.URL.Path) && !hasCompleteProxyRouteHeaders(r.Header) { + setGatewayRouteSource(w, routeSourceHeader) + if _, hasSandbox := sandboxIDFromHeaders(r.Header); !hasSandbox { + http.Error(w, "sandbox id header required", http.StatusBadRequest) + return + } + http.Error(w, "target port header required", http.StatusBadRequest) + return + } if r.URL.Path == "/health" || r.URL.Path == "/metrics" { hostRoute, hostRouteErr := parseHostRoute(r.Host, s.sandboxProxyDomains) if hostRoute != nil || hostRouteErr != nil { @@ -122,7 +141,7 @@ func (s *Server) Handler() http.Handler { } s.handleProxy(w, r) }) - return s.instrumentGatewayHTTP(core) + return s.instrumentGatewayHTTP(s.authenticate(core)) } func (s *Server) writeJSON(w http.ResponseWriter, status int, value any) { @@ -532,6 +551,12 @@ func hasProxyRoutingHeaders(h http.Header) bool { return false } +func hasCompleteProxyRouteHeaders(h http.Header) bool { + _, hasSandbox := sandboxIDFromHeaders(h) + _, hasTargetPort := targetPortFromHeaders(h) + return hasSandbox && hasTargetPort +} + func targetPortFromHeaders(h http.Header) (string, bool) { for _, name := range []string{headerTargetPort, headerE2BTargetPort} { v := strings.TrimSpace(h.Get(name)) @@ -812,3 +837,52 @@ func extractSandboxIDsFromResponse(body []byte) []string { } return unique } + +func singleHeaderMatches(headers http.Header, name string, expected []byte) bool { + values := headers.Values(name) + if len(values) != 1 || len(values[0]) != len(expected) { + return false + } + return subtle.ConstantTimeCompare([]byte(values[0]), expected) == 1 +} + +func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { + if isExplicitProxyPath(r.URL.Path) { + // The explicit proxy prefix cannot dispatch to a control-plane handler. + // Let the core handler return a stable 400 for incomplete routing data. + return true + } + + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if hostRoute != nil { + return true + } + if err != nil { + return false + } + + return !isSandboxControlPlaneRequest(r) && hasCompleteProxyRouteHeaders(r.Header) +} + +func isExplicitProxyPath(path string) bool { + return path == "/proxy" || strings.HasPrefix(path, "/proxy/") +} + +func (s *Server) authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dataPlane := s.isSandboxDataPlaneRequest(r) + if dataPlane || r.URL.Path == "/health" || r.URL.Path == "/metrics" { + // Sandbox-scoped ingress and envd authorization depend on runtime + // metadata and are enforced by the owning runtime node. + next.ServeHTTP(w, r) + return + } + + if !singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) { + w.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) +} diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 09881150..c9d4be87 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,6 +148,8 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } +const testAPIKey = "test-api-key" + type testServerOption func(*ServerOptions) func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, timeout time.Duration, maxRespSize int64, opts ...testServerOption) *Server { @@ -156,6 +158,7 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti options := ServerOptions{ RequestTimeout: timeout, MaxResponseSize: maxRespSize, + APIKey: testAPIKey, } for _, opt := range opts { opt(&options) @@ -168,6 +171,165 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti return server } +func authenticatedTestHandler(server *Server) http.Handler { + handler := server.Handler() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Header.Set(headerAPIKey, testAPIKey) + handler.ServeHTTP(w, r) + }) +} + +func TestNewServerRejectsEmptyAPIKey(t *testing.T) { + _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ + RequestTimeout: time.Second, + MaxResponseSize: 1024, + }) + if err == nil { + t.Fatal("NewServer accepted an empty API key") + } +} + +func TestGatewayRequiresExactAPIKey(t *testing.T) { + server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) + handler := server.Handler() + tests := []struct { + name string + addHeaders func(http.Header) + wantStatus int + }{ + { + name: "missing", + addHeaders: func(http.Header) {}, + wantStatus: http.StatusUnauthorized, + }, + { + name: "wrong", + addHeaders: func(headers http.Header) { + headers.Set(headerAPIKey, "wrong-key") + }, + wantStatus: http.StatusUnauthorized, + }, + { + name: "authorization is application data", + addHeaders: func(headers http.Header) { + headers.Set("Authorization", "Bearer "+testAPIKey) + }, + wantStatus: http.StatusUnauthorized, + }, + { + name: "valid", + addHeaders: func(headers http.Header) { + headers.Set(headerAPIKey, testAPIKey) + }, + wantStatus: http.StatusBadGateway, + }, + { + name: "duplicate", + addHeaders: func(headers http.Header) { + headers.Add(headerAPIKey, testAPIKey) + headers.Add(headerAPIKey, testAPIKey) + }, + wantStatus: http.StatusUnauthorized, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/nodes", nil) + tt.addHeaders(req.Header) + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, req) + + if recorder.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, tt.wantStatus) + } + }) + } +} + +func TestGatewayMetricsPathDoesNotRequireAPIKey(t *testing.T) { + server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) + recorder := httptest.NewRecorder() + + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusNotFound) + } +} + +func TestGatewayLeavesDataPlaneAuthorizationToRuntime(t *testing.T) { + const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" + lookupCalls := 0 + server := newTestServer(t, stubSchedulerClient{ + lookupNodeFunc: func(context.Context, *schedulerv1.LookupNodeRequest, ...grpc.CallOption) (*schedulerv1.LookupNodeResponse, error) { + lookupCalls++ + return nil, fmt.Errorf("lookup reached") + }, + }, time.Second, 1024) + handler := server.Handler() + + for i, tt := range []struct { + port, header, value string + }{ + {port: "8080"}, + {port: "49983", header: headerTrafficToken, value: "runtime-validates-this-token"}, + {port: "49983", header: headerTrafficToken, value: "wrong-token"}, + {port: "8080", header: headerEnvdAccessToken, value: "runtime-validates-this-token"}, + } { + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, tt.port) + if tt.header != "" { + req.Header.Set(tt.header, tt.value) + } + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code == http.StatusUnauthorized || lookupCalls != i+1 { + t.Fatalf("data plane case %d: status=%d lookup calls=%d", i, recorder.Code, lookupCalls) + } + } + + req := httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusUnauthorized || lookupCalls != 4 { + t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } +} + +func TestGatewayRejectsIncompleteProxyRouteBeforeScheduling(t *testing.T) { + scheduleCalls := 0 + server := newTestServer(t, stubSchedulerClient{ + scheduleFunc: func(context.Context, *schedulerv1.ScheduleRequest, ...grpc.CallOption) (*schedulerv1.ScheduleResponse, error) { + scheduleCalls++ + return nil, fmt.Errorf("schedule reached") + }, + }, time.Second, 1024) + handler := server.Handler() + + for _, headers := range []http.Header{ + {}, + {headerE2BSandboxID: []string{"sandbox-only"}}, + {headerE2BTargetPort: []string{"8080"}}, + } { + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header = headers + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest) + } + } + if scheduleCalls != 0 { + t.Fatalf("schedule calls = %d, want 0", scheduleCalls) + } +} + func withSandboxProxyDomains(domains ...string) testServerOption { return func(options *ServerOptions) { options.SandboxProxyDomains = domains @@ -398,7 +560,7 @@ func TestHandleProxyReturnsAggregatedNodesFromScheduler(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "http://gateway.test/nodes?clusterID=cluster-1", nil) response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) + authenticatedTestHandler(server).ServeHTTP(response, request) if response.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", response.Code) @@ -453,7 +615,7 @@ func TestHandleProxyDirectForwardsNodeDetail(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "http://gateway.test/nodes/node-a?clusterID=cluster-1", nil) response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) + authenticatedTestHandler(server).ServeHTTP(response, request) if response.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", response.Code) @@ -486,7 +648,7 @@ func TestLookupNodeUsesQueryOnlySchedulerClient(t *testing.T) { } server := newTestServer(t, mainScheduler, time.Second, 1024, withQueryOnlyScheduler(queryScheduler)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/health", nil) @@ -555,7 +717,7 @@ func TestSandboxControlPlaneRequestWithE2BHeadersUsesPathRoute(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes/sbx-path/connect", strings.NewReader(`{"timeout":60}`)) @@ -833,7 +995,7 @@ func TestHandleProxyAggregatesSandboxListAcrossNodes(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/sandboxes?metadata=team%3Dalpha", nil) @@ -909,7 +1071,7 @@ func TestHandleProxyAggregatesV2SandboxesWithGlobalPagination(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/v2/sandboxes?metadata=team%3Dalpha&state=running%2Cpaused&limit=2", nil) @@ -1012,7 +1174,7 @@ func TestHandleProxyAggregatesSandboxListDedupsDuplicateSandboxIDs(t *testing.T) }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/v2/sandboxes?limit=10") @@ -1055,7 +1217,7 @@ func TestHandleProxyClusterListFailsWhenNodeFails(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/sandboxes") @@ -1085,7 +1247,7 @@ func TestHandleProxyClusterListPropagatesUnauthorized(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/sandboxes") @@ -1379,7 +1541,7 @@ func TestMetricsEndpointReturnsNotFoundWithoutProxyRouting(t *testing.T) { func TestHealthEndpointReturnsGatewayHealthWithoutProxyHeaders(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/health") @@ -1430,7 +1592,7 @@ func TestHealthAndMetricsEndpointsWithSandboxHeadersProxyToSandbox(t *testing.T) }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() for _, path := range []string{"/health", "/metrics"} { @@ -1468,7 +1630,7 @@ func TestHealthAndMetricsEndpointsWithSandboxHeadersProxyToSandbox(t *testing.T) func TestHealthAndMetricsEndpointsWithProxyHeadersMissingSandboxIDReturnBadRequest(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() for _, path := range []string{"/health", "/metrics"} { @@ -1523,7 +1685,7 @@ func TestHealthAndMetricsEndpointsWithHostRoutingProxyToSandbox(t *testing.T) { }, nil }, }, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() for _, path := range []string{"/health", "/metrics"} { @@ -1602,7 +1764,7 @@ func TestHandleProxyHostBasedRoutingForwardsToSandboxProxy(t *testing.T) { }, }, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/readyz?x=1", nil) @@ -1650,7 +1812,7 @@ func TestHandleProxyHostBasedRoutingForwardsToSandboxProxy(t *testing.T) { func TestHandleProxyHostBasedRoutingRejectsInvalidHost(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/readyz", nil) @@ -1736,7 +1898,7 @@ func TestHandleProxyHTTPForwardingAndRecordAssignment(t *testing.T) { }, }, time.Second, 1024, withDebugMode(true)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes", strings.NewReader(`{"template":"base"}`)) @@ -1860,7 +2022,7 @@ func TestHandleProxyColdSandboxCreateRecordsAssignment(t *testing.T) { }, }, time.Second, 1024, withDebugMode(true)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes-cold", strings.NewReader(`{"image":"ubuntu:24.04"}`)) @@ -1992,7 +2154,7 @@ func TestHandleProxyWebSocketForwarding(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() gatewayURL, err := url.Parse(gatewayServer.URL) @@ -2108,7 +2270,7 @@ func TestHandleProxyPreservesEncodedPathSegments(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() tests := []struct { diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index c6362583..518f97e0 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -1,41 +1,129 @@ +use super::{ApiImpl, Claims}; +use crate::{api::proxy, types::SandboxId}; +use agentenv_http_server::apis; use async_trait::async_trait; -use axum::http::header::HeaderMap; +use axum::{ + body::Body, + extract::{Request, State}, + http::{header::HeaderMap, HeaderValue, StatusCode}, + middleware::Next, + response::{IntoResponse, Response}, +}; -use agentenv_http_server::apis; +pub(crate) const API_KEY_HEADER: &str = "x-api-key"; +pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; +pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; -use super::{ApiImpl, Claims}; +fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a HeaderValue> { + let mut values = headers.get_all(name).iter(); + let value = values.next()?; + values.next().is_none().then_some(value) +} + +impl ApiImpl { + pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { + single_header(headers, API_KEY_HEADER) + .is_some_and(|value| self.api_key.matches(value.as_bytes())) + } + + pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { + self.orchestrator.traffic_access_token(sandbox_id) + } -fn non_empty_header(headers: &HeaderMap, name: &str) -> bool { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| !value.is_empty()) + fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: SandboxId) -> bool { + single_header(headers, TRAFFIC_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| { + self.orchestrator + .validate_traffic_access_token(sandbox_id, candidate) + }) + } +} + +pub(crate) async fn require_auth( + State(api_impl): State, + mut request: Request, + next: Next, +) -> Response +where + I: AsRef + Clone + Send + Sync + 'static, +{ + let proxy_request = + proxy::is_sandbox_proxy_request(&request, api_impl.as_ref().sandbox_proxy_domains()); + if matches!(request.uri().path(), "/health" | "/metrics") && !proxy_request { + return next.run(request).await; + } + + let api_impl = api_impl.as_ref(); + if !proxy_request { + return if api_impl.has_valid_api_key(request.headers()) { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + }; + } + + let has_api_key = api_impl.has_valid_api_key(request.headers()); + + let Some((sandbox_id, target_port)) = + proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) + else { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + return if proxy::has_proxy_prefix(request.uri().path()) || has_api_key { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + }; + }; + if has_api_key { + request.headers_mut().remove(API_KEY_HEADER); + } + let metadata = match api_impl.orchestrator().get_sandbox(&sandbox_id).await { + Ok(Some(metadata)) => metadata, + Ok(None) => { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + return proxy::sandbox_not_found_response(sandbox_id); + } + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + + let envd_request = target_port == proxy::effective_envd_port(&metadata); + let envd_authorized = envd_request + && metadata.secure + && single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| { + api_impl + .orchestrator() + .validate_envd_access_token(sandbox_id, candidate) + }); + let authorized = if envd_request { + !metadata.secure || envd_authorized + } else { + metadata.network_policy.allow_public_traffic + || api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id) + }; + + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + if !envd_authorized { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + } + + next.run(request).await } #[async_trait] impl apis::ApiKeyAuthHeader for ApiImpl { type Claims = Claims; - // TODO: Validate configured authentication credentials instead of only - // checking that they are present. async fn extract_claims_from_header( &self, headers: &HeaderMap, - key: &str, + _key: &str, ) -> Option { - let admin_token = non_empty_header(headers, "X-Admin-Token"); - if key == "X-Admin-Token" { - return admin_token.then_some(Claims); - } - - if non_empty_header(headers, "X-API-Key") - || non_empty_header(headers, "X-Team-ID") - || admin_token - { - Some(Claims) - } else { - None - } + self.has_valid_api_key(headers).then_some(Claims) } } @@ -45,17 +133,12 @@ impl apis::ApiAuthBasic for ApiImpl { async fn extract_claims_from_auth_header( &self, - kind: apis::BasicAuthKind, + _kind: apis::BasicAuthKind, headers: &HeaderMap, - key: &str, + _key: &str, ) -> Option { - let expected_scheme = match kind { - apis::BasicAuthKind::Basic => "Basic", - apis::BasicAuthKind::Bearer => "Bearer", - _ => return None, - }; - let value = headers.get(key)?.to_str().ok()?; - let (scheme, credentials) = value.split_once(' ')?; - (scheme.eq_ignore_ascii_case(expected_scheme) && !credentials.is_empty()).then_some(Claims) + // The outer middleware is authoritative. This adapter keeps the + // E2B-compatible generated router from rejecting its API-key request. + self.has_valid_api_key(headers).then_some(Claims) } } diff --git a/src/api/impls/mod.rs b/src/api/impls/mod.rs index 1e1d50bc..47787eb5 100644 --- a/src/api/impls/mod.rs +++ b/src/api/impls/mod.rs @@ -1,6 +1,6 @@ mod admin; mod attached_drives; -mod auth; +pub(crate) mod auth; mod pagination; mod sandbox; mod snapshots; @@ -13,6 +13,7 @@ use anyhow::Error as AnyhowError; use async_trait::async_trait; use super::proxy::{build_proxy_client, ProxyClient}; +use crate::api_key::ApiKey; use crate::image::ImageResolver; use crate::observability::ObservabilityService; use crate::orchestrator::Orchestrator; @@ -33,6 +34,7 @@ pub struct ApiImpl { observability: Option>, proxy_client: ProxyClient, sandbox_proxy_domains: Vec, + api_key: ApiKey, } impl ApiImpl { @@ -43,6 +45,7 @@ impl ApiImpl { image_resolver: Arc, observability: Option>, sandbox_proxy_domains: Vec, + api_key: ApiKey, ) -> Self { Self { orchestrator, @@ -52,6 +55,7 @@ impl ApiImpl { observability, proxy_client: build_proxy_client(), sandbox_proxy_domains, + api_key, } } diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 5382dfb6..dbd41737 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -145,7 +145,7 @@ impl From<&SandboxNetworkPolicy> for models::SandboxNetworkConfig { fn from(policy: &SandboxNetworkPolicy) -> Self { let egress = &policy.egress; Self { - allow_public_traffic: Some(true), + allow_public_traffic: Some(policy.allow_public_traffic), allow_out: (!egress.allowed_cidrs.is_empty() || !egress.allowed_domains.is_empty()) .then(|| { egress @@ -179,10 +179,9 @@ fn allow_internet_access_from_base_policy(policy: BaseSandboxNetworkPolicy) -> N impl From for models::SandboxDetail { fn from(m: SandboxMetadata) -> Self { - let network = m - .network_policy - .has_explicit_egress_rules() - .then(|| models::SandboxNetworkConfig::from(&m.network_policy)); + let network = (!m.network_policy.allow_public_traffic + || m.network_policy.has_explicit_egress_rules()) + .then(|| models::SandboxNetworkConfig::from(&m.network_policy)); let allow_internet_access = Some(allow_internet_access_from_base_policy( m.network_policy.base_policy, )); @@ -214,12 +213,18 @@ impl From for models::SandboxDetail { impl ApiImpl { fn sandbox_model(&self, metadata: SandboxMetadata) -> models::Sandbox { + let traffic_access_token = (!metadata.network_policy.allow_public_traffic) + .then(|| self.traffic_access_token(metadata.id)); let envd_access_token = self .orchestrator .get_envd_access_token(&metadata) .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; + sandbox.traffic_access_token = Some(match traffic_access_token { + Some(token) => Nullable::Present(token), + None => Nullable::Null, + }); sandbox.domain = self .sandbox_proxy_domains() .first() @@ -364,7 +369,10 @@ fn network_policy_from_create( let allow_out = network.and_then(|network| network.allow_out.clone()); let deny_out = network.and_then(|network| network.deny_out.clone()); let egress = SandboxNetworkEgressPolicy::new(allow_out, deny_out)?; - let policy = SandboxNetworkPolicy::new(base_policy, egress); + let allow_public_traffic = network + .and_then(|network| network.allow_public_traffic) + .unwrap_or(true); + let policy = SandboxNetworkPolicy::new(allow_public_traffic, base_policy, egress); if policy.has_domain_allow_rules() { anyhow::bail!( "domain entries in allowOut are not supported until TCP egress proxy is enabled" @@ -383,6 +391,7 @@ fn network_policy_from_update( ); } Ok(SandboxNetworkPolicy::new( + true, base_policy_from_allow_internet_access(body.allow_internet_access), policy, )) @@ -1503,6 +1512,20 @@ mod tests { assert_eq!(policy.egress.denied_cidrs, ["203.0.113.0/24"]); } + #[test] + fn network_create_preserves_private_ingress() { + let mut network = models::SandboxNetworkConfig::new(); + network.allow_public_traffic = Some(false); + + let policy = network_policy_from_create(None, Some(&network)).unwrap(); + + assert!(!policy.allow_public_traffic); + assert_eq!( + models::SandboxNetworkConfig::from(&policy).allow_public_traffic, + Some(false) + ); + } + #[test] fn empty_network_update_clears_base_policy_and_egress() { let policy = network_policy_from_update(&models::SandboxNetworkUpdateConfig::new()) diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 326c847f..ead17710 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -7,7 +7,7 @@ use axum::{ rejection::WebSocketUpgradeRejection, CloseFrame, Message as WebSocketMessage, WebSocket, WebSocketUpgrade, }, - FromRequestParts, Request, State, + FromRequestParts, MatchedPath, Request, State, }, http::{header, HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, Uri}, middleware::Next, @@ -35,13 +35,19 @@ use tokio_tungstenite::{ use tracing::{debug, info, trace, warn}; use crate::{ - api::ApiImpl, + api::{impls::auth::TRAFFIC_ACCESS_TOKEN_HEADER, ApiImpl}, cfg::ConfigManager, observability::prometheus::HttpRouteSource, - orchestrator::{NewTimeout, OrchestratorError, ProxyLookupResult, ProxyTarget, SandboxState}, + orchestrator::{ + NewTimeout, OrchestratorError, ProxyLookupResult, ProxyTarget, SandboxMetadata, + SandboxState, + }, types::SandboxId, }; +#[cfg(test)] +use crate::api::impls::auth::{API_KEY_HEADER, ENVD_ACCESS_TOKEN_HEADER}; + /// Shared outbound HTTP client for the client-facing reverse proxy. pub(crate) type ProxyClient = Client; type UpstreamWebSocket = WebSocketStream>; @@ -83,8 +89,6 @@ const E2B_SANDBOX_ID_HEADER: &str = "e2b-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; /// E2B-compatible alias for the target port header. const E2B_TARGET_PORT_HEADER: &str = "e2b-sandbox-port"; -const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; - #[cfg(test)] const PROXY_CONNECT_TIMEOUT: Duration = Duration::from_millis(100); #[cfg(not(test))] @@ -142,6 +146,49 @@ where .with_state(api_impl) } +pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { + if !has_proxy_prefix(request.uri().path()) { + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), + Ok(None) => {} + Err(_) => return None, + } + } + + Some(( + parse_sandbox_id_header(request.headers()).ok()?, + parse_target_port_header(request.headers()).ok()?, + )) +} + +pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { + let path = request.uri().path(); + if has_proxy_prefix(path) { + return true; + } + + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(_)) | Err(_) => true, + Ok(None) => { + request.extensions().get::().is_none() + && has_routing_header(request.headers()) + } + } +} + +fn request_host(request: &Request) -> Option<&str> { + request + .headers() + .get(header::HOST) + .and_then(|host| host.to_str().ok()) + .or_else(|| { + request + .uri() + .authority() + .map(|authority| authority.as_str()) + }) +} + pub(crate) async fn sandbox_proxy_classifier( State(api_impl): State, request: Request, @@ -151,23 +198,18 @@ where I: AsRef + Clone + Send + Sync + 'static, { let path = request.uri().path(); - if path == PROXY_ROUTE || path.starts_with("/proxy/") { + if has_proxy_prefix(path) { return next.run(request).await; } - let host = request - .headers() - .get(header::HOST) - .and_then(|host| host.to_str().ok()) - .or_else(|| { - request - .uri() - .authority() - .map(|authority| authority.as_str()) - }); - let host_route = match parse_host_proxy_route(host, api_impl.as_ref().sandbox_proxy_domains()) { + let host_route = match parse_host_proxy_route( + request_host(&request), + api_impl.as_ref().sandbox_proxy_domains(), + ) { Ok(Some(route)) => route, - Ok(None) => return next.run(request).await, + Ok(None) => { + return next.run(request).await; + } Err(err) => { return with_route_source(proxy_error_response(&err), HttpRouteSource::ProxyHost); } @@ -278,6 +320,10 @@ fn strip_proxy_prefix(path: &str) -> &str { path.strip_prefix(PROXY_ROUTE).unwrap_or("") } +pub(crate) fn has_proxy_prefix(path: &str) -> bool { + path == PROXY_ROUTE || path.starts_with("/proxy/") +} + fn parse_host_proxy_route( raw_host: Option<&str>, domains: &[String], @@ -358,6 +404,14 @@ fn has_routing_header(headers: &HeaderMap) -> bool { headers.get(SANDBOX_ID_HEADER).is_some() || headers.get(E2B_SANDBOX_ID_HEADER).is_some() } +pub(crate) fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { + metadata + .paused_state + .as_ref() + .and_then(|state| state.control_plane_port()) + .unwrap_or_else(|| ConfigManager::global_config().tools.control_plane_port) +} + /// Proxies a standard HTTP request to the resolved upstream URI and returns the response. async fn proxy_http_request( api_impl: &ApiImpl, @@ -680,13 +734,6 @@ async fn resolve_proxy_request( sandbox_id, ))); } - authorize_secure_envd_auto_resume( - api_impl, - sandbox_id, - target_port, - &parts.headers, - ) - .await?; try_auto_resume(api_impl, sandbox_id).await?; auto_resume_attempted = true; continue; @@ -749,45 +796,6 @@ async fn resolve_proxy_request( }) } -async fn authorize_secure_envd_auto_resume( - api_impl: &ApiImpl, - sandbox_id: SandboxId, - target_port: u16, - headers: &HeaderMap, -) -> Result<(), Response> { - if target_port != ConfigManager::global_config().tools.control_plane_port { - return Ok(()); - } - let metadata = api_impl - .orchestrator() - .get_sandbox(&sandbox_id) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; - if !metadata.secure { - return Ok(()); - } - let candidate = headers - .get(ENVD_ACCESS_TOKEN_HEADER) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default(); - if api_impl - .orchestrator() - .validate_envd_access_token(sandbox_id, candidate) - { - return Ok(()); - } - - Err(Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ) - .body(Body::from("invalid or missing envd access token")) - .expect("static unauthorized proxy response is valid")) -} - async fn try_auto_resume(api_impl: &ApiImpl, sandbox_id: SandboxId) -> Result<(), Response> { match timeout( PROXY_AUTO_RESUME_TIMEOUT, @@ -839,6 +847,10 @@ fn parse_target_port_header(headers: &HeaderMap) -> Result Response { + proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)) +} + fn proxy_error_response(error: &ProxyRequestError) -> Response { let (status, message) = match error { ProxyRequestError::MissingSandboxId => { @@ -970,6 +982,7 @@ fn sanitize_request_headers(headers: &mut HeaderMap) { headers.remove(E2B_SANDBOX_ID_HEADER); headers.remove(TARGET_PORT_HEADER); headers.remove(E2B_TARGET_PORT_HEADER); + headers.remove(TRAFFIC_ACCESS_TOKEN_HEADER); headers.remove(header::HOST); remove_hop_by_hop_headers(headers); } @@ -1219,6 +1232,7 @@ mod tests { use crate::{ api::server, + api_key::ApiKey, cfg::AppConfig, image::ImageResolver, orchestrator::{FileBackedSandboxPersister, Orchestrator}, @@ -1226,6 +1240,9 @@ mod tests { template::TemplateBuilder, }; + const TEST_API_KEY: &str = + "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + #[test] fn strip_host_port_handles_dns_and_ipv6_hosts() { for (host, expected) in [ @@ -1389,9 +1406,14 @@ mod tests { "e2b_sandbox_header_seen": headers.get(E2B_SANDBOX_ID_HEADER).is_some(), "target_port_header_seen": headers.get(TARGET_PORT_HEADER).is_some(), "e2b_target_port_header_seen": headers.get(E2B_TARGET_PORT_HEADER).is_some(), - "envd_access_token": headers + "api_key_header_seen": headers.get(API_KEY_HEADER).is_some(), + "traffic_token_header_seen": headers.get(TRAFFIC_ACCESS_TOKEN_HEADER).is_some(), + "access_token": headers .get(ENVD_ACCESS_TOKEN_HEADER) .and_then(|value| value.to_str().ok()), + "authorization": headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), "forwarded_host": headers .get("x-forwarded-host") .and_then(|value| value.to_str().ok()), @@ -1604,6 +1626,10 @@ mod tests { } async fn build_api_with_sandbox_proxy_domains(domains: Vec) -> Arc { + build_api_with_auth(domains, TEST_API_KEY).await + } + + async fn build_api_with_auth(domains: Vec, api_key: &str) -> Arc { let root = tempfile::tempdir().unwrap(); let orchestrator = Orchestrator::new( crate::orchestrator::InMemoryMetadataStore::new(), @@ -1622,6 +1648,7 @@ mod tests { image_resolver, None, domains, + ApiKey::new(api_key).unwrap(), )) } @@ -1650,11 +1677,31 @@ mod tests { .await } + async fn proxy_app_with_access_token_for_sandbox( + sandbox_id: &SandboxId, + ) -> (axum::Router, String) { + let api = build_api().await; + let access_token = api.traffic_access_token(*sandbox_id); + api.orchestrator() + .set_proxy_target_for_test( + *sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + api.orchestrator() + .set_allow_public_traffic_for_test(sandbox_id, false) + .await + .unwrap(); + (server::new(api), access_token) + } + async fn proxy_app_for_sandbox_with_domains( sandbox_id: &SandboxId, domains: Vec, - ) -> axum::Router { + ) -> (axum::Router, String) { let api = build_api_with_sandbox_proxy_domains(domains).await; + let access_token = api.traffic_access_token(*sandbox_id); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -1662,7 +1709,11 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; - server::new(api) + api.orchestrator() + .set_allow_public_traffic_for_test(sandbox_id, false) + .await + .unwrap(); + (server::new(api), access_token) } async fn proxy_app_for_running_sandbox_without_route(sandbox_id: &SandboxId) -> axum::Router { @@ -1681,6 +1732,20 @@ mod tests { spawn_upstream(proxy_app_for_sandbox(sandbox_id).await).await } + async fn get_status(app: &axum::Router, uri: &str, headers: &[(&str, &str)]) -> StatusCode { + let mut request = Request::builder() + .uri(uri) + .header(header::HOST, "localhost"); + for (name, value) in headers { + request = request.header(*name, *value); + } + app.clone() + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap() + .status() + } + #[test] fn parses_agentenv_headers() { let sandbox_id = SandboxId::new().to_string(); @@ -1725,6 +1790,11 @@ mod tests { ); headers.insert(TARGET_PORT_HEADER, HeaderValue::from_static("8080")); headers.insert(E2B_TARGET_PORT_HEADER, HeaderValue::from_static("8080")); + headers.insert(API_KEY_HEADER, HeaderValue::from_static("application-key")); + headers.insert( + TRAFFIC_ACCESS_TOKEN_HEADER, + HeaderValue::from_static("traffic-token"), + ); headers.insert(HOST, HeaderValue::from_static("client.example")); headers.insert(header::CONNECTION, HeaderValue::from_static("keep-alive")); headers.insert( @@ -1738,6 +1808,8 @@ mod tests { assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); assert!(headers.get(TARGET_PORT_HEADER).is_none()); assert!(headers.get(E2B_TARGET_PORT_HEADER).is_none()); + assert_eq!(headers.get(API_KEY_HEADER).unwrap(), "application-key"); + assert!(headers.get(TRAFFIC_ACCESS_TOKEN_HEADER).is_none()); assert!(headers.get(HOST).is_none()); assert!(headers.get(header::CONNECTION).is_none()); assert_eq!(headers.get("x-extra").unwrap(), "keep"); @@ -1786,6 +1858,181 @@ mod tests { assert!(!is_send_request_failure_text(&"client error (Connect)")); } + #[tokio::test] + async fn control_plane_auth_is_separate_from_sandbox_auth() { + let app = server::new(build_api().await); + + for headers in [ + vec![], + vec![(header::AUTHORIZATION.as_str(), "Bearer test-key")], + vec![(API_KEY_HEADER, "wrong-key")], + vec![ + (API_KEY_HEADER, TEST_API_KEY), + (API_KEY_HEADER, TEST_API_KEY), + ], + ] { + assert_eq!( + get_status(&app, "/nonexistent/path", &headers).await, + StatusCode::UNAUTHORIZED + ); + } + + for (path, headers, expected) in [ + ( + "/nonexistent/path", + vec![(API_KEY_HEADER, TEST_API_KEY)], + StatusCode::NOT_FOUND, + ), + ("/health", vec![], StatusCode::NO_CONTENT), + ] { + assert_eq!(get_status(&app, path, &headers).await, expected); + } + assert_ne!( + get_status(&app, "/metrics", &[]).await, + StatusCode::UNAUTHORIZED + ); + + let sandbox_id = SandboxId::new().to_string(); + let route = [ + (SANDBOX_ID_HEADER, sandbox_id.as_str()), + (TARGET_PORT_HEADER, "8080"), + ]; + assert_eq!( + get_status(&app, "/sandboxes", &route).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + get_status(&app, "/proxy/health", &route).await, + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn application_proxy_auth_respects_public_and_private_ingress() { + let upstream_addr = start_upstream_server().await; + let api = build_api().await; + let sandbox_id = SandboxId::new(); + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + + let app = server::new(Arc::clone(&api)); + let sandbox_id_text = sandbox_id.to_string(); + let port = upstream_addr.port().to_string(); + let route = [ + (SANDBOX_ID_HEADER, sandbox_id_text.as_str()), + (TARGET_PORT_HEADER, port.as_str()), + ]; + assert_eq!( + get_status(&app, "/proxy/public", &route).await, + StatusCode::OK + ); + + api.orchestrator() + .set_allow_public_traffic_for_test(&sandbox_id, false) + .await + .unwrap(); + let traffic_token = api.traffic_access_token(sandbox_id); + + for credential in [ + None, + Some((API_KEY_HEADER, TEST_API_KEY)), + Some((TRAFFIC_ACCESS_TOKEN_HEADER, "incorrect")), + Some((ENVD_ACCESS_TOKEN_HEADER, "envd-token")), + ] { + let mut headers = route.to_vec(); + if let Some((header_name, value)) = credential { + headers.push((header_name, value)); + } + assert_eq!( + get_status(&app, "/proxy/private", &headers).await, + StatusCode::UNAUTHORIZED + ); + } + + let mut headers = route.to_vec(); + headers.push((TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str())); + headers.push((API_KEY_HEADER, "application-api-key")); + assert_eq!( + get_status(&app, "/proxy/private", &headers).await, + StatusCode::OK + ); + } + + #[tokio::test] + async fn envd_proxy_auth_depends_only_on_secure_mode_and_envd_token() { + let api = build_api().await; + let sandbox_id = SandboxId::new(); + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + let target_port = ConfigManager::global_config() + .tools + .control_plane_port + .to_string(); + let app = server::new(Arc::clone(&api)); + let sandbox_id_text = sandbox_id.to_string(); + let route = [ + (SANDBOX_ID_HEADER, sandbox_id_text.as_str()), + (TARGET_PORT_HEADER, target_port.as_str()), + ]; + let envd_paths = ["/proxy/health", "/proxy/metrics"]; + for path in envd_paths { + assert_ne!( + get_status(&app, path, &route).await, + StatusCode::UNAUTHORIZED + ); + } + + api.orchestrator() + .set_secure_for_test(&sandbox_id, true) + .await + .unwrap(); + let metadata = api + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .unwrap() + .unwrap(); + let envd_token = api.orchestrator().get_envd_access_token(&metadata).unwrap(); + let traffic_token = api.traffic_access_token(sandbox_id); + + for credential in [ + None, + Some((API_KEY_HEADER, TEST_API_KEY)), + Some((TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str())), + Some((ENVD_ACCESS_TOKEN_HEADER, "incorrect")), + ] { + for path in envd_paths { + let mut headers = route.to_vec(); + if let Some((header_name, value)) = credential { + headers.push((header_name, value)); + } + assert_eq!( + get_status(&app, path, &headers).await, + StatusCode::UNAUTHORIZED + ); + } + } + + let mut headers = route.to_vec(); + headers.push((ENVD_ACCESS_TOKEN_HEADER, envd_token.expose())); + for path in envd_paths { + assert_ne!( + get_status(&app, path, &headers).await, + StatusCode::UNAUTHORIZED + ); + } + } + #[tokio::test] async fn proxy_requires_routing_headers() { let app = server::new(build_api().await); @@ -1795,7 +2042,6 @@ mod tests { .oneshot( Request::builder() .uri("/proxy/hello") - .header("x-api-key", "test-key") .body(Body::empty()) .unwrap(), ) @@ -1825,7 +2071,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") + .header("x-api-key", "application-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -1858,7 +2104,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -1907,7 +2152,6 @@ mod tests { let mut request = Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header( TARGET_PORT_HEADER, @@ -1932,7 +2176,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header( TARGET_PORT_HEADER, @@ -1961,7 +2204,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -1988,7 +2230,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, "0") .body(Body::empty()) @@ -2016,7 +2257,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy?foo=bar") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2034,7 +2274,7 @@ mod tests { } #[tokio::test] - async fn proxy_forwards_request_and_strips_internal_headers() { + async fn proxy_forwards_application_headers_and_strips_internal_headers() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); let app = proxy_app_for_sandbox(&sandbox_id).await; @@ -2045,10 +2285,12 @@ mod tests { .method(Method::GET) .uri("/proxy/echo/test?foo=bar".to_string()) .header("host", "client.example") - .header("x-api-key", "test-key") + .header(API_KEY_HEADER, "application-key") + .header(header::AUTHORIZATION, "Bearer application-token") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .header(ENVD_ACCESS_TOKEN_HEADER, "envd-token") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, "traffic-token") .body(Body::empty()) .unwrap(), ) @@ -2065,8 +2307,11 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); - assert_eq!(payload["envd_access_token"], "envd-token"); + assert!(payload["access_token"].is_null()); + assert_eq!(payload["traffic_token_header_seen"], false); assert_eq!(payload["forwarded_host"], "client.example"); + assert_eq!(payload["api_key_header_seen"], true); + assert_eq!(payload["authorization"], "Bearer application-token"); } #[tokio::test] @@ -2080,7 +2325,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/a%2Fb/%2525") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2107,7 +2351,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy//api") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2134,7 +2377,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/check") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .header(header::CONNECTION, "foo") @@ -2170,7 +2412,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/reject") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2204,7 +2445,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/api/files?path=/") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2233,7 +2473,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/envd/health") - .header("x-api-key", "test-key") .header(E2B_SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(E2B_TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2256,7 +2495,7 @@ mod tests { .oneshot( Request::builder() .uri("/nonexistent/path") - .header("x-api-key", "test-key") + .header(API_KEY_HEADER, TEST_API_KEY) .body(Body::empty()) .unwrap(), ) @@ -2288,18 +2527,40 @@ mod tests { async fn sandbox_proxy_host_routes_control_paths_and_skips_explicit_proxy() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); - let app = proxy_app_for_sandbox_with_domains( + let (app, access_token) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) .await; + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/health") + .header( + "host", + format!( + "{}-{}.sandbox.example.invalid", + upstream_addr.port(), + sandbox_id + ), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = app .clone() .oneshot( Request::builder() .method(Method::GET) .uri("/health?foo=bar") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .header( "host", format!( @@ -2332,6 +2593,7 @@ mod tests { upstream_addr.port(), sandbox_id )) + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .body(Body::empty()) .unwrap(), ) @@ -2343,7 +2605,7 @@ mod tests { let payload: Value = serde_json::from_slice(&body).unwrap(); assert_eq!(payload["path"], "/authority"); - let app = proxy_app_for_sandbox_with_domains( + let (app, access_token) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) @@ -2354,6 +2616,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .header( "host", format!( @@ -2375,6 +2638,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header( "host", format!( @@ -2401,14 +2665,14 @@ mod tests { async fn proxy_accepts_e2b_compatible_headers() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); - let app = proxy_app_for_sandbox(&sandbox_id).await; + let (app, access_token) = proxy_app_with_access_token_for_sandbox(&sandbox_id).await; let response = app .oneshot( Request::builder() .method(Method::GET) .uri("/proxy/e2b/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header(E2B_SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(E2B_TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2426,6 +2690,10 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); + assert!( + payload["access_token"].is_null(), + "envd credential leaked to application port" + ); } #[tokio::test] @@ -2439,7 +2707,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/events") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2488,7 +2755,6 @@ mod tests { Request::builder() .method(Method::POST) .uri("/proxy/upload") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::from_stream(body_stream)) @@ -2529,7 +2795,6 @@ mod tests { Request::builder() .method(Method::POST) .uri("/proxy/upload") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::from_stream(body_stream)) @@ -2556,7 +2821,6 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/slow") - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2593,7 +2857,6 @@ mod tests { Request::builder() .method(Method::POST) .uri(ENVD_STREAM_INPUT_PATH) - .header("x-api-key", "test-key") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::from_stream(body_stream)) @@ -2618,7 +2881,7 @@ mod tests { .unwrap(); request .headers_mut() - .insert("x-api-key", HeaderValue::from_static("test-key")); + .insert("x-api-key", HeaderValue::from_static(TEST_API_KEY)); request.headers_mut().insert( SANDBOX_ID_HEADER, HeaderValue::from_str(&sandbox_id.to_string()).unwrap(), @@ -2693,7 +2956,7 @@ mod tests { .unwrap(); request .headers_mut() - .insert("x-api-key", HeaderValue::from_static("test-key")); + .insert("x-api-key", HeaderValue::from_static(TEST_API_KEY)); request.headers_mut().insert( SANDBOX_ID_HEADER, HeaderValue::from_str(&sandbox_id.to_string()).unwrap(), diff --git a/src/api/server.rs b/src/api/server.rs index 2a738898..0aed4b7c 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -1,6 +1,6 @@ use axum::{middleware, routing::get, Router}; -use super::{proxy, ApiImpl}; +use super::{impls::auth, proxy, ApiImpl}; use crate::observability::prometheus; use agentenv_http_server::apis; use agentenv_observability::metrics_handler; @@ -28,8 +28,12 @@ where .merge(proxy::router(api_impl.clone())) .route("/metrics", get(metrics_handler)) .layer(middleware::from_fn_with_state( - api_impl, + api_impl.clone(), proxy::sandbox_proxy_classifier::, )) + .layer(middleware::from_fn_with_state( + api_impl, + auth::require_auth::, + )) .layer(middleware::from_fn(prometheus::http_metrics_middleware)) } diff --git a/src/api_key.rs b/src/api_key.rs new file mode 100644 index 00000000..62dd2490 --- /dev/null +++ b/src/api_key.rs @@ -0,0 +1,252 @@ +use std::ffi::OsStr; +use std::fmt; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use rand::{rngs::SysRng, TryRng}; +use subtle::ConstantTimeEq; +use tracing::info; + +use crate::cfg::AppConfig; +use crate::managed_secret::{self, CreateOutcome}; + +const API_KEY_ENV: &str = "AENV_API_KEY"; +const EXTERNAL_API_KEY_PATH: &str = "/run/secrets/api-key"; +const MANAGED_API_KEY_RELATIVE_PATH: &str = "secrets/api-key"; +const API_KEY_MAX_LEN: usize = 256; +const API_KEY_FILE_MAX_LEN: usize = API_KEY_MAX_LEN + 2; +const GENERATED_API_KEY_PREFIX: &str = "e2b_"; + +#[derive(Clone)] +pub struct ApiKey(String); + +impl fmt::Debug for ApiKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ApiKey([REDACTED])") + } +} + +impl ApiKey { + pub fn resolve(config: &AppConfig) -> Result { + Self::resolve_from( + std::env::var_os(API_KEY_ENV).as_deref(), + Path::new(EXTERNAL_API_KEY_PATH), + &config.home_path, + ) + } + + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if !(32..=API_KEY_MAX_LEN).contains(&value.len()) + || !value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-') + }) + { + bail!("API key must contain between 32 and {API_KEY_MAX_LEN} URL-safe characters"); + } + Ok(Self(value)) + } + + pub(crate) fn matches(&self, candidate: &[u8]) -> bool { + candidate.len() == self.0.len() && bool::from(candidate.ct_eq(self.0.as_bytes())) + } + + fn resolve_from( + explicit: Option<&OsStr>, + external_path: &Path, + home_path: &Path, + ) -> Result { + if let Some(explicit) = explicit { + return Self::new( + explicit + .to_str() + .context("AENV_API_KEY must contain valid UTF-8")?, + ) + .context("invalid AENV_API_KEY"); + } + + match Self::read_external(external_path) { + Ok(key) => { + info!(path = %external_path.display(), "loaded API key from external secret"); + return Ok(key); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("load external API key"), + } + + let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); + if let Some(value) = managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN) + .context("load managed API key")? + { + return Self::from_file_contents(&value).context("invalid managed API key"); + } + + Self::create(&managed_path) + } + + fn read_external(path: &Path) -> Result { + let file = Self::open_external(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "API key secret must be a regular file", + )); + } + let value = Self::read_bounded(file)?; + Self::from_file_contents(&value).map_err(io::Error::other) + } + + fn open_external(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.custom_flags(libc::O_NONBLOCK); + } + + options.open(path) + } + + fn read_bounded(file: File) -> Result { + let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); + file.take((API_KEY_FILE_MAX_LEN + 1) as u64) + .read_to_string(&mut value)?; + if value.len() > API_KEY_FILE_MAX_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("API key file must be at most {API_KEY_FILE_MAX_LEN} bytes"), + )); + } + Ok(value) + } + + fn from_file_contents(value: &str) -> Result { + let value = value.strip_suffix('\n').unwrap_or(value); + Self::new(value.strip_suffix('\r').unwrap_or(value)) + } + + fn create(path: &Path) -> Result { + let mut random = [0_u8; 32]; + SysRng + .try_fill_bytes(&mut random) + .context("generate managed API key")?; + let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); + + match managed_secret::create(path, format!("{key}\n").as_bytes())? { + CreateOutcome::Created => { + info!(path = %path.display(), "generated managed API key"); + Self::new(key) + } + CreateOutcome::Existing(file) => { + let value = managed_secret::read_file(path, file, API_KEY_FILE_MAX_LEN) + .context("load concurrently generated API key")?; + Self::from_file_contents(&value).context("invalid concurrently generated API key") + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + use tempfile::TempDir; + + const TEST_KEY: &str = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + #[test] + fn configured_sources_take_precedence() -> Result<()> { + let temp = TempDir::new()?; + let external_path = temp.path().join("external"); + fs::write(&external_path, format!("{TEST_KEY}\n"))?; + + assert_eq!( + ApiKey::resolve_from(Some(OsStr::new(TEST_KEY)), &external_path, temp.path())?.0, + TEST_KEY, + ); + assert_eq!( + ApiKey::resolve_from(None, &external_path, temp.path())?.0, + TEST_KEY, + ); + assert!(!temp.path().join(MANAGED_API_KEY_RELATIVE_PATH).exists()); + Ok(()) + } + + #[test] + fn external_secret_must_be_a_regular_file() -> Result<()> { + let temp = TempDir::new()?; + let external_path = temp.path().join("external"); + fs::create_dir(&external_path)?; + + let error = ApiKey::resolve_from(None, &external_path, temp.path()).unwrap_err(); + + assert!(error.to_string().contains("load external API key")); + assert!(format!("{error:#}").contains("must be a regular file")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn external_secret_allows_kubernetes_style_symlinks() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = TempDir::new()?; + let target_path = temp.path().join("target"); + let external_path = temp.path().join("external"); + fs::write(&target_path, format!("{TEST_KEY}\n"))?; + symlink(&target_path, &external_path)?; + + assert_eq!( + ApiKey::resolve_from(None, &external_path, temp.path())?.0, + TEST_KEY, + ); + Ok(()) + } + + #[test] + fn managed_key_is_private_and_stable() -> Result<()> { + let temp = TempDir::new()?; + let missing_external = temp.path().join("missing"); + let first = ApiKey::resolve_from(None, &missing_external, temp.path())?; + + let second = ApiKey::resolve_from(None, &missing_external, temp.path())?; + assert!(second.matches(first.0.as_bytes())); + assert!(first.0.starts_with(GENERATED_API_KEY_PREFIX)); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let path = temp.path().join(MANAGED_API_KEY_RELATIVE_PATH); + assert_eq!( + fs::metadata(path.parent().unwrap())?.permissions().mode() & 0o777, + 0o700 + ); + assert_eq!(fs::metadata(path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn validation_enforces_length_and_url_safe_characters() { + assert!(ApiKey::new("a".repeat(32)).is_ok()); + assert!(ApiKey::new("a".repeat(API_KEY_MAX_LEN)).is_ok()); + assert!(ApiKey::new("a".repeat(31)).is_err()); + assert!(ApiKey::new("a".repeat(API_KEY_MAX_LEN + 1)).is_err()); + assert!(ApiKey::new(format!("{}!", "a".repeat(31))).is_err()); + } + + #[test] + fn matches_uses_the_validated_key() -> Result<()> { + let key = ApiKey::new(TEST_KEY)?; + + assert!(key.matches(TEST_KEY.as_bytes())); + assert!(!key.matches(b"wrong-key")); + Ok(()) + } +} diff --git a/src/bin/server.rs b/src/bin/server.rs index 479d038c..0a32d456 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -1,6 +1,7 @@ use std::sync::{Arc, RwLock}; use agentenv::api::{server, ApiImpl}; +use agentenv::api_key::ApiKey; use agentenv::identity::NodeIdentity; use agentenv::image::ImageResolver; use agentenv::observability::{ObservabilityReporter, ObservabilityService}; @@ -79,6 +80,8 @@ async fn main() -> anyhow::Result<()> { agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; + let api_key = ApiKey::resolve(config)?; + let addr = std::env::var("API_ADDR").unwrap_or_else(|_| "0.0.0.0:8000".to_string()); let identity = NodeIdentity::from_config(&config.node_identity); let p2p_transport = agentenv::p2p::transport_from_config(config, &identity).await?; @@ -147,6 +150,7 @@ async fn main() -> anyhow::Result<()> { image_resolver, observability, config.sandbox_proxy.domains.clone(), + api_key, )); let app = server::new(api_impl); let shutdown_orchestrator = Arc::clone(&orchestrator); diff --git a/src/lib.rs b/src/lib.rs index 5968c26e..9223e1bf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,12 @@ pub mod api; +pub mod api_key; pub mod cfg; mod digest; pub mod identity; pub mod image; mod local_store; pub mod logging; +mod managed_secret; pub mod observability; pub mod orchestrator; pub mod overlaybd; diff --git a/src/managed_secret.rs b/src/managed_secret.rs new file mode 100644 index 00000000..cb365a2b --- /dev/null +++ b/src/managed_secret.rs @@ -0,0 +1,273 @@ +#[cfg(unix)] +use std::fs::OpenOptions; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; + +pub(crate) enum CreateOutcome { + Created, + Existing(File), +} + +pub(crate) fn read(path: &Path, max_len: usize) -> Result> { + ensure_supported()?; + let parent = managed_parent(path)?; + let file = match open_secret(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("open managed secret {}", path.display())); + } + }; + validate_directory(parent) + .with_context(|| format!("validate managed secret directory {}", parent.display()))?; + read_file(path, file, max_len).map(Some) +} + +pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result { + let metadata = file + .metadata() + .with_context(|| format!("inspect managed secret {}", path.display()))?; + if !metadata.is_file() { + bail!("managed secret {} must be a regular file", path.display()); + } + + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let mode = metadata.permissions().mode() & 0o7777; + if mode != 0o600 { + bail!( + "managed secret {} must have permissions 0600, found {mode:04o}", + path.display() + ); + } + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + bail!( + "managed secret {} must be owned by uid {expected_uid}, found uid {}", + path.display(), + metadata.uid() + ); + } + } + + if metadata.len() > max_len as u64 { + bail!( + "managed secret {} must be at most {max_len} bytes", + path.display() + ); + } + let read_limit = max_len + .checked_add(1) + .context("managed secret size limit is too large")?; + let mut contents = String::with_capacity(max_len); + Read::by_ref(&mut file) + .take(read_limit as u64) + .read_to_string(&mut contents) + .with_context(|| format!("read managed secret {}", path.display()))?; + if contents.len() > max_len { + bail!( + "managed secret {} must be at most {max_len} bytes", + path.display() + ); + } + Ok(contents) +} + +pub(crate) fn create(path: &Path, contents: &[u8]) -> Result { + ensure_supported()?; + let parent = managed_parent(path)?; + create_directory(parent)?; + validate_directory_identity(parent).with_context(|| { + format!( + "validate managed secret directory ownership {}", + parent.display() + ) + })?; + set_permissions(parent, 0o700)?; + validate_directory(parent) + .with_context(|| format!("validate managed secret directory {}", parent.display()))?; + + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temporary secret in {}", parent.display()))?; + set_permissions(temporary.path(), 0o600)?; + temporary + .write_all(contents) + .with_context(|| format!("write temporary secret in {}", parent.display()))?; + temporary + .as_file() + .sync_all() + .with_context(|| format!("sync temporary secret in {}", parent.display()))?; + + match temporary.persist_noclobber(path) { + Ok(_) => { + File::open(parent) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("sync managed secret directory {}", parent.display()))?; + Ok(CreateOutcome::Created) + } + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { + validate_directory(parent).with_context(|| { + format!("validate managed secret directory {}", parent.display()) + })?; + open_secret(path) + .map(CreateOutcome::Existing) + .with_context(|| format!("open managed secret {}", path.display())) + } + Err(error) => { + Err(error.error).with_context(|| format!("persist managed secret {}", path.display())) + } + } +} + +fn managed_parent(path: &Path) -> Result<&Path> { + let parent = path.parent().context("managed secret path has no parent")?; + if parent.file_name().is_none_or(|name| name != "secrets") { + bail!( + "managed secret parent {} must be a dedicated directory named secrets", + parent.display() + ); + } + Ok(parent) +} + +fn create_directory(path: &Path) -> Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(path) + .with_context(|| format!("create managed secret directory {}", path.display())) +} + +#[cfg(unix)] +fn open_secret(path: &Path) -> io::Result { + use std::os::unix::fs::OpenOptionsExt; + + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) +} + +#[cfg(not(unix))] +fn open_secret(_path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "managed secrets require Unix no-follow file semantics", + )) +} + +#[cfg(unix)] +fn ensure_supported() -> Result<()> { + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_supported() -> Result<()> { + bail!("managed secrets require Unix no-follow file semantics") +} + +fn validate_directory(path: &Path) -> io::Result<()> { + let metadata = validate_directory_identity(path)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = metadata.permissions().mode() & 0o7777; + if mode != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("must have permissions 0700, found {mode:04o}"), + )); + } + } + Ok(()) +} + +fn validate_directory_identity(path: &Path) -> io::Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "must be a directory and not a symbolic link", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "must be owned by uid {expected_uid}, found uid {}", + metadata.uid() + ), + )); + } + } + Ok(metadata) +} + +#[cfg(unix)] +fn set_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} + +#[cfg(all(test, unix))] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + #[test] + fn creation_tightens_an_empty_volume_directory() -> Result<()> { + let root = tempfile::tempdir()?; + let directory = root.path().join("secrets"); + let secret = directory.join("api-key"); + fs::create_dir(&directory)?; + fs::set_permissions(&directory, fs::Permissions::from_mode(0o755))?; + + assert!(read(&secret, 64)?.is_none()); + assert!(matches!( + create(&secret, b"secret"), + Ok(CreateOutcome::Created) + )); + assert_eq!( + fs::metadata(directory)?.permissions().mode() & 0o7777, + 0o700 + ); + Ok(()) + } + + #[test] + fn creation_rejects_a_non_dedicated_parent() -> Result<()> { + let parent = tempfile::tempdir()?; + fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o750))?; + + assert!(create(&parent.path().join("api-key"), b"secret").is_err()); + assert_eq!( + fs::metadata(parent.path())?.permissions().mode() & 0o7777, + 0o750 + ); + Ok(()) + } +} diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index 079362ed..542f145b 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -162,12 +162,12 @@ where // Restore persisted sandboxes from the previous run, keeping the paused // ones (with their state) for the paused-protection reconcile below. let persisted = persister.load_all(&factory).await?; - let managed_seed_must_exist = persisted.iter().any(|metadata| metadata.secure); + let managed_seed_must_exist = persisted_sandboxes_require_managed_seed(&persisted); let access_tokens = tokio::task::spawn_blocking(move || { SandboxAccessTokenGenerator::load_or_create(app_config, managed_seed_must_exist) }) .await - .context("join envd access-token seed loader")??; + .context("join sandbox access-token seed loader")??; let restored_paused: Vec<(SandboxId, Arc)> = persisted .iter() .filter(|metadata| metadata.state == SandboxState::Paused) @@ -741,6 +741,14 @@ where self.access_tokens.matches(sandbox_id, candidate) } + pub fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { + self.access_tokens.generate_traffic(sandbox_id) + } + + pub fn validate_traffic_access_token(&self, sandbox_id: SandboxId, candidate: &str) -> bool { + self.access_tokens.matches_traffic(sandbox_id, candidate) + } + /// Resolves the current proxyability of a sandbox without touching the sandbox mutex. #[tracing::instrument(skip(self), fields(sandbox_id = %sandbox_id))] pub async fn proxy_lookup_for(&self, sandbox_id: &SandboxId) -> Result { @@ -1540,7 +1548,7 @@ where async fn replace_sandbox_network_policy_inner( &self, sandbox_id: SandboxId, - network_policy: SandboxNetworkPolicy, + mut network_policy: SandboxNetworkPolicy, ) -> Result<()> { let metadata = self .store @@ -1553,6 +1561,7 @@ where state: metadata.state, }); } + network_policy.allow_public_traffic = metadata.network_policy.allow_public_traffic; let sandbox = { let sandboxes = self.sandboxes.read().await; @@ -2450,6 +2459,12 @@ where } } +fn persisted_sandboxes_require_managed_seed(persisted: &[SandboxMetadata]) -> bool { + persisted + .iter() + .any(|metadata| metadata.secure || !metadata.network_policy.allow_public_traffic) +} + #[cfg(test)] impl Orchestrator where @@ -2526,6 +2541,19 @@ where Ok(()) } + pub(crate) async fn set_allow_public_traffic_for_test( + &self, + sandbox_id: &SandboxId, + allow_public_traffic: bool, + ) -> Result<()> { + let Some(mut metadata) = self.store.get(sandbox_id).await? else { + return Err(OrchestratorError::SandboxNotFound(*sandbox_id)); + }; + metadata.network_policy.allow_public_traffic = allow_public_traffic; + self.store.update(metadata).await?; + Ok(()) + } + pub(crate) async fn remove_proxy_route_for_test(&self, sandbox_id: &SandboxId) { let _ = self.proxy_routes.write().await.remove(sandbox_id); } diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index af4d24ae..0f89f19f 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -677,6 +677,22 @@ async fn new_loads_persisted_sandboxes_into_store() -> Result<()> { Ok(()) } +#[test] +fn managed_seed_continuity_only_applies_to_token_protected_sandboxes() { + let public_insecure = SandboxMetadata::default(); + assert!(!persisted_sandboxes_require_managed_seed( + std::slice::from_ref(&public_insecure) + )); + + let mut secure = public_insecure.clone(); + secure.secure = true; + assert!(persisted_sandboxes_require_managed_seed(&[secure])); + + let mut private = public_insecure; + private.network_policy.allow_public_traffic = false; + assert!(persisted_sandboxes_require_managed_seed(&[private])); +} + #[tokio::test] async fn new_returns_error_when_loading_persisted_sandboxes_fails() { setup(); @@ -1236,6 +1252,7 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { make_orchestrator_with_factory(MockBackendFactory::with_behavior(Arc::clone(&behavior))) .await; let initial_policy = SandboxNetworkPolicy::new( + false, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new( Some(vec!["8.8.8.8".to_string()]), @@ -1249,11 +1266,17 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { assert_eq!(created.network_policy, initial_policy); let updated_policy = SandboxNetworkPolicy::new( + true, + BaseSandboxNetworkPolicy::Allow, + SandboxNetworkEgressPolicy::new(None, Some(vec!["198.51.100.0/24".to_string()]))?, + ); + let expected_policy = SandboxNetworkPolicy::new( + false, BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new(None, Some(vec!["198.51.100.0/24".to_string()]))?, ); orchestrator - .replace_sandbox_network_policy(created.id, updated_policy.clone()) + .replace_sandbox_network_policy(created.id, updated_policy) .await?; assert_eq!(behavior.update_network_calls(), 1); @@ -1261,7 +1284,7 @@ async fn sandbox_network_policy_is_applied_and_persisted() -> Result<()> { .get_sandbox(&created.id) .await? .expect("sandbox metadata should exist"); - assert_eq!(updated.network_policy, updated_policy); + assert_eq!(updated.network_policy, expected_policy); Ok(()) } diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index 3a620430..d00e5fd9 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -1,6 +1,4 @@ use std::fmt; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Read, Write}; use std::path::Path; use anyhow::{bail, Context, Result}; @@ -10,6 +8,7 @@ use sha2::Sha256; use tracing::{info, warn}; use crate::cfg::AppConfig; +use crate::managed_secret::{self, CreateOutcome}; use crate::types::SandboxId; type HmacSha256 = Hmac; @@ -18,6 +17,7 @@ const MANAGED_SEED_RELATIVE_PATH: &str = "secrets/sandbox-access-token-hash-seed const MANAGED_SEED_BYTES: usize = 32; const SEED_HEX_LEN: usize = MANAGED_SEED_BYTES * 2; const MANAGED_SEED_FILE_MAX_LEN: usize = SEED_HEX_LEN + 1; +const TRAFFIC_ACCESS_TOKEN_PREFIX: &str = "sandbox-traffic"; #[derive(Clone, PartialEq, Eq)] pub struct EnvdAccessToken(String); @@ -58,12 +58,10 @@ impl SandboxAccessTokenGenerator { let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; - if config.sandbox.access_token_hash_seed.is_none() - && config.cluster.scheduler_endpoint.is_some() - { + if config.cluster.scheduler_endpoint.is_some() { warn!( path = %managed_seed_path.display(), - "using a node-local managed envd access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node before enabling cross-node sandbox recovery" + "using a node-local managed sandbox access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node in a clustered deployment" ); } @@ -71,18 +69,37 @@ impl SandboxAccessTokenGenerator { } pub fn generate(&self, subject: SandboxId) -> EnvdAccessToken { + EnvdAccessToken(self.generate_for(subject.to_string().as_bytes())) + } + + pub fn generate_traffic(&self, subject: SandboxId) -> String { + self.generate_for(format!("{TRAFFIC_ACCESS_TOKEN_PREFIX}-{subject}").as_bytes()) + } + + fn generate_for(&self, subject: &[u8]) -> String { let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); - mac.update(subject.to_string().as_bytes()); - EnvdAccessToken(hex::encode(mac.finalize().into_bytes())) + mac.update(subject); + hex::encode(mac.finalize().into_bytes()) } pub fn matches(&self, subject: SandboxId, candidate: &str) -> bool { + self.matches_for(subject.to_string().as_bytes(), candidate) + } + + pub fn matches_traffic(&self, subject: SandboxId, candidate: &str) -> bool { + self.matches_for( + format!("{TRAFFIC_ACCESS_TOKEN_PREFIX}-{subject}").as_bytes(), + candidate, + ) + } + + fn matches_for(&self, subject: &[u8], candidate: &str) -> bool { let mut candidate_bytes = [0_u8; 32]; let decoded = hex::decode_to_slice(candidate, &mut candidate_bytes).is_ok(); let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); - mac.update(subject.to_string().as_bytes()); + mac.update(subject); mac.verify_slice(&candidate_bytes).is_ok() & decoded } } @@ -96,35 +113,13 @@ fn validate_explicit_seed(seed: &str) -> Result<&str> { } fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { - let parent = managed_path - .parent() - .context("managed envd access-token seed path has no parent")?; - match validate_managed_seed_directory(parent) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!("validate managed secret directory {}", parent.display()) - }); - } - } - - match open_managed_seed(managed_path) { - Ok(file) => return read_managed_seed(managed_path, file), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!( - "open managed envd access-token seed {}", - managed_path.display() - ) - }); - } + if let Some(contents) = managed_secret::read(managed_path, MANAGED_SEED_FILE_MAX_LEN)? { + return validate_managed_seed(managed_path, &contents); } if managed_seed_must_exist { bail!( - "managed envd access-token seed {} is missing while persisted secure sandboxes exist; restore the file or configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED", + "managed sandbox access-token seed {} is missing while token-protected persisted sandboxes exist; restore the file or configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED", managed_path.display() ); } @@ -132,80 +127,11 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result io::Result { - let mut options = OpenOptions::new(); - options.read(true); - - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - - options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); - } - - options.open(path) -} - -fn validate_managed_seed_file(path: &Path, file: &File) -> Result { - let metadata = file - .metadata() - .with_context(|| format!("inspect managed envd access-token seed {}", path.display()))?; - if !metadata.is_file() { - bail!( - "managed envd access-token seed {} must be a regular file", - path.display() - ); - } - - #[cfg(unix)] - { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let mode = metadata.permissions().mode() & 0o777; - if mode != 0o600 { - bail!( - "managed envd access-token seed {} must have permissions 0600, found {mode:04o}", - path.display() - ); - } - let expected_uid = nix::unistd::Uid::effective().as_raw(); - if metadata.uid() != expected_uid { - bail!( - "managed envd access-token seed {} must be owned by uid {expected_uid}, found uid {}", - path.display(), - metadata.uid() - ); - } - } - - Ok(metadata) -} - -fn read_managed_seed(path: &Path, mut file: File) -> Result { - let metadata = validate_managed_seed_file(path, &file)?; - - if metadata.len() > MANAGED_SEED_FILE_MAX_LEN as u64 { - bail!( - "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", - path.display() - ); - } - - let mut contents = String::with_capacity(MANAGED_SEED_FILE_MAX_LEN); - Read::by_ref(&mut file) - .take((MANAGED_SEED_FILE_MAX_LEN + 1) as u64) - .read_to_string(&mut contents) - .with_context(|| format!("read managed envd access-token seed {}", path.display()))?; - if contents.len() > MANAGED_SEED_FILE_MAX_LEN { - bail!( - "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", - path.display() - ); - } - let seed = contents.strip_suffix('\n').unwrap_or(&contents); +fn validate_managed_seed(path: &Path, contents: &str) -> Result { + let seed = contents.strip_suffix('\n').unwrap_or(contents); if !is_valid_managed_seed(seed) { bail!( - "managed envd access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", + "managed sandbox access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", path.display() ); } @@ -214,53 +140,21 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { } fn create_managed_seed(path: &Path) -> Result { - let parent = path - .parent() - .context("managed envd access-token seed path has no parent")?; - fs::create_dir_all(parent) - .with_context(|| format!("create managed secret directory {}", parent.display()))?; - validate_managed_seed_directory_identity(parent).with_context(|| { - format!( - "validate managed secret directory ownership {}", - parent.display() - ) - })?; - set_permissions(parent, 0o700)?; - validate_managed_seed_directory(parent) - .with_context(|| format!("validate managed secret directory {}", parent.display()))?; - let mut random = [0_u8; MANAGED_SEED_BYTES]; SysRng .try_fill_bytes(&mut random) - .context("generate managed envd access-token seed")?; + .context("generate managed sandbox access-token seed")?; let seed = hex::encode(random); - let mut temporary = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("create temporary seed file in {}", parent.display()))?; - set_permissions(temporary.path(), 0o600)?; - writeln!(temporary, "{seed}") - .with_context(|| format!("write temporary seed file in {}", parent.display()))?; - temporary - .as_file() - .sync_all() - .with_context(|| format!("sync temporary seed file in {}", parent.display()))?; - - match temporary.persist_noclobber(path) { - Ok(_) => { - fs::File::open(parent) - .and_then(|directory| directory.sync_all()) - .with_context(|| format!("sync managed secret directory {}", parent.display()))?; - info!(path = %path.display(), "generated managed envd access-token seed"); + match managed_secret::create(path, format!("{seed}\n").as_bytes())? { + CreateOutcome::Created => { + info!(path = %path.display(), "generated managed sandbox access-token seed"); Ok(seed) } - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { - let file = open_managed_seed(path).with_context(|| { - format!("open managed envd access-token seed {}", path.display()) - })?; - read_managed_seed(path, file) - } - Err(error) => Err(error.error) - .with_context(|| format!("persist managed envd access-token seed {}", path.display())), + CreateOutcome::Existing(file) => validate_managed_seed( + path, + &managed_secret::read_file(path, file, MANAGED_SEED_FILE_MAX_LEN)?, + ), } } @@ -271,66 +165,6 @@ fn is_valid_managed_seed(seed: &str) -> bool { .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } -fn validate_managed_seed_directory(path: &Path) -> io::Result<()> { - let metadata = validate_managed_seed_directory_identity(path)?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - let mode = metadata.permissions().mode() & 0o777; - if mode != 0o700 { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - format!("must have permissions 0700, found {mode:04o}"), - )); - } - } - - Ok(()) -} - -fn validate_managed_seed_directory_identity(path: &Path) -> io::Result { - let metadata = fs::symlink_metadata(path)?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "must be a directory and not a symbolic link", - )); - } - - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - - let expected_uid = nix::unistd::Uid::effective().as_raw(); - if metadata.uid() != expected_uid { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - format!( - "must be owned by uid {expected_uid}, found uid {}", - metadata.uid() - ), - )); - } - } - - Ok(metadata) -} - -#[cfg(unix)] -fn set_permissions(path: &Path, mode: u32) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { - Ok(()) -} - impl fmt::Debug for SandboxAccessTokenGenerator { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("SandboxAccessTokenGenerator()") @@ -340,32 +174,48 @@ impl fmt::Debug for SandboxAccessTokenGenerator { #[cfg(test)] mod tests { use super::*; + use std::fs; use std::sync::{Arc, Barrier}; use tempfile::TempDir; fn create_private_managed_seed_directory(path: &Path) -> Result<()> { fs::create_dir_all(path)?; - set_permissions(path, 0o700) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + } + Ok(()) } #[test] - fn generates_lowercase_hex_hmac_sha256() { + fn generates_e2b_compatible_access_tokens() { let generator = SandboxAccessTokenGenerator::new("test-seed").unwrap(); let subject = SandboxId::try_from("01936f8e-72f5-7000-8000-000000000001").unwrap(); - let token = generator.generate(subject); + let envd_token = generator.generate(subject); + let traffic_token = generator.generate_traffic(subject); - assert_eq!(token.expose().len(), 64); assert_eq!( - token.expose(), + envd_token.expose(), "4f00f2a93a87c37161ae01c59b6d4f84506668113441277e9f6272dd4bfae1a7" ); - assert!(token.expose().bytes().all(|byte| byte.is_ascii_hexdigit())); - assert_eq!(token.expose(), token.expose().to_ascii_lowercase()); - assert!(generator.matches(subject, token.expose())); + assert_eq!( + traffic_token, + "586547d7c10facb0f4871297fdbfd9d2b4376f4b02b2e1487646c1c87a293bd8" + ); + assert!(envd_token + .expose() + .bytes() + .all(|byte| byte.is_ascii_hexdigit())); + assert!(traffic_token.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert!(generator.matches(subject, envd_token.expose())); + assert!(generator.matches_traffic(subject, &traffic_token)); assert!(!generator.matches(subject, "not-a-token")); assert!(!generator.matches(subject, &"0".repeat(64))); + assert!(!generator.matches_traffic(subject, envd_token.expose())); } #[test] @@ -469,7 +319,7 @@ mod tests { for contents in ["", "invalid\n"] { fs::write(&managed_path, contents)?; - set_permissions(&managed_path, 0o600)?; + set_test_permissions(&managed_path, 0o600)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -486,7 +336,7 @@ mod tests { let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); create_private_managed_seed_directory(managed_path.parent().unwrap())?; fs::write(&managed_path, format!("{}\n", "a".repeat(64)))?; - set_permissions(&managed_path, 0o640)?; + set_test_permissions(&managed_path, 0o640)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -504,14 +354,12 @@ mod tests { let target_path = temp.path().join("seed-target"); create_private_managed_seed_directory(managed_path.parent().unwrap())?; fs::write(&target_path, format!("{}\n", "a".repeat(64)))?; - set_permissions(&target_path, 0o600)?; + set_test_permissions(&target_path, 0o600)?; symlink(&target_path, &managed_path)?; let error = resolve_seed(&managed_path, false).unwrap_err(); - assert!(error - .to_string() - .contains("open managed envd access-token seed")); + assert!(error.to_string().contains("open managed secret")); Ok(()) } @@ -520,9 +368,9 @@ mod tests { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); create_private_managed_seed_directory(managed_path.parent().unwrap())?; - let file = File::create(&managed_path)?; + let file = fs::File::create(&managed_path)?; file.set_len(1024 * 1024)?; - set_permissions(&managed_path, 0o600)?; + set_test_permissions(&managed_path, 0o600)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -530,20 +378,6 @@ mod tests { Ok(()) } - #[cfg(unix)] - #[test] - fn permissive_managed_seed_directory_is_rejected() -> Result<()> { - let temp = TempDir::new()?; - let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); - fs::create_dir_all(managed_path.parent().unwrap())?; - set_permissions(managed_path.parent().unwrap(), 0o770)?; - - let error = resolve_seed(&managed_path, false).unwrap_err(); - - assert!(format!("{error:#}").contains("permissions 0700")); - Ok(()) - } - #[cfg(unix)] #[test] fn managed_seed_directory_symlink_is_rejected() -> Result<()> { @@ -562,16 +396,27 @@ mod tests { } #[test] - fn missing_managed_seed_is_not_recreated_for_secure_state() -> Result<()> { + fn missing_managed_seed_is_not_recreated_for_persisted_state() -> Result<()> { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); let error = resolve_seed(&managed_path, true).unwrap_err(); - assert!(error - .to_string() - .contains("persisted secure sandboxes exist")); + assert!(error.to_string().contains("persisted sandboxes exist")); assert!(!managed_path.exists()); Ok(()) } + + #[cfg(unix)] + fn set_test_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode))?; + Ok(()) + } + + #[cfg(not(unix))] + fn set_test_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) + } } diff --git a/src/sandbox/backend.rs b/src/sandbox/backend.rs index 26702917..575e3a62 100644 --- a/src/sandbox/backend.rs +++ b/src/sandbox/backend.rs @@ -35,6 +35,10 @@ pub trait PausedSandboxState: Any + fmt::Debug + Send + Sync + 'static { /// The orchestrator only carries this value to the image-liveness layer; it /// does not interpret the backend-specific artifact identities inside it. fn runtime_artifacts(&self) -> RuntimeArtifactSet; + /// Effective envd control-plane port persisted with the paused runtime, when available. + fn control_plane_port(&self) -> Option { + None + } } impl dyn PausedSandboxState { diff --git a/src/sandbox/firecracker/sandbox.rs b/src/sandbox/firecracker/sandbox.rs index a3a7c903..52eb2e56 100644 --- a/src/sandbox/firecracker/sandbox.rs +++ b/src/sandbox/firecracker/sandbox.rs @@ -229,6 +229,11 @@ impl FirecrackerPausedState { } impl PausedSandboxState for FirecrackerPausedState { + fn control_plane_port(&self) -> Option { + let port = self.snapshot_config.common.control_plane_port; + (port != 0).then_some(port) + } + fn encode(&self) -> Result { serde_json::to_value(&self.snapshot_config).context("serialize Firecracker paused state") } diff --git a/src/sandbox/network/policy.rs b/src/sandbox/network/policy.rs index 205578de..c58507cf 100644 --- a/src/sandbox/network/policy.rs +++ b/src/sandbox/network/policy.rs @@ -73,15 +73,36 @@ impl SandboxNetworkEgressPolicy { } } -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct SandboxNetworkPolicy { + #[serde(default = "default_allow_public_traffic")] + pub allow_public_traffic: bool, pub base_policy: BaseSandboxNetworkPolicy, pub egress: SandboxNetworkEgressPolicy, } +fn default_allow_public_traffic() -> bool { + true +} + +impl Default for SandboxNetworkPolicy { + fn default() -> Self { + Self { + allow_public_traffic: true, + base_policy: BaseSandboxNetworkPolicy::default(), + egress: SandboxNetworkEgressPolicy::default(), + } + } +} + impl SandboxNetworkPolicy { - pub fn new(base_policy: BaseSandboxNetworkPolicy, egress: SandboxNetworkEgressPolicy) -> Self { + pub fn new( + allow_public_traffic: bool, + base_policy: BaseSandboxNetworkPolicy, + egress: SandboxNetworkEgressPolicy, + ) -> Self { Self { + allow_public_traffic, base_policy, egress, } @@ -329,20 +350,36 @@ mod tests { } #[test] - fn new_sets_base_policy() { + fn new_sets_explicit_policy() { let policy = SandboxNetworkPolicy::new( + false, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new(Some(vec!["8.8.8.8/32".to_string()]), None).unwrap(), ); + assert!(!policy.allow_public_traffic); assert_eq!(policy.base_policy, BaseSandboxNetworkPolicy::Deny); assert!(policy.egress.denied_cidrs.is_empty()); assert!(policy.has_runtime_egress_rules()); } + #[test] + fn missing_ingress_policy_deserializes_as_public() { + let mut value = serde_json::to_value(SandboxNetworkPolicy::default()).unwrap(); + value + .as_object_mut() + .unwrap() + .remove("allow_public_traffic"); + + let policy: SandboxNetworkPolicy = serde_json::from_value(value).unwrap(); + + assert!(policy.allow_public_traffic); + } + #[test] fn build_rules_keeps_allow_before_deny() { let policy = SandboxNetworkPolicy { + allow_public_traffic: true, base_policy: BaseSandboxNetworkPolicy::Deny, egress: SandboxNetworkEgressPolicy { allowed_cidrs: vec!["8.8.8.8/32".to_string()], @@ -371,6 +408,7 @@ mod tests { #[test] fn build_policy_replacement_flushes_before_installing_rules() { let policy = SandboxNetworkPolicy { + allow_public_traffic: true, base_policy: BaseSandboxNetworkPolicy::Deny, egress: SandboxNetworkEgressPolicy { allowed_cidrs: vec!["8.8.8.8/32".to_string()], diff --git a/storage/overlaybd/src/lsmt/file/helper.rs b/storage/overlaybd/src/lsmt/file/helper.rs index 2d275c52..e61eabf2 100644 --- a/storage/overlaybd/src/lsmt/file/helper.rs +++ b/storage/overlaybd/src/lsmt/file/helper.rs @@ -9,7 +9,7 @@ use std::io::{self, ErrorKind}; use std::mem::size_of; use std::os::unix::fs::{FileExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex as StdMutex, OnceLock, Weak}; +use std::sync::{Arc, OnceLock, Weak}; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::{Mutex, OwnedMutexGuard}; use uuid::Uuid; @@ -472,157 +472,50 @@ pub(super) fn decode_premerged_index_artifact( ))) } -/// Growth in artifact bytes that amortizes one full-dir prune scan. -const PREMERGED_INDEX_PRUNE_SCAN_FRACTION: u64 = 8; - -/// Per-cache-dir growth accounting and scan serialization. -pub(super) struct PremergedIndexPruneState { - account: StdMutex, -} - -struct PremergedIndexScanAccount { - bytes_since_scan: u64, - scan_in_progress: bool, -} - -pub(super) async fn premerged_index_prune_state(cache_dir: &Path) -> Arc { - static STATES: OnceLock>>> = - OnceLock::new(); - STATES - .get_or_init(|| Mutex::new(HashMap::new())) - .lock() - .await - .entry(cache_dir.to_path_buf()) - .or_insert_with(|| { - Arc::new(PremergedIndexPruneState { - account: StdMutex::new(PremergedIndexScanAccount { - bytes_since_scan: 0, - scan_in_progress: false, - }), - }) - }) - .clone() -} - -impl PremergedIndexPruneState { - /// Charge `written_bytes` and elect the single caller that runs the next scan. - pub(super) fn elect_scan(&self, written_bytes: u64, max_dir_bytes: u64) -> bool { - let threshold = (max_dir_bytes / PREMERGED_INDEX_PRUNE_SCAN_FRACTION).max(1); - let mut account = self.account.lock().unwrap_or_else(|err| err.into_inner()); - account.bytes_since_scan = account.bytes_since_scan.saturating_add(written_bytes); - if account.bytes_since_scan >= threshold && !account.scan_in_progress { - account.bytes_since_scan = 0; - account.scan_in_progress = true; - return true; - } - false - } - - /// End the scan; returns true when growth charged while it ran crosses - /// the trigger and a follow-up scan should run. - pub(super) fn scan_finished(&self, max_dir_bytes: u64) -> bool { - let threshold = (max_dir_bytes / PREMERGED_INDEX_PRUNE_SCAN_FRACTION).max(1); - let mut account = self.account.lock().unwrap_or_else(|err| err.into_inner()); - account.scan_in_progress = false; - if account.bytes_since_scan >= threshold { - account.bytes_since_scan = 0; - account.scan_in_progress = true; - return true; - } - false - } - - /// Release a failed scan; concurrent charges are kept so the next write - /// re-elects. - pub(super) fn scan_aborted(&self) { - let mut account = self.account.lock().unwrap_or_else(|err| err.into_inner()); - account.scan_in_progress = false; - } -} - -/// One blocking task for the whole scan: async `tokio::fs` would cost a -/// blocking-pool round-trip per directory entry. async fn prune_premerged_index_dir(dir: &Path, max_dir_bytes: u64) -> Result<()> { - let dir = dir.to_path_buf(); - tokio::task::spawn_blocking(move || prune_premerged_index_dir_blocking(&dir, max_dir_bytes)) - .await - .context("join premerged index cache prune task")? -} - -struct PremergedArtifactEntry { - path: PathBuf, - len: u64, - modified: SystemTime, -} - -/// Delete oldest premerged index artifacts until `dir` fits `max_dir_bytes`. -fn prune_premerged_index_dir_blocking(dir: &Path, max_dir_bytes: u64) -> Result<()> { let mut entries = Vec::new(); let mut total = 0u64; - for entry in std::fs::read_dir(dir) - .with_context(|| format!("read premerged index cache dir {}", dir.display()))? - { - let entry = entry?; + let mut reader = tokio::fs::read_dir(dir).await?; + + while let Some(entry) = reader.next_entry().await? { let path = entry.path(); if path.extension().and_then(|v| v.to_str()) != Some(PREMERGED_INDEX_EXT) { continue; } - let metadata = match entry.metadata() { - Ok(metadata) => metadata, - // Entries deleted concurrently with the scan only shrink it. - Err(err) if err.kind() == ErrorKind::NotFound => continue, - Err(err) => { - return Err(err).with_context(|| format!("stat {}", path.display())); - } - }; + let metadata = entry.metadata().await?; if !metadata.is_file() { continue; } let len = metadata.len(); let modified = metadata.modified().unwrap_or(UNIX_EPOCH); total = total.saturating_add(len); - entries.push(PremergedArtifactEntry { - path, - len, - modified, - }); + entries.push((path, len, modified)); } if total <= max_dir_bytes { return Ok(()); } - entries.sort_by_key(|entry| entry.modified); - for entry in entries { + entries.sort_by_key(|(_, _, modified): &(PathBuf, u64, SystemTime)| *modified); + for (path, len, _) in entries { if total <= max_dir_bytes { break; } - match std::fs::remove_file(&entry.path) { - Ok(()) => total = total.saturating_sub(entry.len), - // Already removed by someone else since the scan: those bytes - // left the dir too, so count them as freed. - Err(err) if err.kind() == ErrorKind::NotFound => { - total = total.saturating_sub(entry.len); - } + match tokio::fs::remove_file(&path).await { + Ok(()) => total = total.saturating_sub(len), Err(err) => { - tracing::warn!( - ?err, - path = %entry.path.display(), - "remove premerged index artifact failed" - ) + tracing::warn!(?err, path = %path.display(), "remove premerged index artifact failed") } } } Ok(()) } -/// Write the artifact atomically (tmp file + rename) and return its size in -/// bytes so callers can account cache growth. async fn write_premerged_index_artifact( cache_dir: &Path, key: &PremergedIndexCacheKey, index: &ReadOnlyIndex, -) -> Result { +) -> Result<()> { let dir = cache_dir.join(PREMERGED_INDEX_DIR); tokio::fs::create_dir_all(&dir) .await @@ -657,7 +550,7 @@ async fn write_premerged_index_artifact( }); } - Ok(artifact.len() as u64) + Ok(()) } pub(super) async fn try_read_premerged_index_artifact( @@ -689,34 +582,6 @@ pub(super) async fn try_read_premerged_index_artifact( } } -/// Charge `written` artifact bytes for `cache_dir` and run prune scans -/// while the growth trigger keeps electing. -async fn prune_premerged_index_cache(cache_dir: &Path, written: u64, max_dir_bytes: u64) { - let state = premerged_index_prune_state(cache_dir).await; - if !state.elect_scan(written, max_dir_bytes) { - return; - } - let dir = cache_dir.join(PREMERGED_INDEX_DIR); - loop { - match prune_premerged_index_dir(&dir, max_dir_bytes).await { - Ok(()) => { - if !state.scan_finished(max_dir_bytes) { - return; - } - } - Err(err) => { - tracing::warn!( - ?err, - path = %dir.display(), - "prune premerged index cache dir failed" - ); - state.scan_aborted(); - return; - } - } - } -} - pub(super) fn spawn_premerged_index_artifact_write( cache_dir: PathBuf, key: PremergedIndexCacheKey, @@ -734,15 +599,18 @@ pub(super) fn spawn_premerged_index_artifact_write( "write premerged index artifact failed" ); } - // Release the merged index and the digest lock before the prune tail: - // the index can be hundreds of MB and the prune scan may pin it for - // the whole scan, while the held lock would block the next writer for - // the same digest. - drop(merged); + let key_digest = key.digest_hex.clone(); drop(guard); - release_premerged_index_lock(&key.digest_hex, &lock).await; - if let Ok(written) = write_result { - prune_premerged_index_cache(&cache_dir, written, max_dir_bytes).await; + release_premerged_index_lock(&key_digest, &lock).await; + if write_result.is_ok() { + let dir = cache_dir.join(PREMERGED_INDEX_DIR); + if let Err(err) = prune_premerged_index_dir(&dir, max_dir_bytes).await { + tracing::warn!( + ?err, + path = %dir.display(), + "prune premerged index cache dir failed" + ); + } } }); } diff --git a/storage/overlaybd/src/lsmt/file/tests.rs b/storage/overlaybd/src/lsmt/file/tests.rs index ea93c694..907ccd0f 100644 --- a/storage/overlaybd/src/lsmt/file/tests.rs +++ b/storage/overlaybd/src/lsmt/file/tests.rs @@ -2536,60 +2536,6 @@ async fn test_premerged_index_lock_map_drops_idle_entries() { release_premerged_index_lock(&stale_key, &replacement).await; } -#[tokio::test] -async fn test_premerged_index_prune_state_is_per_dir_and_elects_one_scan() { - let temp_dir = TempDir::new().unwrap(); - let cache_a = temp_dir.path().join("cache-a"); - let cache_b = temp_dir.path().join("cache-b"); - - let a = premerged_index_prune_state(&cache_a).await; - assert!(Arc::ptr_eq( - &a, - &premerged_index_prune_state(&cache_a).await - )); - let b = premerged_index_prune_state(&cache_b).await; - assert!(!Arc::ptr_eq(&a, &b)); - - // Budget 64 triggers one scan per 8 new bytes (`max_dir_bytes` / 8). - let budget = 64; - assert!(!a.elect_scan(7, budget)); - // Growth charged to A leaves B below B's own trigger. - assert!(!b.elect_scan(7, budget)); - // The winner's charge resets: the next scan needs a fresh trigger of - // post-election growth. - assert!(a.elect_scan(1, budget)); - - // Crossings while a scan runs lose the election but keep their charge: - // when they cross the trigger, scan_finished chains one follow-up scan - // instead of waiting for new writes. - assert!(!a.elect_scan(8, budget)); - assert!(a.scan_finished(budget)); - assert!(!a.elect_scan(1, budget)); - // Below the trigger: the chain ends and the gate is released. - assert!(!a.scan_finished(budget)); - assert!(a.elect_scan(8, budget)); - // No growth during the scan: no follow-up. - assert!(!a.scan_finished(budget)); - - // A failed scan releases the gate but keeps concurrent charges. - assert!(a.elect_scan(8, budget)); - assert!(!a.elect_scan(7, budget)); - a.scan_aborted(); - assert!(a.elect_scan(1, budget)); - assert!(!a.scan_finished(budget)); - - // A sub-fraction budget floors the trigger at one byte instead of zero - // (which would elect a scan per write). - assert!(a.elect_scan(1, 0)); - assert!(!a.scan_finished(0)); - - // Saturating accounting: a u64::MAX charge on a near-trigger counter - // must cross the trigger, not wrap 7 + MAX back below it. - assert!(!a.elect_scan(7, budget)); - assert!(a.elect_scan(u64::MAX, budget)); - assert!(!a.scan_finished(budget)); -} - fn premerged_artifact_count(cache_dir: &std::path::Path) -> usize { let dir = cache_dir.join(PREMERGED_INDEX_DIR); match std::fs::read_dir(dir) { diff --git a/storage/ublk-daemon/Cargo.toml b/storage/ublk-daemon/Cargo.toml index 61b53b86..69736265 100644 --- a/storage/ublk-daemon/Cargo.toml +++ b/storage/ublk-daemon/Cargo.toml @@ -24,7 +24,6 @@ tokio = { version = "1.44.2", features = ["full"] } tracing = "0.1.41" tracing-log = "0.2.0" reqwest = { version = "0.13", default-features = false, features = ["rustls", "json"] } -tikv-jemallocator = { version = "0.6", features = ["background_threads"] } [dev-dependencies] tempfile = "3" diff --git a/storage/ublk-daemon/src/main.rs b/storage/ublk-daemon/src/main.rs index 260b7070..4a945026 100644 --- a/storage/ublk-daemon/src/main.rs +++ b/storage/ublk-daemon/src/main.rs @@ -15,10 +15,6 @@ use uvm_ublk_daemon::{server::UblkDaemonServer, ResizeToolSpec}; mod metrics_server; -// Mirrors src/bin/server.rs. -#[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - #[derive(Debug, Parser)] #[command( name = "uvm-ublk-daemon", diff --git a/tests/integration/fc.rs b/tests/integration/fc.rs index 80cccc2c..981c6136 100644 --- a/tests/integration/fc.rs +++ b/tests/integration/fc.rs @@ -537,6 +537,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { common::setup().await; let mut sandbox_config = common::default_sandbox_config()?; sandbox_config.common.network_policy = Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new(Some(vec!["8.8.8.8".to_string()]), None)?, )); @@ -549,6 +550,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { sandbox .update_network_policy(Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Deny, SandboxNetworkEgressPolicy::new(Some(vec!["1.1.1.1".to_string()]), None)?, ))) @@ -559,6 +561,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { sandbox .update_network_policy(Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new( Some(vec!["8.8.8.8".to_string()]), @@ -578,6 +581,7 @@ async fn microvm_network_policy_controls_egress() -> Result<()> { sandbox .update_network_policy(Some(SandboxNetworkPolicy::new( + true, BaseSandboxNetworkPolicy::Allow, SandboxNetworkEgressPolicy::new(Some(vec!["10.0.0.0/8".to_string()]), None)?, )))