From 92bcbad67f415039c295f4c099f1aedd5df1e833 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 09:09:55 +0000 Subject: [PATCH 1/6] feat: add single-tenant API key authentication --- deploy/docker-compose.yml | 8 +- deploy/k8s/base/agentenv-daemonset.yaml | 7 +- deploy/k8s/base/gateway-deployment.yaml | 5 + deploy/k8s/base/kustomization.yaml | 5 + deploy/k8s/run.sh | 41 +++ docs/src/SUMMARY.md | 1 + docs/src/concepts/sandboxes.md | 2 +- docs/src/configuration/authentication.md | 121 +++++++ docs/src/configuration/env-vars.md | 13 +- docs/src/deployment/docker-compose.md | 36 +- docs/src/deployment/docker.md | 12 + docs/src/deployment/kubernetes.md | 13 + docs/src/deployment/manual-compile.md | 13 +- docs/src/deployment/pvm.md | 5 +- docs/src/deployment/static-multi-node.md | 39 +- docs/src/getting-started/aenv-cli.md | 2 +- docs/src/getting-started/quickstart.md | 17 +- docs/src/integration/e2b.md | 10 +- docs/src/internals/architecture.md | 3 +- docs/src/internals/services.md | 4 +- docs/src/security/secure-sandboxes.md | 14 +- scripts/install.sh | 4 +- scripts/tests/e2e/lib/helpers.sh | 11 +- scripts/tests/e2e/lib/runtime.sh | 12 +- scripts/tests/e2e/lib/server.sh | 3 +- scripts/tests/e2e/suites/08_auth.sh | 19 + scripts/tests/e2e/suites/09_e2b_compat.sh | 3 +- .../tests/e2e/suites/14_code_interpreter.sh | 2 +- services/README.md | 9 +- services/gateway/cmd/main.go | 48 +++ services/gateway/cmd/main_test.go | 77 ++++ services/gateway/internal/server.go | 105 +++++- services/gateway/internal/server_test.go | 178 ++++++++- src/api/impls/auth.rs | 179 +++++++-- src/api/impls/mod.rs | 5 +- src/api/impls/sandbox.rs | 3 + src/api/proxy.rs | 340 ++++++++++++++++-- src/api/server.rs | 8 +- src/api_key.rs | 195 ++++++++++ src/bin/server.rs | 3 + src/lib.rs | 1 + src/sandbox/backend.rs | 4 + src/sandbox/firecracker/sandbox.rs | 4 + 43 files changed, 1454 insertions(+), 130 deletions(-) create mode 100644 docs/src/configuration/authentication.md create mode 100644 services/gateway/cmd/main_test.go create mode 100644 src/api_key.rs diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index bca0e3c0..87cff923 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -16,8 +16,9 @@ x-agentenv-base: &agentenv-base - /dev:/dev - ${CONFIG_PATH:-../config/default.toml}:/workspace/config/default.toml:ro # Runtime assets are baked into the image by `server --setup-only`; compose - # persists only committed snapshots across container restarts. + # persists committed snapshots and the deployment API key across restarts. - agentenv-snapshot-store:/workspace/env/snapshot-store + - agentenv-auth:/workspace/env/secrets devices: - /dev/kvm:/dev/kvm privileged: true @@ -69,7 +70,9 @@ services: depends_on: scheduler: condition: service_healthy - volumes: *control-plane-config-volume + volumes: + - ./docker/config/default.json:/config/default.json:ro + - agentenv-auth:/run/secrets:ro environment: GATEWAY_HTTP_LISTEN_ADDR: :8080 GATEWAY_SCHEDULER_ADDR: scheduler:9090 @@ -102,4 +105,5 @@ services: - "8002:8000" volumes: + agentenv-auth: agentenv-snapshot-store: diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index 2f9c43f4..470b2a9c 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -21,6 +21,11 @@ spec: image: agentenv-runtime:latest imagePullPolicy: IfNotPresent env: + - name: AENV_API_KEY + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_API_KEY - name: AENV_CONFIG_PATH value: /workspace/config/agentenv.toml - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED @@ -76,7 +81,7 @@ spec: - | echo "preStop: waiting for sandboxes to drain..." while true; do - count=$(curl -sf -H 'X-API-Key: preStop' http://localhost:8000/sandboxes | jq 'length') || count="" + count=$(curl -sf -H "X-API-Key: ${AENV_API_KEY}" http://localhost:8000/sandboxes | jq 'length') || count="" if [ -z "$count" ]; then echo "preStop: failed to query sandbox count, retrying..." sleep 3 diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 81f222c8..08536fc0 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -27,6 +27,11 @@ spec: - name: http containerPort: 8080 env: + - name: AENV_API_KEY + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_API_KEY - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 797a3bb7..556f0d70 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -30,6 +30,11 @@ configMapGenerator: literals: - SANDBOX_PROXY_DOMAINS= +secretGenerator: + - name: agentenv-auth + literals: + - AENV_API_KEY= + images: - name: agentenv-gateway newName: agentenv-gateway diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index d0322c01..4d39680b 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -10,6 +10,7 @@ MODE="$1" shift KUBECTL_BIN="${KUBECTL:-kubectl}" OVERLAY_NAME="${K8S_OVERLAY:-default}" +NAMESPACE="${K8S_NAMESPACE:-agentenv-system}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" @@ -28,6 +29,43 @@ sed_in_place() { cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" +if [[ "${MODE}" != "delete" ]]; then + API_KEY_VALUE="" + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + API_KEY_VALUE="${AENV_API_KEY}" + elif [[ "${MODE}" == "apply" ]]; then + encoded_key="" + if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then + echo "failed to check namespace ${NAMESPACE}" >&2 + exit 1 + fi + if [[ -n "${namespace_name}" ]]; then + if ! encoded_key="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o jsonpath='{.data.AENV_API_KEY}')"; then + echo "failed to read existing Secret ${NAMESPACE}/agentenv-auth" >&2 + exit 1 + fi + fi + if [[ -n "${encoded_key}" ]]; then + if ! API_KEY_VALUE="$(printf '%s' "${encoded_key}" | base64 -d)"; then + echo "failed to decode the existing agentenv-auth Secret" >&2 + exit 1 + fi + fi + fi + + if [[ -z "${API_KEY_VALUE}" ]]; then + API_KEY_VALUE="e2b_$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" + fi + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then + echo "AENV_API_KEY must contain at least 32 URL-safe characters" >&2 + exit 1 + fi + + sed_in_place \ + "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ + "${TEMP_DIR}/k8s/base/kustomization.yaml" +fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then ESCAPED_SANDBOX_PROXY_DOMAINS="${SANDBOX_PROXY_DOMAINS//\\/\\\\}" @@ -65,6 +103,9 @@ case "${MODE}" in ;; apply) "${KUBECTL_BIN}" apply -k "${OVERLAY_PATH}" "$@" + echo "AgentENV API key stored in Secret ${NAMESPACE}/agentenv-auth." >&2 + echo "Read it with:" >&2 + echo " ${KUBECTL_BIN} -n ${NAMESPACE} get secret agentenv-auth -o go-template='{{index .data \"AENV_API_KEY\" | base64decode}}{{\"\\n\"}}'" >&2 ;; delete) "${KUBECTL_BIN}" delete --ignore-not-found -k "${OVERLAY_PATH}" "$@" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index f1d8d120..74795cb2 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -18,6 +18,7 @@ # Configuration +- [Authentication](./configuration/authentication.md) - [Configuration Reference](./configuration/reference.md) - [Environment Variables](./configuration/env-vars.md) diff --git a/docs/src/concepts/sandboxes.md b/docs/src/concepts/sandboxes.md index ab6be6ea..31f91e5b 100644 --- a/docs/src/concepts/sandboxes.md +++ b/docs/src/concepts/sandboxes.md @@ -59,7 +59,7 @@ aenv start --cold ubuntu:24.04 The cold-start API accepts an optional `diskSizeMB` field to set the root filesystem's virtual size in MiB. Explicit values must be at least 1024 MiB and divisible by 1024 because the current resize tool operates at 1 GiB granularity. Growth is allowed by default; shrinking below the source image size requires `ublk.overlaybd.allow_shrink = true`. If omitted, the image's built-in virtual size is used. Resizing applies only when creating a fresh writable root filesystem, not to read-only images, images with an existing upper, or snapshot resume. Sandbox responses also report disk size as `diskSizeMB`. -Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port only; it does not add authentication to application ports. Each fork derives a distinct envd token from the child sandbox ID. +Use `aenv start --secure` with either warm or cold starts to require an envd access token for command and file operations. The CLI obtains and sends the token automatically. Secure mode protects the envd control port, while application proxy requests use the independent sandbox-scoped traffic token. Each fork derives distinct envd and traffic credentials from the child sandbox ID. --- diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md new file mode 100644 index 00000000..b57dac1e --- /dev/null +++ b/docs/src/configuration/authentication.md @@ -0,0 +1,121 @@ +# Authentication + +AgentENV uses one shared API key for a single-tenant deployment. The gateway +and every runtime node in a cluster must resolve the same key. + +Clients authenticate API requests with: + +```text +X-API-Key: +``` + +`Authorization`, `X-Admin-Token`, and `X-Team-ID` do not authenticate +AgentENV. The `Authorization` header is left unchanged when a request is +proxied into a sandbox, so applications inside a sandbox can use it normally. +`GET /health` is public for load balancer and container health checks. + +E2B SDK users set `E2B_API_KEY` to the same value. Sandbox create responses +include an independent `trafficAccessToken`; send it as +`e2b-traffic-access-token` on application proxy requests. The token is scoped to +the sandbox and is not accepted for control-plane API calls. + +For secure sandboxes, `envdAccessToken` is a separate credential for envd +control traffic and must be sent as `X-Access-Token` only when targeting the +envd control-plane port. It is absent for insecure sandboxes. + +## Key Resolution + +On normal startup, a runtime node uses the first available source: + +1. `AENV_API_KEY` +2. `/run/secrets/api-key` +3. `$AENV_HOME/secrets/api-key` + +If neither an environment value nor an external secret exists, the server +generates a 256-bit key and atomically stores it in the managed path with +`0600` permissions. It reuses that key on later starts. Dependency and host +setup modes do not create a key. + +The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a +key because every gateway and runtime node in a cluster must share one. + +## Installation Methods + +For a native installation, start the service once and read the managed key: + +```bash +sudo cat /var/lib/aenv/secrets/api-key +``` + +When upgrading an installation that already has `AENV_API_KEY` in +`/etc/default/aenv`, the installer preserves that entry and the server keeps +using it. Fresh installations leave key creation to the server. + +For a single Docker container, no auth volume is required. The server creates +the key in its writable container layer: + +```bash +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Removing the container removes this generated key. Supply an explicit key or +mount a secret at `/run/secrets/api-key` when it must remain stable across +container replacements. + +The checked-in Compose deployment mounts one named volume read-write on both +runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node +startup is safe: atomic creation makes both nodes converge on the same key. +Read it with: + +```bash +docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ + cat /workspace/env/secrets/api-key +``` + +`docker compose down` preserves the key. `docker compose down -v` removes the +auth volume, so the next startup generates a new key. + +`make k8s-apply` creates `Secret/agentenv-auth` on the first apply and reuses +the existing key on later applies. Read it with: + +```bash +kubectl -n agentenv-system get secret agentenv-auth \ + -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' +``` + +For a single-node manual build, start the server and read +`$AENV_HOME/secrets/api-key`. To provide your own key instead, export it before +startup: + +```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +make start-server +``` + +Custom keys must contain at least 32 URL-safe characters. In a multi-node +deployment, use exactly the same value for the gateway and every runtime node. +The generated keys use `e2b_` followed by hexadecimal characters so they pass +the E2B SDK default API-key validation. Use that format for custom keys when +you need E2B SDK compatibility. + +Docker Compose secrets can supply a pre-existing key without another AgentENV +configuration variable. In an override file, define a file-backed secret and +mount it with `target: api-key` on the gateway and every runtime node. Compose +then exposes the standard `/run/secrets/api-key` path. Compose secret sources +must already exist, so the named-volume setup remains the zero-configuration +default that allows Rust to generate the key during startup. + +## Transport Security + +API key authentication does not encrypt HTTP traffic. Do not send the key over +an untrusted plaintext network. Keep AgentENV on loopback or a trusted private +network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. + +## Rotation + +Set a new `AENV_API_KEY` on the gateway and every runtime node, or replace the +shared secret file, then restart them. Existing clients must switch to the new +value. Previously issued +`trafficAccessToken` values stop working when the key changes. +`envdAccessToken` values are unaffected and rotate only when the optional envd +seed changes. diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index 8581458d..fc70ccaa 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -12,6 +12,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | Variable | Default | Description | |----------|---------|-------------| +| `AENV_API_KEY` | generated under `$AENV_HOME/secrets/api-key` | Optional API-key override. Runtime nodes also check `/run/secrets/api-key` before creating a managed key. Use one shared value or secret in multi-node deployments. | | `API_ADDR` | `0.0.0.0:8000` | Address and port the API server listens on | | `AENV_CONFIG_PATH` | `config/default.toml` | Path to the TOML configuration file | | `AENV_LOG_FORMAT` | `compact` | Server log output format: `compact`, `pretty`, or `json` | @@ -46,8 +47,7 @@ These variables configure the E2B SDK and CLI to point at an AgentENV server. Va |----------|-------------| | `E2B_API_URL` | AgentENV server API base URL | | `E2B_SANDBOX_URL` | Sandbox proxy URL (for WebSocket and process interaction) | -| `E2B_API_KEY` | API key for authentication | -| `E2B_ACCESS_TOKEN` | Access token (used by `e2b template` commands) | +| `E2B_API_KEY` | Set to the deployment's `AENV_API_KEY` | ### Values by Deployment Mode @@ -56,8 +56,7 @@ These variables configure the E2B SDK and CLI to point at an AgentENV server. Va ```bash export E2B_API_URL=http://127.0.0.1:8000 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` **Docker Compose / Kubernetes (multi-node)**: @@ -65,15 +64,14 @@ export E2B_ACCESS_TOKEN=dummy ```bash export E2B_API_URL=http://127.0.0.1:8080 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` > In both modes, sandbox data-plane requests can use routing headers with > `E2B_SANDBOX_URL=${E2B_API_URL}`. The explicit `/proxy` prefix > (`${E2B_API_URL}/proxy`) is still accepted for back-compat. -> For local development, any non-empty value works for `E2B_API_KEY` and `E2B_ACCESS_TOKEN` because the server only checks that the auth header is present. +See [Authentication](./authentication.md) for key generation and storage. ## Gateway and Scheduler @@ -88,6 +86,7 @@ These variables apply to both the gateway and scheduler processes. | Variable | Default | Description | |----------|---------|-------------| +| `AENV_API_KEY` | unset | Shared single-tenant API key. The gateway uses the environment value when set, otherwise it reads `/run/secrets/api-key`. | | `GATEWAY_HTTP_LISTEN_ADDR` | `:8080` | HTTP listen address | | `GATEWAY_METRICS_LISTEN_ADDR` | `:9102` | Prometheus metrics listen address | | `GATEWAY_SCHEDULER_ADDR` | `127.0.0.1:9090` | Scheduler gRPC address for routing and node lookup | diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index e39636a0..0d03444f 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -44,6 +44,11 @@ sudo bash scripts/docker-setup.sh make deploy-up ``` +On first startup, the runtime nodes atomically generate one API key in the +shared `agentenv-auth` volume. The gateway mounts that volume read-only at +`/run/secrets`, so all three services use the same key. Normal +`make deploy-down` calls preserve the volume and key. + To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: @@ -62,8 +67,10 @@ usually through wildcard DNS for `*.sandbox.example.com`. # Health check via gateway curl http://127.0.0.1:8080/health -# Cluster node snapshots via gateway -curl http://127.0.0.1:8080/nodes +# Authenticated cluster node snapshots via gateway +export AENV_API_KEY="$(docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ + cat /workspace/env/secrets/api-key)" +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes # Direct health check on a backend node curl http://127.0.0.1:8001/health @@ -77,6 +84,31 @@ make deploy-logs # Stream logs from all services make deploy-down # Tear down the cluster ``` +Removing Compose volumes with `docker compose down -v` also removes the API +key. The next startup generates a new key and existing clients must be updated. + +To provide an existing key through Docker Compose secrets, add a file-backed +secret in an override file and mount it with `target: api-key` on the gateway +and both runtime nodes. AgentENV automatically reads `/run/secrets/api-key`; +no file-path environment variable is needed. + +```yaml +services: + gateway: + secrets: [api-key] + agentenv-a: + secrets: [api-key] + agentenv-b: + secrets: [api-key] + +secrets: + api-key: + file: ./api-key +``` + +The secret name is also its default target filename, so this mounts the key at +`/run/secrets/api-key` in each service. + ## Configuration Container deployments use `deploy/docker/config/default.json`. Scheduler and backend node endpoints are configured for the Docker network. diff --git a/docs/src/deployment/docker.md b/docs/src/deployment/docker.md index 84d01578..b7c5ffbf 100644 --- a/docs/src/deployment/docker.md +++ b/docs/src/deployment/docker.md @@ -44,6 +44,7 @@ docker build \ ```bash docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -p 8000:8000 \ ghcr.io/kvcache-ai/aenv-server:latest # or aenv:latest if built from source @@ -51,6 +52,17 @@ docker run --rm -it \ The `--privileged` flag is required for Firecracker's network namespace operations (veth pairs, iptables). The server auto-downloads runtime assets on first start and is accessible at `http://127.0.0.1:8000` once ready. +On normal startup, the server generates the API key inside the container at +`/workspace/env/secrets/api-key`. Read it while the container is running with: + +```bash +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Removing the container also removes this generated key. Supply an explicit +`AENV_API_KEY` or a secret at `/run/secrets/api-key` when the key must remain +stable across container replacements. + ## Verify ```bash diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 4e72591a..12d0097d 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -60,6 +60,19 @@ make k8s-render make k8s-apply ``` +`make k8s-apply` generates a 256-bit API key on the first deployment and +stores it in `Secret/agentenv-auth`. Later applies reuse that key. Read it +locally when configuring clients: + +```bash +kubectl -n agentenv-system get secret agentenv-auth \ + -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' +``` + +Set `AENV_API_KEY` when applying to supply your own key instead. A standalone +`make k8s-render` uses a temporary generated value because it does not modify +or read cluster state. + To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: diff --git a/docs/src/deployment/manual-compile.md b/docs/src/deployment/manual-compile.md index 1c95d59c..701bf21c 100644 --- a/docs/src/deployment/manual-compile.md +++ b/docs/src/deployment/manual-compile.md @@ -33,6 +33,9 @@ make release ## Start the Server +Start the server. On first normal startup it generates an API key under +`$AENV_HOME/secrets/api-key` and reuses it on later starts: + ```bash # Debug build API_ADDR=0.0.0.0:8000 make start-server @@ -41,14 +44,22 @@ API_ADDR=0.0.0.0:8000 make start-server API_ADDR=0.0.0.0:8000 make start-server-release ``` -The server auto-downloads runtime assets (Firecracker binary, kernel, rootfs) on first start. Once ready, it listens at `http://127.0.0.1:8000`. +The server auto-downloads runtime assets (Firecracker binary, kernel, rootfs) on first start. Once ready, it listens at `http://127.0.0.1:8000`. Read the generated key before making authenticated requests: + +```bash +export AENV_API_KEY="$(cat "${AENV_HOME_PATH:-/var/lib/aenv}/secrets/api-key")" +``` ## Verify ```bash curl http://127.0.0.1:8000/health +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8000/sandboxes ``` +HTTP does not protect the key in transit. Use a trusted network, VPN, or +TLS-terminating reverse proxy for remote clients. + ## Configuration The server reads `config/default.toml` by default. Override with: diff --git a/docs/src/deployment/pvm.md b/docs/src/deployment/pvm.md index 47781d78..1b896a65 100644 --- a/docs/src/deployment/pvm.md +++ b/docs/src/deployment/pvm.md @@ -197,7 +197,7 @@ Use the dedicated PVM image: ```bash docker pull ghcr.io/kvcache-ai/aenv-server:latest-pvm -docker run --rm -it \ +docker run --rm -it --name aenv-server \ --device /dev/kvm \ --privileged \ -v /dev:/dev \ @@ -218,6 +218,9 @@ cargo run --bin server -- --setup-only make start-server ``` +The server generates and persists the API key under +`$AENV_HOME/secrets/api-key` on its first normal startup. + You can also set the mode in the TOML configuration: ```toml diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 80533fff..875d95b2 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -21,9 +21,9 @@ AgentENV runtime nodes: | Runtime node A | `10.0.0.21:8000` | Runs Firecracker sandboxes as `node-a` | | Runtime node B | `10.0.0.22:8000` | Runs Firecracker sandboxes as `node-b` | -Use private addresses or an otherwise trusted network. AgentENV does not -currently provide an authentication boundary suitable for exposing these -services directly to the public Internet. +AgentENV authenticates HTTP requests but does not encrypt them. Use private +addresses, a VPN, or TLS termination before traffic crosses an untrusted +network. ## Prerequisites @@ -54,14 +54,24 @@ an external metrics collector needs them. ## 1. Install the runtime nodes +Generate one API key, deliver it through your normal secret-management channel, +and use the same value on every runtime node and the Gateway: + +```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +``` + Run the installation on each runtime node: ```bash -curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh \ - | sudo bash +curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh | sudo bash ``` -Edit `/etc/default/aenv` on each machine without removing the paths written by the installer. +Edit `/etc/default/aenv` on each machine without removing the paths written by +the installer, and add `AENV_API_KEY=` before starting the +services. A multi-node deployment must not let each node generate an +independent managed key. + See [Secure Sandboxes](../security/secure-sandboxes.md) if the deployment needs future cross-node sandbox recovery. Node A uses: @@ -106,6 +116,17 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` +Create `/etc/agentenv/auth.env` with the same key used on the runtime nodes: + +```bash +sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env +sudoedit /etc/agentenv/auth.env +``` + +```text +AENV_API_KEY= +``` + If the `agentenv-control` account already exists, the `useradd` command reports that fact and can be skipped. @@ -191,6 +212,7 @@ After=network-online.target agentenv-scheduler.service [Service] User=agentenv-control Group=agentenv-control +EnvironmentFile=/etc/agentenv/auth.env ExecStart=/usr/local/bin/agentenv-gateway -config /etc/agentenv/control-plane.json Restart=on-failure RestartSec=5 @@ -218,7 +240,8 @@ curl http://10.0.0.22:8000/health curl http://127.0.0.1:8080/health # Wait for node heartbeats, then inspect the cluster through the Gateway -curl http://127.0.0.1:8080/nodes +export AENV_API_KEY="$(sudo sed -n 's/^AENV_API_KEY=//p' /etc/agentenv/auth.env)" +curl -H "X-API-Key: ${AENV_API_KEY}" http://127.0.0.1:8080/nodes ``` The node list should contain `node-a` and `node-b`. Point clients at the @@ -227,7 +250,7 @@ Gateway, not directly at a runtime node: ```bash aenv auth # AENV server URL: http://10.0.0.10:8080 -# API key: dummy +# API key: ``` Sandbox create, list, lifecycle, and data-plane requests can then be routed diff --git a/docs/src/getting-started/aenv-cli.md b/docs/src/getting-started/aenv-cli.md index 7d1227ac..79500c7e 100644 --- a/docs/src/getting-started/aenv-cli.md +++ b/docs/src/getting-started/aenv-cli.md @@ -27,7 +27,7 @@ Save the server URL and API key. Credentials are stored at `~/.config/aenv/crede ```bash aenv auth # AENV server URL [http://localhost:8000]: The address of the AgentENV server -# API key: dummy (Any non-empty string works for local development.) +# API key: ``` --- diff --git a/docs/src/getting-started/quickstart.md b/docs/src/getting-started/quickstart.md index 0db8b658..9ebd1757 100644 --- a/docs/src/getting-started/quickstart.md +++ b/docs/src/getting-started/quickstart.md @@ -50,6 +50,7 @@ default. Transient namespace and daemon-socket state lives under `/run/aenv`. curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash docker pull ghcr.io/kvcache-ai/aenv-server:latest docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -p 8000:8000 \ ghcr.io/kvcache-ai/aenv-server:latest @@ -61,6 +62,7 @@ To customize the server configuration, download and edit the configuration file, curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/config/default.toml -o config.toml vim config.toml docker run --rm -it \ + --name aenv-server \ --device /dev/kvm --privileged -v /dev:/dev \ -v "$PWD/config.toml:/workspace/config/default.toml:ro" \ -p 8000:8000 \ @@ -87,14 +89,23 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in ### 3. Authenticate +The server generates the key on its first normal startup. Native installations +reuse the managed key; a normal Docker container keeps it in its writable +container layer: + +```bash +# Native +sudo cat /var/lib/aenv/secrets/api-key +# Docker +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + ```bash aenv auth # AENV server URL [http://localhost:8000]: http://127.0.0.1:8000 -# API key: dummy +# API key: ``` -For local development, any non-empty string works as the API key. - ### 4. Pull a template and run a sandbox ```bash diff --git a/docs/src/integration/e2b.md b/docs/src/integration/e2b.md index 6a6db325..653f3037 100644 --- a/docs/src/integration/e2b.md +++ b/docs/src/integration/e2b.md @@ -12,10 +12,16 @@ Set environment variables to point at your AgentENV server. See [Environment Var # Single-node example export E2B_API_URL=http://127.0.0.1:8000 export E2B_SANDBOX_URL=${E2B_API_URL} -export E2B_API_KEY=e2b_000000 -export E2B_ACCESS_TOKEN=dummy +export E2B_API_KEY=${AENV_API_KEY} ``` +No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` for +application proxy traffic and (for secure sandboxes) `envdAccessToken` for envd +control traffic. These credentials have different headers and trust boundaries: +use `e2b-traffic-access-token` for application routes and `X-Access-Token` only +for envd. This is transport data, not the deprecated user-supplied +`E2B_ACCESS_TOKEN`. + ### TypeScript SDK #### Setup diff --git a/docs/src/internals/architecture.md b/docs/src/internals/architecture.md index 1cf4fa21..3f4399f5 100644 --- a/docs/src/internals/architecture.md +++ b/docs/src/internals/architecture.md @@ -220,10 +220,11 @@ Discovery modes: **Deployment**: ```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" # shared by local runtime and gateway processes # local dev (single node) make start-server && make -C services run-scheduler && make -C services run-gateway -# docker compose (multi-node) +# docker compose (multi-node; shared auth volume is provisioned automatically) make deploy-up # gateway + scheduler + 2 backend nodes make deploy-down # teardown diff --git a/docs/src/internals/services.md b/docs/src/internals/services.md index 98dffb02..5343cf08 100644 --- a/docs/src/internals/services.md +++ b/docs/src/internals/services.md @@ -25,7 +25,8 @@ make proto # regenerate protobuf # Start scheduler (default: 127.0.0.1:9090) make -C services run-scheduler -# Start gateway +# Start gateway (use the same key on runtime nodes) +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make -C services run-gateway ``` @@ -41,6 +42,7 @@ The scheduler supports two node discovery modes: ### Docker Compose ```bash +# Run scripts/docker-setup.sh first for host prerequisites. make deploy-up # gateway + scheduler + 2 backend nodes make deploy-ps # status make deploy-logs # logs diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 9b763a4a..b04948df 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -2,8 +2,10 @@ Secure sandboxes use an envd access token for control-plane communication. This protects envd operations such as command execution and file access. -> [!WARNING] -> This feature does not add authentication to application ports exposed by the sandbox. +> [!NOTE] +> Secure mode protects envd control-plane operations. Application traffic uses +> the sandbox-scoped `trafficAccessToken` described in +> [Authentication](../configuration/authentication.md). Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to enable secure mode. Or use the CLI: @@ -11,7 +13,7 @@ Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to aenv start --secure ``` -The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Forked sandboxes get independent tokens. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. +The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. The application proxy credential is independent and is sent as `e2b-traffic-access-token`. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. ## Access-Token Seed @@ -36,11 +38,11 @@ The runtime DaemonSet reads the optional `agentenv-runtime-secrets` Secret. To c ```bash kubectl apply -f deploy/k8s/base/namespace.yaml -AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ - --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ + --from-literal="sandbox-access-token-hash-seed=${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" \ --dry-run=client -o yaml | kubectl apply -f - -unset AENV_ACCESS_TOKEN_HASH_SEED +unset AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED ``` Run this once for a new cluster and preserve the existing Secret during upgrades. An external secret manager may be used instead, provided it creates the same Secret name and key: diff --git a/scripts/install.sh b/scripts/install.sh index 1cc31e2d..9ccff8e0 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -256,6 +256,7 @@ sudo chmod 0640 "$CONFIG_PATH" if [[ -d /run/systemd/system ]]; then ENV_FILE_STATUS="exists" if [[ ! -f "$ENV_FILE" ]]; then + sudo install -o root -g "$SERVICE_GROUP" -m 0640 /dev/null "$ENV_FILE" sudo tee "$ENV_FILE" > /dev/null <> "$tmp_env" fi - sudo install -m 0644 "$tmp_env" "$ENV_FILE" + sudo install -o root -g "$SERVICE_GROUP" -m 0640 "$tmp_env" "$ENV_FILE" rm -f "$current_env" "$tmp_env" ENV_FILE_STATUS="updated" fi @@ -370,6 +371,7 @@ echo " CLI : ${INSTALL_DIR}/aenv" echo " Server : ${INSTALL_DIR}/server" echo " Data : ${DATA_DIR}" echo " Config : ${CONFIG_PATH}" +echo " API key: generated on first server start in ${DATA_DIR}/secrets/api-key" echo " Mode : ${VIRTUALIZATION_MODE}" if [[ -d /run/systemd/system ]]; then if [[ "$ENV_FILE_STATUS" == "written" ]]; then diff --git a/scripts/tests/e2e/lib/helpers.sh b/scripts/tests/e2e/lib/helpers.sh index b84b9aad..f5803b70 100644 --- a/scripts/tests/e2e/lib/helpers.sh +++ b/scripts/tests/e2e/lib/helpers.sh @@ -5,8 +5,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then E2E_HELPERS_SH_LOADED=1 : "${AENV_URL:?AENV_URL must be set}" - : "${AENV_API_KEY:=e2e-test-key}" - : "${AENV_ADMIN_TOKEN:=e2e-admin-token}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${AENV_TEMPLATE_ID:=ubuntu}" : "${AENV_PROXY_URL:=${AENV_URL}/proxy}" : "${E2E_MODE:=single-node}" @@ -124,7 +123,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then api_admin_get() { local path="$1" _curl_do -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}${path}" } @@ -132,7 +131,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then local base_url="$1" local path="$2" _curl_do -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${base_url}${path}" } @@ -140,7 +139,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then local path="$1" [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}${path}" > "$_E2E_STATUS" 2>/dev/null || true HTTP_STATUS=$(<"$_E2E_STATUS") HTTP_BODY=$(<"$_E2E_BODY") @@ -152,7 +151,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then local path="$2" [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ "${base_url}${path}" > "$_E2E_STATUS" 2>/dev/null || true HTTP_STATUS=$(<"$_E2E_STATUS") HTTP_BODY=$(<"$_E2E_BODY") diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index 723c61fe..e22d2a90 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -11,8 +11,8 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then source "${E2E_RUNTIME_DIR}/server.sh" : "${E2E_MODE:=single-node}" - : "${AENV_API_KEY:=e2e-test-key}" - : "${AENV_ADMIN_TOKEN:=e2e-admin-token}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" + export AENV_API_KEY : "${E2E_COMPOSE_FILE:=deploy/docker-compose.yml}" : "${E2E_COMPOSE_OVERRIDE_FILE:=scripts/tests/e2e/docker-compose.e2e.yml}" : "${E2E_COMPOSE_START_TIMEOUT:=120}" @@ -237,7 +237,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then log "Waiting for scheduler to observe ${expected_count} ready node(s) via ${AENV_URL}/nodes (timeout ${timeout}s) ..." for ((i = 1; i <= timeout; i++)); do response=$(curl -s \ - -H "X-Admin-Token: ${AENV_ADMIN_TOKEN}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ -w $'\n%{http_code}' \ "${AENV_URL}/nodes" 2>/dev/null || true) status="${response##*$'\n'}" @@ -398,6 +398,12 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _wait_for_health_url "agentenv-b" "${AENV_NODE_B_URL}" "${timeout}" || die "agentenv-b failed to become ready within ${timeout}s" + AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || + die "Failed to read the Compose deployment API key" + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,}$ ]] || + die "Compose deployment returned an invalid API key" + export AENV_API_KEY + expected_nodes="$(_runtime_node_count)" [[ "${expected_nodes}" -gt 0 ]] || expected_nodes=1 _wait_for_scheduler_ready_nodes "${timeout}" "${expected_nodes}" || diff --git a/scripts/tests/e2e/lib/server.sh b/scripts/tests/e2e/lib/server.sh index c550b661..1def3025 100644 --- a/scripts/tests/e2e/lib/server.sh +++ b/scripts/tests/e2e/lib/server.sh @@ -6,7 +6,7 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then : "${AENV_PORT:=18080}" : "${AENV_URL:=http://127.0.0.1:${AENV_PORT}}" - : "${AENV_API_KEY:=e2e-test-key}" + : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${SERVER_START_TIMEOUT:=30}" _SERVER_PID="" @@ -17,6 +17,7 @@ if [[ -z "${E2E_SERVER_SH_LOADED:-}" ]]; then local env_vars=( "API_ADDR=127.0.0.1:${AENV_PORT}" + "AENV_API_KEY=${AENV_API_KEY}" "RUST_LOG=agentenv=info,envd=info" ) [[ -n "$config" ]] && env_vars+=("AENV_CONFIG_PATH=${config}") diff --git a/scripts/tests/e2e/suites/08_auth.sh b/scripts/tests/e2e/suites/08_auth.sh index 5f0d30da..47837517 100755 --- a/scripts/tests/e2e/suites/08_auth.sh +++ b/scripts/tests/e2e/suites/08_auth.sh @@ -12,6 +12,25 @@ log "Suite: Authentication" api_get_no_auth "/sandboxes" assert_status "$HTTP_STATUS" "401" "no auth header returns 401" +# -- Alternative and malformed credentials are rejected -- +_curl_do -s -H "X-API-Key: wrong-key" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "wrong API key returns 401" + +_curl_do -s -H "Authorization: Bearer ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "Authorization does not authenticate AgentENV" + +_curl_do -s -H "X-Admin-Token: ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "legacy admin token does not authenticate AgentENV" + +_curl_do -s -H "X-Team-ID: ${AENV_API_KEY}" "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "legacy team key does not authenticate AgentENV" + +_curl_do -s \ + -H "X-API-Key: ${AENV_API_KEY}" \ + -H "X-API-Key: ${AENV_API_KEY}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "duplicate API key headers return 401" + # -- Request with valid API key succeeds -- api_get "/sandboxes" assert_not_eq "$HTTP_STATUS" "401" "valid API key does not return 401" diff --git a/scripts/tests/e2e/suites/09_e2b_compat.sh b/scripts/tests/e2e/suites/09_e2b_compat.sh index a5bb8ff6..611b09ad 100755 --- a/scripts/tests/e2e/suites/09_e2b_compat.sh +++ b/scripts/tests/e2e/suites/09_e2b_compat.sh @@ -10,8 +10,7 @@ log "Suite: E2B Compatibility" export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" -export E2B_API_KEY="e2b_000000" -export E2B_ACCESS_TOKEN="${AENV_API_KEY}" +export E2B_API_KEY="${AENV_API_KEY}" export E2B_COMPAT_USER_IMAGE="${E2B_COMPAT_USER_IMAGE:-${E2E_TEMPLATE_USER_IMAGE:-ghcr.io/linuxserver/baseimage-ubuntu:noble}}" cli_available=0 diff --git a/scripts/tests/e2e/suites/14_code_interpreter.sh b/scripts/tests/e2e/suites/14_code_interpreter.sh index e95b2a8b..2b84c70a 100755 --- a/scripts/tests/e2e/suites/14_code_interpreter.sh +++ b/scripts/tests/e2e/suites/14_code_interpreter.sh @@ -18,7 +18,7 @@ exit 0 export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" -export E2B_API_KEY="e2b_000000" +export E2B_API_KEY="${AENV_API_KEY}" if ! python3 -c 'import e2b_code_interpreter' >/dev/null 2>&1; then warn "e2b_code_interpreter Python package not installed; skipping" diff --git a/services/README.md b/services/README.md index ee2880ca..ff6743d6 100644 --- a/services/README.md +++ b/services/README.md @@ -69,14 +69,20 @@ Start scheduler: make run-scheduler ``` -Start gateway: +Start gateway with the same API key configured on every AgentENV runtime node: ```bash +export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. +The gateway and runtime nodes require the same API key. The gateway reads an +explicit `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate one. +Application proxy requests may additionally use the sandbox response's +`trafficAccessToken` in the `e2b-traffic-access-token` header. + ## Scheduler configuration Scheduler discovery modes: @@ -176,6 +182,7 @@ LOG_FORMAT=json make run-gateway From **repository root**, start gateway + scheduler + two backend nodes: ```bash +# Run scripts/docker-setup.sh first for host prerequisites. make deploy-up ``` diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index b21738bf..11a8255b 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -4,10 +4,12 @@ import ( "context" "errors" "flag" + "fmt" "log" "net/http" "os" "os/signal" + "strings" "syscall" "time" @@ -22,6 +24,11 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +const ( + apiKeyEnv = "AENV_API_KEY" + defaultAPIKeyPath = "/run/secrets/api-key" +) + func newSchedulerConn(addr string) (*grpc.ClientConn, error) { return grpc.NewClient( addr, @@ -29,6 +36,42 @@ func newSchedulerConn(addr string) (*grpc.ClientConn, error) { ) } +func loadAPIKey() (string, error) { + return loadAPIKeyFrom(os.LookupEnv, defaultAPIKeyPath) +} + +func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { + if value, present := lookupEnv(apiKeyEnv); present { + return validateAPIKey(value, apiKeyEnv) + } + + contents, err := os.ReadFile(secretPath) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + } + return "", fmt.Errorf("read API key secret %s: %w", secretPath, err) + } + return validateAPIKey(string(contents), secretPath) +} + +func validateAPIKey(value, source string) (string, error) { + value = strings.TrimSpace(value) + if len(value) < 32 { + return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + } + for _, char := range []byte(value) { + if (char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '.' || char == '_' || char == '~' || char == '-' { + continue + } + return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + } + return value, nil +} + func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -37,6 +80,10 @@ func main() { if err != nil { log.Fatalf("load config failed: %v", err) } + apiKey, err := loadAPIKey() + if err != nil { + log.Fatalf("load API key failed: %v", err) + } logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { @@ -65,6 +112,7 @@ func main() { s, err := gateway.NewServer(logger, schedulerClient, gateway.ServerOptions{ RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, + APIKey: apiKey, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go new file mode 100644 index 00000000..9064986b --- /dev/null +++ b/services/gateway/cmd/main_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestValidateAPIKey(t *testing.T) { + t.Parallel() + + got, err := validateAPIKey(" "+testAPIKey+"\n", "test") + if err != nil { + t.Fatalf("validateAPIKey() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("validateAPIKey() = %q, want %q", got, testAPIKey) + } + + for _, invalid := range []string{"", "too-short", strings.Repeat("a", 31), strings.Repeat("a", 31) + "!"} { + if _, err := validateAPIKey(invalid, "test"); err == nil { + t.Errorf("validateAPIKey(%q) unexpectedly succeeded", invalid) + } + } +} + +func TestLoadAPIKeyFromEnvironment(t *testing.T) { + got, err := loadAPIKeyFrom( + func(name string) (string, bool) { return testAPIKey, name == apiKeyEnv }, + filepath.Join(t.TempDir(), "missing"), + ) + if err != nil { + t.Fatalf("loadAPIKey() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKey() = %q, want %q", got, testAPIKey) + } +} + +func TestLoadAPIKeyRejectsExplicitEmptyEnvironment(t *testing.T) { + if _, err := loadAPIKeyFrom( + func(name string) (string, bool) { return "", name == apiKeyEnv }, + filepath.Join(t.TempDir(), "missing"), + ); err == nil { + t.Fatal("loadAPIKey() unexpectedly accepted an empty environment value") + } +} + +func TestLoadAPIKeyFromFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(path, []byte(testAPIKey+"\n"), 0o444); err != nil { + t.Fatal(err) + } + got, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAPIKeyFrom() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKeyFrom() = %q, want %q", got, testAPIKey) + } +} + +func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missing := filepath.Join(dir, "missing") + if _, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, missing); err == nil { + t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index dcacf570..70cf02aa 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,6 +3,9 @@ package gateway import ( "bytes" "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" "encoding/json" "errors" "io" @@ -22,6 +25,12 @@ import ( ) const ( + headerAPIKey = "X-API-Key" + headerTrafficToken = "e2b-traffic-access-token" + headerEnvdAccessToken = "X-Access-Token" + trafficTokenPrefix = "aenv_trf_" + trafficTokenContext = "agentenv-sandbox-traffic-v1\x00" + envdControlPlanePort = 49983 headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" headerTargetPort = "x-agentenv-target-port" @@ -41,6 +50,7 @@ const ( ) type ServerOptions struct { + APIKey string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -53,6 +63,7 @@ type Server struct { scheduler schedulerv1.SchedulerClient queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client + apiKey []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -63,6 +74,11 @@ type Server struct { } func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, options ServerOptions) (*Server, error) { + apiKey := strings.TrimSpace(options.APIKey) + if apiKey == "" { + return nil, errors.New("API key is required") + } + sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { return nil, err @@ -80,6 +96,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, httpClient: &http.Client{}, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, + apiKey: []byte(apiKey), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -119,7 +136,7 @@ func (s *Server) Handler() http.Handler { } s.handleProxy(w, r) }) - return s.instrumentGatewayHTTP(core) + return s.instrumentGatewayHTTP(s.authenticate(core)) } func (s *Server) writeJSON(w http.ResponseWriter, status int, value any) { @@ -809,3 +826,89 @@ func extractSandboxIDsFromResponse(body []byte) []string { } return unique } + +func singleHeaderMatches(headers http.Header, name string, expected []byte) bool { + values := headers.Values(name) + return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) +} + +func trafficAccessToken(apiKey []byte, sandboxID string) string { + mac := hmac.New(sha256.New, apiKey) + _, _ = mac.Write([]byte(trafficTokenContext)) + _, _ = mac.Write([]byte(sandboxID)) + return trafficTokenPrefix + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { + if strings.TrimRight(r.URL.Path, "/") == "/proxy" || strings.HasPrefix(r.URL.Path, "/proxy/") { + return true + } + + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if hostRoute != nil || err != nil { + return true + } + + return !isSandboxControlPlaneRequest(r) && hasProxyRoutingHeaders(r.Header) +} + +func (s *Server) sandboxIDForDataPlaneAuth(r *http.Request) (string, bool) { + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if err != nil { + return "", false + } + if hostRoute != nil { + return hostRoute.sandboxID, true + } + return sandboxIDFromHeaders(r.Header) +} + +func (s *Server) isEnvdDataPlaneRequest(r *http.Request) bool { + hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) + if err != nil { + return false + } + if hostRoute != nil { + return hostRoute.targetPort == envdControlPlanePort + } + targetPort, ok := targetPortFromHeaders(r.Header) + if !ok { + return false + } + port, err := strconv.Atoi(targetPort) + return err == nil && port == envdControlPlanePort +} + +func hasSingleNonEmptyHeader(headers http.Header, name string) bool { + values := headers.Values(name) + return len(values) == 1 && strings.TrimSpace(values[0]) != "" +} + +func (s *Server) authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dataPlane := s.isSandboxDataPlaneRequest(r) + if r.URL.Path == "/health" && !dataPlane { + next.ServeHTTP(w, r) + return + } + + authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) + if !authorized && dataPlane { + if sandboxID, ok := s.sandboxIDForDataPlaneAuth(r); ok { + expected := trafficAccessToken(s.apiKey, sandboxID) + authorized = singleHeaderMatches(r.Header, headerTrafficToken, []byte(expected)) + } + } + if !authorized && dataPlane && s.isEnvdDataPlaneRequest(r) { + // The runtime node owns the envd token seed and performs the definitive + // sandbox-scoped validation before forwarding the request to envd. + authorized = hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) + } + if !authorized { + w.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) +} diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 028b3709..fa8358d0 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,6 +148,8 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } +const testAPIKey = "test-api-key" + type testServerOption func(*ServerOptions) func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, timeout time.Duration, maxRespSize int64, opts ...testServerOption) *Server { @@ -156,6 +158,7 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti options := ServerOptions{ RequestTimeout: timeout, MaxResponseSize: maxRespSize, + APIKey: testAPIKey, } for _, opt := range opts { opt(&options) @@ -168,6 +171,143 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti return server } +func authenticatedTestHandler(server *Server) http.Handler { + handler := server.Handler() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Header.Set(headerAPIKey, testAPIKey) + handler.ServeHTTP(w, r) + }) +} + +func TestNewServerRejectsEmptyAPIKey(t *testing.T) { + _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ + RequestTimeout: time.Second, + MaxResponseSize: 1024, + }) + if err == nil { + t.Fatal("NewServer accepted an empty API key") + } +} + +func TestGatewayRequiresExactAPIKey(t *testing.T) { + server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) + handler := server.Handler() + tests := []struct { + name string + addHeaders func(http.Header) + wantStatus int + }{ + { + name: "missing", + addHeaders: func(http.Header) {}, + wantStatus: http.StatusUnauthorized, + }, + { + name: "wrong", + addHeaders: func(headers http.Header) { + headers.Set(headerAPIKey, "wrong-key") + }, + wantStatus: http.StatusUnauthorized, + }, + { + name: "authorization is application data", + addHeaders: func(headers http.Header) { + headers.Set("Authorization", "Bearer "+testAPIKey) + }, + wantStatus: http.StatusUnauthorized, + }, + { + name: "valid", + addHeaders: func(headers http.Header) { + headers.Set(headerAPIKey, testAPIKey) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "duplicate", + addHeaders: func(headers http.Header) { + headers.Add(headerAPIKey, testAPIKey) + headers.Add(headerAPIKey, testAPIKey) + }, + wantStatus: http.StatusUnauthorized, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + tt.addHeaders(req.Header) + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, req) + + if recorder.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, tt.wantStatus) + } + }) + } +} + +func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { + const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" + lookupCalls := 0 + server := newTestServer(t, stubSchedulerClient{ + lookupNodeFunc: func(context.Context, *schedulerv1.LookupNodeRequest, ...grpc.CallOption) (*schedulerv1.LookupNodeResponse, error) { + lookupCalls++ + return nil, fmt.Errorf("lookup reached") + }, + }, time.Second, 1024) + handler := server.Handler() + + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "49983") + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("valid scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "49983") + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), "another-sandbox")) + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("wrong scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "8080") + req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAPIKey), sandboxID)) + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("envd token authorized application proxy: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } +} + +func TestTrafficAccessTokenVector(t *testing.T) { + const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" + const want = "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" + if got := trafficAccessToken([]byte("test-key"), sandboxID); got != want { + t.Fatalf("trafficAccessToken() = %q, want %q", got, want) + } +} + func withSandboxProxyDomains(domains ...string) testServerOption { return func(options *ServerOptions) { options.SandboxProxyDomains = domains @@ -398,7 +538,7 @@ func TestHandleProxyReturnsAggregatedNodesFromScheduler(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "http://gateway.test/nodes?clusterID=cluster-1", nil) response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) + authenticatedTestHandler(server).ServeHTTP(response, request) if response.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", response.Code) @@ -453,7 +593,7 @@ func TestHandleProxyDirectForwardsNodeDetail(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "http://gateway.test/nodes/node-a?clusterID=cluster-1", nil) response := httptest.NewRecorder() - server.Handler().ServeHTTP(response, request) + authenticatedTestHandler(server).ServeHTTP(response, request) if response.Code != http.StatusOK { t.Fatalf("expected status 200, got %d", response.Code) @@ -486,7 +626,7 @@ func TestLookupNodeUsesQueryOnlySchedulerClient(t *testing.T) { } server := newTestServer(t, mainScheduler, time.Second, 1024, withQueryOnlyScheduler(queryScheduler)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/health", nil) @@ -555,7 +695,7 @@ func TestSandboxControlPlaneRequestWithE2BHeadersUsesPathRoute(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes/sbx-path/connect", strings.NewReader(`{"timeout":60}`)) @@ -833,7 +973,7 @@ func TestHandleProxyAggregatesSandboxListAcrossNodes(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/sandboxes?metadata=team%3Dalpha", nil) @@ -909,7 +1049,7 @@ func TestHandleProxyAggregatesV2SandboxesWithGlobalPagination(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/v2/sandboxes?metadata=team%3Dalpha&state=running%2Cpaused&limit=2", nil) @@ -1012,7 +1152,7 @@ func TestHandleProxyAggregatesSandboxListDedupsDuplicateSandboxIDs(t *testing.T) }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/v2/sandboxes?limit=10") @@ -1055,7 +1195,7 @@ func TestHandleProxyClusterListFailsWhenNodeFails(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/sandboxes") @@ -1085,7 +1225,7 @@ func TestHandleProxyClusterListPropagatesUnauthorized(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/sandboxes") @@ -1363,7 +1503,7 @@ func TestFlushInterval(t *testing.T) { func TestHealthEndpointReturnsGatewayHealthWithoutProxyHeaders(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() resp, err := http.Get(gatewayServer.URL + "/health") @@ -1414,7 +1554,7 @@ func TestHealthEndpointWithSandboxHeadersProxiesToSandbox(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/health", nil) @@ -1448,7 +1588,7 @@ func TestHealthEndpointWithSandboxHeadersProxiesToSandbox(t *testing.T) { func TestHealthEndpointWithProxyHeadersMissingSandboxIDReturnsBadRequest(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/health", nil) @@ -1499,7 +1639,7 @@ func TestHealthEndpointWithHostRoutingProxiesToSandbox(t *testing.T) { }, nil }, }, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/health", nil) @@ -1574,7 +1714,7 @@ func TestHandleProxyHostBasedRoutingForwardsToSandboxProxy(t *testing.T) { }, }, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/readyz?x=1", nil) @@ -1622,7 +1762,7 @@ func TestHandleProxyHostBasedRoutingForwardsToSandboxProxy(t *testing.T) { func TestHandleProxyHostBasedRoutingRejectsInvalidHost(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024, withSandboxProxyDomains("sandbox-proxy.example.invalid")) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodGet, gatewayServer.URL+"/readyz", nil) @@ -1708,7 +1848,7 @@ func TestHandleProxyHTTPForwardingAndRecordAssignment(t *testing.T) { }, }, time.Second, 1024, withDebugMode(true)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes", strings.NewReader(`{"template":"base"}`)) @@ -1832,7 +1972,7 @@ func TestHandleProxyColdSandboxCreateRecordsAssignment(t *testing.T) { }, }, time.Second, 1024, withDebugMode(true)) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() req, err := http.NewRequest(http.MethodPost, gatewayServer.URL+"/sandboxes-cold", strings.NewReader(`{"image":"ubuntu:24.04"}`)) @@ -1964,7 +2104,7 @@ func TestHandleProxyWebSocketForwarding(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() gatewayURL, err := url.Parse(gatewayServer.URL) @@ -2080,7 +2220,7 @@ func TestHandleProxyPreservesEncodedPathSegments(t *testing.T) { }, }, time.Second, 1024) - gatewayServer := httptest.NewServer(server.Handler()) + gatewayServer := httptest.NewServer(authenticatedTestHandler(server)) defer gatewayServer.Close() tests := []struct { diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index c6362583..6d525724 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -1,41 +1,120 @@ use async_trait::async_trait; -use axum::http::header::HeaderMap; +use axum::{ + body::Body, + extract::{Request, State}, + http::{header::HeaderMap, StatusCode}, + middleware::Next, + response::{IntoResponse, Response}, +}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; use agentenv_http_server::apis; use super::{ApiImpl, Claims}; +use crate::api::proxy; -fn non_empty_header(headers: &HeaderMap, name: &str) -> bool { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| !value.is_empty()) +pub(crate) const API_KEY_HEADER: &str = "x-api-key"; +pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; +pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; +const TRAFFIC_TOKEN_PREFIX: &str = "aenv_trf_"; +const TRAFFIC_TOKEN_CONTEXT: &[u8] = b"agentenv-sandbox-traffic-v1\0"; + +fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> bool { + let mut values = headers.get_all(name).iter(); + let Some(value) = values.next() else { + return false; + }; + if values.next().is_some() { + return false; + } + + value.as_bytes() == expected.as_bytes() +} + +fn derive_traffic_access_token(api_key: &[u8], sandbox_id: &str) -> String { + let mut mac = + Hmac::::new_from_slice(api_key).expect("HMAC accepts API keys of any length"); + mac.update(TRAFFIC_TOKEN_CONTEXT); + mac.update(sandbox_id.as_bytes()); + format!( + "{TRAFFIC_TOKEN_PREFIX}{}", + URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) + ) +} + +impl ApiImpl { + pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { + single_header_matches(headers, API_KEY_HEADER, &self.api_key) + } + + pub(crate) fn traffic_access_token(&self, sandbox_id: &str) -> String { + derive_traffic_access_token(self.api_key.as_bytes(), sandbox_id) + } + + fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: &str) -> bool { + let expected = self.traffic_access_token(sandbox_id); + single_header_matches(headers, TRAFFIC_ACCESS_TOKEN_HEADER, &expected) + } +} + +pub(crate) async fn require_auth( + State(api_impl): State, + request: Request, + next: Next, +) -> Response +where + I: AsRef + Clone + Send + Sync + 'static, +{ + let proxy_request = + proxy::is_sandbox_proxy_request(&request, api_impl.as_ref().sandbox_proxy_domains()); + if request.uri().path() == "/health" && !proxy_request { + return next.run(request).await; + } + + let mut authorized = api_impl.as_ref().has_valid_api_key(request.headers()); + if !authorized && proxy_request { + authorized = + proxy::sandbox_id_for_proxy_auth(&request, api_impl.as_ref().sandbox_proxy_domains()) + .is_some_and(|sandbox_id| { + api_impl + .as_ref() + .has_valid_traffic_access_token(request.headers(), &sandbox_id) + }); + } + if !authorized && proxy_request { + if let Some((sandbox_id, target_port, candidate)) = proxy::envd_access_token_for_proxy_auth( + &request, + api_impl.as_ref().sandbox_proxy_domains(), + ) { + authorized = proxy::has_valid_envd_access_token( + api_impl.as_ref(), + sandbox_id, + target_port, + candidate, + ) + .await; + } + } + + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + + next.run(request).await } #[async_trait] impl apis::ApiKeyAuthHeader for ApiImpl { type Claims = Claims; - // TODO: Validate configured authentication credentials instead of only - // checking that they are present. async fn extract_claims_from_header( &self, headers: &HeaderMap, - key: &str, + _key: &str, ) -> Option { - let admin_token = non_empty_header(headers, "X-Admin-Token"); - if key == "X-Admin-Token" { - return admin_token.then_some(Claims); - } - - if non_empty_header(headers, "X-API-Key") - || non_empty_header(headers, "X-Team-ID") - || admin_token - { - Some(Claims) - } else { - None - } + self.has_valid_api_key(headers).then_some(Claims) } } @@ -45,17 +124,53 @@ impl apis::ApiAuthBasic for ApiImpl { async fn extract_claims_from_auth_header( &self, - kind: apis::BasicAuthKind, + _kind: apis::BasicAuthKind, headers: &HeaderMap, - key: &str, + _key: &str, ) -> Option { - let expected_scheme = match kind { - apis::BasicAuthKind::Basic => "Basic", - apis::BasicAuthKind::Bearer => "Bearer", - _ => return None, - }; - let value = headers.get(key)?.to_str().ok()?; - let (scheme, credentials) = value.split_once(' ')?; - (scheme.eq_ignore_ascii_case(expected_scheme) && !credentials.is_empty()).then_some(Claims) + // The outer middleware is authoritative. This adapter keeps the + // E2B-compatible generated router from rejecting its API-key request. + self.has_valid_api_key(headers).then_some(Claims) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_match_requires_one_exact_value() { + let mut headers = HeaderMap::new(); + assert!(!single_header_matches( + &headers, + API_KEY_HEADER, + "correct-key" + )); + headers.insert(API_KEY_HEADER, "correct-key".parse().unwrap()); + assert!(single_header_matches( + &headers, + API_KEY_HEADER, + "correct-key" + )); + assert!(!single_header_matches( + &headers, + API_KEY_HEADER, + "wrong-key" + )); + + headers.append(API_KEY_HEADER, "correct-key".parse().unwrap()); + assert!(!single_header_matches( + &headers, + API_KEY_HEADER, + "correct-key" + )); + } + + #[test] + fn traffic_access_token_matches_gateway_contract() { + assert_eq!( + derive_traffic_access_token(b"test-key", "0191f4d0-7b2a-7c11-9c2d-0123456789ab"), + "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" + ); } } diff --git a/src/api/impls/mod.rs b/src/api/impls/mod.rs index 1e1d50bc..fb8f2bb7 100644 --- a/src/api/impls/mod.rs +++ b/src/api/impls/mod.rs @@ -1,6 +1,6 @@ mod admin; mod attached_drives; -mod auth; +pub(crate) mod auth; mod pagination; mod sandbox; mod snapshots; @@ -33,6 +33,7 @@ pub struct ApiImpl { observability: Option>, proxy_client: ProxyClient, sandbox_proxy_domains: Vec, + api_key: String, } impl ApiImpl { @@ -43,6 +44,7 @@ impl ApiImpl { image_resolver: Arc, observability: Option>, sandbox_proxy_domains: Vec, + api_key: String, ) -> Self { Self { orchestrator, @@ -52,6 +54,7 @@ impl ApiImpl { observability, proxy_client: build_proxy_client(), sandbox_proxy_domains, + api_key, } } diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 5382dfb6..85b91f84 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -220,6 +220,9 @@ impl ApiImpl { .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; + sandbox.traffic_access_token = Some(Nullable::Present( + self.traffic_access_token(&sandbox.sandbox_id), + )); sandbox.domain = self .sandbox_proxy_domains() .first() diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 326c847f..1cfcb015 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -7,7 +7,7 @@ use axum::{ rejection::WebSocketUpgradeRejection, CloseFrame, Message as WebSocketMessage, WebSocket, WebSocketUpgrade, }, - FromRequestParts, Request, State, + FromRequestParts, MatchedPath, Request, State, }, http::{header, HeaderMap, HeaderName, HeaderValue, Method, Response, StatusCode, Uri}, middleware::Next, @@ -35,10 +35,16 @@ use tokio_tungstenite::{ use tracing::{debug, info, trace, warn}; use crate::{ - api::ApiImpl, + api::{ + impls::auth::{API_KEY_HEADER, ENVD_ACCESS_TOKEN_HEADER, TRAFFIC_ACCESS_TOKEN_HEADER}, + ApiImpl, + }, cfg::ConfigManager, observability::prometheus::HttpRouteSource, - orchestrator::{NewTimeout, OrchestratorError, ProxyLookupResult, ProxyTarget, SandboxState}, + orchestrator::{ + NewTimeout, OrchestratorError, ProxyLookupResult, ProxyTarget, SandboxMetadata, + SandboxState, + }, types::SandboxId, }; @@ -50,6 +56,8 @@ struct ResolvedProxyRequest { sandbox_id: SandboxId, upstream_uri: Uri, original_host: Option, + target_port: u16, + envd_port: u16, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -83,8 +91,6 @@ const E2B_SANDBOX_ID_HEADER: &str = "e2b-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; /// E2B-compatible alias for the target port header. const E2B_TARGET_PORT_HEADER: &str = "e2b-sandbox-port"; -const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; - #[cfg(test)] const PROXY_CONNECT_TIMEOUT: Duration = Duration::from_millis(100); #[cfg(not(test))] @@ -142,6 +148,91 @@ where .with_state(api_impl) } +fn proxy_route_for_auth(request: &Request, domains: &[String]) -> Option { + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(route)) => return Some(route), + Err(_) => return None, + Ok(None) => {} + } + + Some(HostProxyRoute { + sandbox_id: parse_sandbox_id_header(request.headers()).ok()?, + target_port: parse_target_port_header(request.headers()).ok()?, + }) +} + +pub(crate) fn sandbox_id_for_proxy_auth(request: &Request, domains: &[String]) -> Option { + Some( + proxy_route_for_auth(request, domains)? + .sandbox_id + .to_string(), + ) +} + +pub(crate) fn envd_access_token_for_proxy_auth( + request: &Request, + domains: &[String], +) -> Option<(SandboxId, u16, String)> { + let route = proxy_route_for_auth(request, domains)?; + let candidate = { + let mut candidates = request.headers().get_all(ENVD_ACCESS_TOKEN_HEADER).iter(); + let candidate = candidates.next()?; + if candidates.next().is_some() { + return None; + } + let candidate = candidate.to_str().ok()?; + candidate.to_owned() + }; + + Some((route.sandbox_id, route.target_port, candidate)) +} + +pub(crate) async fn has_valid_envd_access_token( + api_impl: &ApiImpl, + sandbox_id: SandboxId, + target_port: u16, + candidate: String, +) -> bool { + let Ok(Some(metadata)) = api_impl.orchestrator().get_sandbox(&sandbox_id).await else { + return false; + }; + if !metadata.secure || target_port != effective_envd_port(&metadata) { + return false; + } + + api_impl + .orchestrator() + .validate_envd_access_token(sandbox_id, &candidate) +} + +pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { + let path = request.uri().path(); + if path == PROXY_ROUTE || path.starts_with("/proxy/") { + return true; + } + + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(_)) | Err(_) => true, + Ok(None) => { + request.extensions().get::().is_none() + && has_routing_header(request.headers()) + } + } +} + +fn request_host(request: &Request) -> Option<&str> { + request + .headers() + .get(header::HOST) + .and_then(|host| host.to_str().ok()) + .or_else(|| { + request + .uri() + .authority() + .map(|authority| authority.as_str()) + }) +} + pub(crate) async fn sandbox_proxy_classifier( State(api_impl): State, request: Request, @@ -167,7 +258,9 @@ where }); let host_route = match parse_host_proxy_route(host, api_impl.as_ref().sandbox_proxy_domains()) { Ok(Some(route)) => route, - Ok(None) => return next.run(request).await, + Ok(None) => { + return next.run(request).await; + } Err(err) => { return with_route_source(proxy_error_response(&err), HttpRouteSource::ProxyHost); } @@ -358,6 +451,14 @@ fn has_routing_header(headers: &HeaderMap) -> bool { headers.get(SANDBOX_ID_HEADER).is_some() || headers.get(E2B_SANDBOX_ID_HEADER).is_some() } +fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { + metadata + .paused_state + .as_ref() + .and_then(|state| state.control_plane_port()) + .unwrap_or_else(|| ConfigManager::global_config().tools.control_plane_port) +} + /// Proxies a standard HTTP request to the resolved upstream URI and returns the response. async fn proxy_http_request( api_impl: &ApiImpl, @@ -369,9 +470,11 @@ async fn proxy_http_request( sandbox_id, upstream_uri, original_host, + target_port, + envd_port, } = resolved; - sanitize_request_headers(&mut parts.headers); + sanitize_request_headers(&mut parts.headers, target_port, envd_port); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -552,9 +655,11 @@ async fn proxy_websocket_request( sandbox_id, upstream_uri, original_host, + target_port, + envd_port, } = resolved; - sanitize_websocket_request_headers(&mut parts.headers); + sanitize_websocket_request_headers(&mut parts.headers, target_port, envd_port); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -735,6 +840,14 @@ async fn resolve_proxy_request( } }; + let metadata = api_impl + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? + .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; + let envd_port = effective_envd_port(&metadata); + let upstream_uri = if is_websocket_request { build_upstream_uri_with_scheme("ws", &target, target_port, proxy_path, parts.uri.query()) } else { @@ -746,6 +859,8 @@ async fn resolve_proxy_request( sandbox_id, upstream_uri, original_host: parts.headers.get(header::HOST).cloned(), + target_port, + envd_port, }) } @@ -755,15 +870,15 @@ async fn authorize_secure_envd_auto_resume( target_port: u16, headers: &HeaderMap, ) -> Result<(), Response> { - if target_port != ConfigManager::global_config().tools.control_plane_port { - return Ok(()); - } let metadata = api_impl .orchestrator() .get_sandbox(&sandbox_id) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; + if target_port != effective_envd_port(&metadata) { + return Ok(()); + } if !metadata.secure { return Ok(()); } @@ -963,19 +1078,24 @@ fn build_upstream_uri_with_scheme( .map_err(|_| StatusCode::BAD_REQUEST) } -fn sanitize_request_headers(headers: &mut HeaderMap) { +fn sanitize_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { // These headers are only for the control-plane hop between the client and // AgentENV. Upstream sandbox services should not see them. headers.remove(SANDBOX_ID_HEADER); headers.remove(E2B_SANDBOX_ID_HEADER); headers.remove(TARGET_PORT_HEADER); headers.remove(E2B_TARGET_PORT_HEADER); + headers.remove(API_KEY_HEADER); + headers.remove(TRAFFIC_ACCESS_TOKEN_HEADER); headers.remove(header::HOST); + if target_port != envd_port { + headers.remove(ENVD_ACCESS_TOKEN_HEADER); + } remove_hop_by_hop_headers(headers); } -fn sanitize_websocket_request_headers(headers: &mut HeaderMap) { - sanitize_request_headers(headers); +fn sanitize_websocket_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { + sanitize_request_headers(headers, target_port, envd_port); headers.remove(header::SEC_WEBSOCKET_ACCEPT); headers.remove(header::SEC_WEBSOCKET_EXTENSIONS); headers.remove(header::SEC_WEBSOCKET_KEY); @@ -1389,9 +1509,14 @@ mod tests { "e2b_sandbox_header_seen": headers.get(E2B_SANDBOX_ID_HEADER).is_some(), "target_port_header_seen": headers.get(TARGET_PORT_HEADER).is_some(), "e2b_target_port_header_seen": headers.get(E2B_TARGET_PORT_HEADER).is_some(), - "envd_access_token": headers + "api_key_header_seen": headers.get(API_KEY_HEADER).is_some(), + "traffic_token_header_seen": headers.get(TRAFFIC_ACCESS_TOKEN_HEADER).is_some(), + "access_token": headers .get(ENVD_ACCESS_TOKEN_HEADER) .and_then(|value| value.to_str().ok()), + "authorization": headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), "forwarded_host": headers .get("x-forwarded-host") .and_then(|value| value.to_str().ok()), @@ -1622,6 +1747,7 @@ mod tests { image_resolver, None, domains, + "test-key".to_string(), )) } @@ -1650,11 +1776,27 @@ mod tests { .await } + async fn proxy_app_with_access_token_for_sandbox( + sandbox_id: &SandboxId, + ) -> (axum::Router, String) { + let api = build_api().await; + let access_token = api.traffic_access_token(&sandbox_id.to_string()); + api.orchestrator() + .set_proxy_target_for_test( + *sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + (server::new(api), access_token) + } + async fn proxy_app_for_sandbox_with_domains( sandbox_id: &SandboxId, domains: Vec, - ) -> axum::Router { + ) -> (axum::Router, String) { let api = build_api_with_sandbox_proxy_domains(domains).await; + let access_token = api.traffic_access_token(&sandbox_id.to_string()); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -1662,7 +1804,7 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; - server::new(api) + (server::new(api), access_token) } async fn proxy_app_for_running_sandbox_without_route(sandbox_id: &SandboxId) -> axum::Router { @@ -1732,7 +1874,11 @@ mod tests { HeaderValue::from_static("keep"), ); - sanitize_request_headers(&mut headers); + sanitize_request_headers( + &mut headers, + 8080, + ConfigManager::global_config().tools.control_plane_port, + ); assert!(headers.get(SANDBOX_ID_HEADER).is_none()); assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); @@ -1786,6 +1932,120 @@ mod tests { assert!(!is_send_request_failure_text(&"client error (Connect)")); } + #[tokio::test] + async fn server_requires_exact_api_key_and_leaves_health_public() { + let app = server::new(build_api().await); + + for request in [ + Request::builder() + .uri("/nonexistent/path") + .body(Body::empty()) + .unwrap(), + Request::builder() + .uri("/nonexistent/path") + .header(header::AUTHORIZATION, "Bearer test-key") + .body(Body::empty()) + .unwrap(), + Request::builder() + .uri("/nonexistent/path") + .header(API_KEY_HEADER, "wrong-key") + .body(Body::empty()) + .unwrap(), + Request::builder() + .uri("/nonexistent/path") + .header(API_KEY_HEADER, "test-key") + .header(API_KEY_HEADER, "test-key") + .body(Body::empty()) + .unwrap(), + ] { + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/nonexistent/path") + .header(API_KEY_HEADER, "test-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/health") + .header(header::HOST, "localhost") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + + let response = app + .oneshot( + Request::builder() + .uri("/health") + .header(header::HOST, "localhost") + .header(SANDBOX_ID_HEADER, SandboxId::new().to_string()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + } + + #[tokio::test] + async fn traffic_token_cannot_authenticate_control_plane() { + let api = build_api().await; + let sandbox_id = SandboxId::new(); + let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let app = server::new(api); + let response = app + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/sandboxes/{sandbox_id}/pause")) + .header(TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token) + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "80") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn envd_token_cannot_authenticate_application_proxy() { + let api = build_api().await; + let sandbox_id = SandboxId::new(); + let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let response = server::new(api) + .oneshot( + Request::builder() + .uri("/proxy/hello") + .header(ENVD_ACCESS_TOKEN_HEADER, traffic_token) + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "80") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + #[tokio::test] async fn proxy_requires_routing_headers() { let app = server::new(build_api().await); @@ -2046,9 +2306,11 @@ mod tests { .uri("/proxy/echo/test?foo=bar".to_string()) .header("host", "client.example") .header("x-api-key", "test-key") + .header(header::AUTHORIZATION, "Bearer application-token") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .header(ENVD_ACCESS_TOKEN_HEADER, "envd-token") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, "traffic-token") .body(Body::empty()) .unwrap(), ) @@ -2065,8 +2327,11 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); - assert_eq!(payload["envd_access_token"], "envd-token"); + assert!(payload["access_token"].is_null()); + assert_eq!(payload["traffic_token_header_seen"], false); assert_eq!(payload["forwarded_host"], "client.example"); + assert_eq!(payload["api_key_header_seen"], false); + assert_eq!(payload["authorization"], "Bearer application-token"); } #[tokio::test] @@ -2288,18 +2553,40 @@ mod tests { async fn sandbox_proxy_host_routes_control_paths_and_skips_explicit_proxy() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); - let app = proxy_app_for_sandbox_with_domains( + let (app, access_token) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) .await; + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/health") + .header( + "host", + format!( + "{}-{}.sandbox.example.invalid", + upstream_addr.port(), + sandbox_id + ), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = app .clone() .oneshot( Request::builder() .method(Method::GET) .uri("/health?foo=bar") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header( "host", format!( @@ -2332,6 +2619,7 @@ mod tests { upstream_addr.port(), sandbox_id )) + .header("x-api-key", "test-key") .body(Body::empty()) .unwrap(), ) @@ -2343,7 +2631,7 @@ mod tests { let payload: Value = serde_json::from_slice(&body).unwrap(); assert_eq!(payload["path"], "/authority"); - let app = proxy_app_for_sandbox_with_domains( + let (app, _) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) @@ -2354,6 +2642,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") + .header("x-api-key", "test-key") .header( "host", format!( @@ -2375,6 +2664,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") + .header("x-api-key", "test-key") .header( "host", format!( @@ -2401,14 +2691,14 @@ mod tests { async fn proxy_accepts_e2b_compatible_headers() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); - let app = proxy_app_for_sandbox(&sandbox_id).await; + let (app, access_token) = proxy_app_with_access_token_for_sandbox(&sandbox_id).await; let response = app .oneshot( Request::builder() .method(Method::GET) .uri("/proxy/e2b/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header(E2B_SANDBOX_ID_HEADER, sandbox_id.to_string()) .header(E2B_TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) @@ -2426,6 +2716,10 @@ mod tests { assert_eq!(payload["e2b_sandbox_header_seen"], false); assert_eq!(payload["target_port_header_seen"], false); assert_eq!(payload["e2b_target_port_header_seen"], false); + assert!( + payload["access_token"].is_null(), + "envd credential leaked to application port" + ); } #[tokio::test] diff --git a/src/api/server.rs b/src/api/server.rs index 2a738898..0aed4b7c 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -1,6 +1,6 @@ use axum::{middleware, routing::get, Router}; -use super::{proxy, ApiImpl}; +use super::{impls::auth, proxy, ApiImpl}; use crate::observability::prometheus; use agentenv_http_server::apis; use agentenv_observability::metrics_handler; @@ -28,8 +28,12 @@ where .merge(proxy::router(api_impl.clone())) .route("/metrics", get(metrics_handler)) .layer(middleware::from_fn_with_state( - api_impl, + api_impl.clone(), proxy::sandbox_proxy_classifier::, )) + .layer(middleware::from_fn_with_state( + api_impl, + auth::require_auth::, + )) .layer(middleware::from_fn(prometheus::http_metrics_middleware)) } diff --git a/src/api_key.rs b/src/api_key.rs new file mode 100644 index 00000000..ec59037e --- /dev/null +++ b/src/api_key.rs @@ -0,0 +1,195 @@ +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use rand::{rngs::SysRng, TryRng}; +use tracing::info; + +use crate::cfg::AppConfig; + +const API_KEY_ENV: &str = "AENV_API_KEY"; +const EXTERNAL_API_KEY_PATH: &str = "/run/secrets/api-key"; +const MANAGED_API_KEY_RELATIVE_PATH: &str = "secrets/api-key"; +const GENERATED_API_KEY_PREFIX: &str = "e2b_"; + +pub fn resolve(config: &AppConfig) -> Result { + resolve_from( + std::env::var_os(API_KEY_ENV).as_deref(), + Path::new(EXTERNAL_API_KEY_PATH), + &config.home_path, + ) +} + +fn resolve_from( + explicit: Option<&OsStr>, + external_path: &Path, + home_path: &Path, +) -> Result { + if let Some(explicit) = explicit { + return validate( + explicit + .to_str() + .context("AENV_API_KEY must contain valid UTF-8")?, + ) + .context("invalid AENV_API_KEY"); + } + + match read(external_path) { + Ok(key) => { + info!(path = %external_path.display(), "loaded API key from external secret"); + return Ok(key); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("load external API key"), + } + + let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); + match read(&managed_path) { + Ok(key) => return Ok(key), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("load managed API key"), + } + + create(&managed_path) +} + +fn read(path: &Path) -> Result { + let value = fs::read_to_string(path)?; + validate(&value).map_err(io::Error::other) +} + +fn validate(value: &str) -> Result { + let value = value.trim(); + if value.len() < 32 + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-')) + { + bail!("API key must contain at least 32 URL-safe characters"); + } + Ok(value.to_owned()) +} + +fn create(path: &Path) -> Result { + let parent = path + .parent() + .context("managed API key path has no parent")?; + fs::create_dir_all(parent) + .with_context(|| format!("create managed secret directory {}", parent.display()))?; + set_permissions(parent, 0o700)?; + + let mut random = [0_u8; 32]; + SysRng + .try_fill_bytes(&mut random) + .context("generate managed API key")?; + let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); + + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temporary API key in {}", parent.display()))?; + set_permissions(temporary.path(), 0o600)?; + writeln!(temporary, "{key}")?; + temporary.as_file().sync_all()?; + + match temporary.persist_noclobber(path) { + Ok(_) => { + File::open(parent)?.sync_all()?; + info!(path = %path.display(), "generated managed API key"); + Ok(key) + } + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { + read(path).context("load concurrently generated API key") + } + Err(error) => { + Err(error.error).with_context(|| format!("persist managed API key {}", path.display())) + } + } +} + +#[cfg(unix)] +fn set_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Barrier}; + + use tempfile::TempDir; + + const TEST_KEY: &str = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + #[test] + fn configured_sources_take_precedence() -> Result<()> { + let temp = TempDir::new()?; + let external_path = temp.path().join("external"); + fs::write(&external_path, format!("{TEST_KEY}\n"))?; + + assert_eq!( + resolve_from(Some(OsStr::new(TEST_KEY)), &external_path, temp.path())?, + TEST_KEY + ); + assert_eq!(resolve_from(None, &external_path, temp.path())?, TEST_KEY); + assert!(!temp.path().join(MANAGED_API_KEY_RELATIVE_PATH).exists()); + Ok(()) + } + + #[test] + fn managed_key_is_private_and_stable() -> Result<()> { + let temp = TempDir::new()?; + let missing_external = temp.path().join("missing"); + let first = resolve_from(None, &missing_external, temp.path())?; + + assert_eq!(resolve_from(None, &missing_external, temp.path())?, first); + assert!(first.starts_with(GENERATED_API_KEY_PREFIX)); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let path = temp.path().join(MANAGED_API_KEY_RELATIVE_PATH); + assert_eq!( + fs::metadata(path.parent().unwrap())?.permissions().mode() & 0o777, + 0o700 + ); + assert_eq!(fs::metadata(path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn concurrent_creation_converges() -> Result<()> { + const THREADS: usize = 8; + let temp = TempDir::new()?; + let home_path = Arc::new(temp.path().to_owned()); + let external_path = Arc::new(temp.path().join("missing")); + let barrier = Arc::new(Barrier::new(THREADS)); + let handles = (0..THREADS) + .map(|_| { + let home_path = Arc::clone(&home_path); + let external_path = Arc::clone(&external_path); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + resolve_from(None, &external_path, &home_path) + }) + }) + .collect::>(); + + let keys = handles + .into_iter() + .map(|handle| handle.join().expect("API key creation thread panicked")) + .collect::>>()?; + assert!(keys.iter().all(|key| key == &keys[0])); + Ok(()) + } +} diff --git a/src/bin/server.rs b/src/bin/server.rs index 479d038c..f755bad9 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -76,6 +76,8 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } + let api_key = agentenv::api_key::resolve(config)?; + agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; @@ -147,6 +149,7 @@ async fn main() -> anyhow::Result<()> { image_resolver, observability, config.sandbox_proxy.domains.clone(), + api_key, )); let app = server::new(api_impl); let shutdown_orchestrator = Arc::clone(&orchestrator); diff --git a/src/lib.rs b/src/lib.rs index 5968c26e..14d02f7e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod api; +pub mod api_key; pub mod cfg; mod digest; pub mod identity; diff --git a/src/sandbox/backend.rs b/src/sandbox/backend.rs index 26702917..575e3a62 100644 --- a/src/sandbox/backend.rs +++ b/src/sandbox/backend.rs @@ -35,6 +35,10 @@ pub trait PausedSandboxState: Any + fmt::Debug + Send + Sync + 'static { /// The orchestrator only carries this value to the image-liveness layer; it /// does not interpret the backend-specific artifact identities inside it. fn runtime_artifacts(&self) -> RuntimeArtifactSet; + /// Effective envd control-plane port persisted with the paused runtime, when available. + fn control_plane_port(&self) -> Option { + None + } } impl dyn PausedSandboxState { diff --git a/src/sandbox/firecracker/sandbox.rs b/src/sandbox/firecracker/sandbox.rs index a8f03b5f..c725c2b5 100644 --- a/src/sandbox/firecracker/sandbox.rs +++ b/src/sandbox/firecracker/sandbox.rs @@ -229,6 +229,10 @@ impl FirecrackerPausedState { } impl PausedSandboxState for FirecrackerPausedState { + fn control_plane_port(&self) -> Option { + Some(self.snapshot_config.common.control_plane_port) + } + fn encode(&self) -> Result { serde_json::to_value(&self.snapshot_config).context("serialize Firecracker paused state") } From 636dc52e8b669b691c168c5d2137ced79774ec14 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:29:56 +0000 Subject: [PATCH 2/6] docs: document API key setup in README --- README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4c649223..0c162746 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ If your server does not support standard KVM, see the [PVM deployment guide](htt ## ⚡ Quick Start (Single Node) > [!WARNING] -> **AgentENV currently does not support authorization.** Do not expose the AgentENV -> API to the public network. Run it only on a trusted network or behind an -> authorization proxy with appropriate network controls. +> AgentENV authenticates API requests but does not encrypt traffic. Do not send +> the API key over an untrusted plaintext network. Run AgentENV on a trusted +> network or terminate HTTPS at a reverse proxy or load balancer. **1. Install and start the server** @@ -66,7 +66,7 @@ Set up the server: ```bash curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash docker pull ghcr.io/kvcache-ai/aenv-server:latest -docker run -d --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-server:latest +docker run -d --name aenv-server --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-server:latest ``` The server is accessible at `http://127.0.0.1:8000` by default. @@ -83,10 +83,23 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in **3. Authenticate** +The server generates an API key on its first startup. Retrieve it for the +installation method used in step 1: + +```bash +# Native install +sudo cat /var/lib/aenv/secrets/api-key + +# Docker +docker exec aenv-server cat /workspace/env/secrets/api-key +``` + +Then run `aenv auth` and paste that key: + ```bash aenv auth # AENV server URL [http://localhost:8000]: http://127.0.0.1:8000 -# API key: dummy +# API key: ``` **4. Pull a template and run a sandbox** From ab10d199fbeb52ff72e3fd38ad89977d26e4ac0a Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:40:00 +0000 Subject: [PATCH 3/6] fix: derive sandbox access tokens from shared seed --- config/default.toml | 2 +- deploy/docker-compose.yml | 2 +- deploy/k8s/base/agentenv-daemonset.yaml | 5 +- deploy/k8s/base/gateway-deployment.yaml | 5 + deploy/k8s/base/kustomization.yaml | 1 + .../k8s/overlays/local-dev/kustomization.yaml | 8 -- deploy/k8s/run.sh | 67 +++++++--- docs/src/configuration/authentication.md | 27 ++-- docs/src/configuration/env-vars.md | 2 +- docs/src/configuration/reference.md | 4 +- docs/src/deployment/docker-compose.md | 17 +-- docs/src/deployment/kubernetes.md | 17 ++- docs/src/deployment/static-multi-node.md | 17 +-- .../persistence-artifact-inventory.md | 4 +- docs/src/security/secure-sandboxes.md | 30 ++--- services/README.md | 6 +- services/gateway/cmd/main.go | 46 ++++++- services/gateway/cmd/main_test.go | 27 ++++ services/gateway/internal/server.go | 23 ++-- services/gateway/internal/server_test.go | 35 +++-- src/api/impls/auth.rs | 49 +++---- src/api/impls/sandbox.rs | 5 +- src/api/proxy.rs | 19 ++- src/orchestrator/service.rs | 12 +- src/sandbox/access.rs | 123 +++++++++++++----- 25 files changed, 349 insertions(+), 204 deletions(-) diff --git a/config/default.toml b/config/default.toml index 00f94bf7..2ec626f9 100644 --- a/config/default.toml +++ b/config/default.toml @@ -188,7 +188,7 @@ init_timeout_secs = 60 poll_ms = 3 [sandbox] -# Optional secret used to derive per-sandbox envd access tokens. When unset, +# Optional secret used to derive per-sandbox envd and traffic access tokens. When unset, # AgentENV creates a node-local seed under $AENV_HOME/secrets. Configure the # same explicit value on every node when cross-node sandbox recovery is required. # access_token_hash_seed = "replace-with-a-secret" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 87cff923..fccc7127 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -16,7 +16,7 @@ x-agentenv-base: &agentenv-base - /dev:/dev - ${CONFIG_PATH:-../config/default.toml}:/workspace/config/default.toml:ro # Runtime assets are baked into the image by `server --setup-only`; compose - # persists committed snapshots and the deployment API key across restarts. + # persists committed snapshots and deployment secrets across restarts. - agentenv-snapshot-store:/workspace/env/snapshot-store - agentenv-auth:/workspace/env/secrets devices: diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index 470b2a9c..e95fefb9 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -31,9 +31,8 @@ spec: - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED valueFrom: secretKeyRef: - name: agentenv-runtime-secrets - key: sandbox-access-token-hash-seed - optional: true + name: agentenv-auth + key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - name: AENV_VIRTUALIZATION_MODE value: "kvm" - name: API_ADDR diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 08536fc0..7f7f8644 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -32,6 +32,11 @@ spec: secretKeyRef: name: agentenv-auth key: AENV_API_KEY + - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED + valueFrom: + secretKeyRef: + name: agentenv-auth + key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 556f0d70..157d8d62 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -34,6 +34,7 @@ secretGenerator: - name: agentenv-auth literals: - AENV_API_KEY= + - AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= images: - name: agentenv-gateway diff --git a/deploy/k8s/overlays/local-dev/kustomization.yaml b/deploy/k8s/overlays/local-dev/kustomization.yaml index 2a71de5e..ab182ed5 100644 --- a/deploy/k8s/overlays/local-dev/kustomization.yaml +++ b/deploy/k8s/overlays/local-dev/kustomization.yaml @@ -6,14 +6,6 @@ namespace: agentenv-system resources: - ../../base -secretGenerator: - - name: agentenv-runtime-secrets - literals: - - sandbox-access-token-hash-seed=agentenv-local-dev-access-token-hash-seed - -generatorOptions: - disableNameSuffixHash: true - patches: - target: kind: DaemonSet diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 4d39680b..49f6aec9 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -29,29 +29,59 @@ sed_in_place() { cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" + +namespace_name="" +if [[ "${MODE}" == "apply" ]]; then + if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then + echo "failed to check namespace ${NAMESPACE}" >&2 + exit 1 + fi +fi + +read_existing_secret() { + local secret="$1" + local key="$2" + local encoded_value="" + + if [[ -z "${namespace_name}" ]]; then + return 0 + fi + if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret "${secret}" \ + --ignore-not-found -o "go-template={{index .data \"${key}\"}}")"; then + echo "failed to read ${key} from Secret ${NAMESPACE}/${secret}" >&2 + return 1 + fi + if [[ -n "${encoded_value}" ]]; then + printf '%s' "${encoded_value}" | base64 -d + fi +} + if [[ "${MODE}" != "delete" ]]; then API_KEY_VALUE="" if [[ "${AENV_API_KEY+x}" == "x" ]]; then API_KEY_VALUE="${AENV_API_KEY}" - elif [[ "${MODE}" == "apply" ]]; then - encoded_key="" - if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then - echo "failed to check namespace ${NAMESPACE}" >&2 + elif ! API_KEY_VALUE="$(read_existing_secret agentenv-auth AENV_API_KEY)"; then + exit 1 + fi + + ACCESS_TOKEN_SEED_VALUE="" + if [[ "${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED+x}" == "x" ]]; then + ACCESS_TOKEN_SEED_VALUE="${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" + elif ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-auth AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED)"; then + exit 1 + fi + if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then + if ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-runtime-secrets sandbox-access-token-hash-seed)"; then exit 1 fi - if [[ -n "${namespace_name}" ]]; then - if ! encoded_key="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ - --ignore-not-found -o jsonpath='{.data.AENV_API_KEY}')"; then - echo "failed to read existing Secret ${NAMESPACE}/agentenv-auth" >&2 - exit 1 - fi - fi - if [[ -n "${encoded_key}" ]]; then - if ! API_KEY_VALUE="$(printf '%s' "${encoded_key}" | base64 -d)"; then - echo "failed to decode the existing agentenv-auth Secret" >&2 - exit 1 - fi - fi + fi + + if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then + ACCESS_TOKEN_SEED_VALUE="$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" + fi + if [[ ! "${ACCESS_TOKEN_SEED_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then + echo "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED must contain at least 32 URL-safe characters" >&2 + exit 1 fi if [[ -z "${API_KEY_VALUE}" ]]; then @@ -65,6 +95,9 @@ if [[ "${MODE}" != "delete" ]]; then sed_in_place \ "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ "${TEMP_DIR}/k8s/base/kustomization.yaml" + sed_in_place \ + "s#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=.*#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=${ACCESS_TOKEN_SEED_VALUE}#" \ + "${TEMP_DIR}/k8s/base/kustomization.yaml" fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index b57dac1e..c04773f7 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -23,6 +23,9 @@ For secure sandboxes, `envdAccessToken` is a separate credential for envd control traffic and must be sent as `X-Access-Token` only when targeting the envd control-plane port. It is absent for insecure sandboxes. +Both sandbox credentials are derived from the sandbox ID and one independent +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED`. They are not derived from the API key. + ## Key Resolution On normal startup, a runtime node uses the first available source: @@ -36,8 +39,10 @@ generates a 256-bit key and atomically stores it in the managed path with `0600` permissions. It reuses that key on later starts. Dependency and host setup modes do not create a key. -The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a -key because every gateway and runtime node in a cluster must share one. +The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`. It also reads the +sandbox seed from `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` or +`/run/secrets/sandbox-access-token-hash-seed`. The gateway never generates +either value because it must share them with every runtime node. ## Installation Methods @@ -64,7 +69,8 @@ container replacements. The checked-in Compose deployment mounts one named volume read-write on both runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node -startup is safe: atomic creation makes both nodes converge on the same key. +startup is safe: atomic creation makes both nodes converge on the same key and +sandbox seed. Read it with: ```bash @@ -75,8 +81,8 @@ docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ `docker compose down` preserves the key. `docker compose down -v` removes the auth volume, so the next startup generates a new key. -`make k8s-apply` creates `Secret/agentenv-auth` on the first apply and reuses -the existing key on later applies. Read it with: +`make k8s-apply` creates `Secret/agentenv-auth` with an API key and sandbox +seed on the first apply, then reuses both values. Read the API key with: ```bash kubectl -n agentenv-system get secret agentenv-auth \ @@ -113,9 +119,8 @@ network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. ## Rotation -Set a new `AENV_API_KEY` on the gateway and every runtime node, or replace the -shared secret file, then restart them. Existing clients must switch to the new -value. Previously issued -`trafficAccessToken` values stop working when the key changes. -`envdAccessToken` values are unaffected and rotate only when the optional envd -seed changes. +Changing `AENV_API_KEY` invalidates existing client API credentials without +changing sandbox credentials. Changing +`AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both `trafficAccessToken` and +`envdAccessToken` values. Apply either change to the gateway and every runtime +node together, then restart them. diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index fc70ccaa..bb458564 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -24,7 +24,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional override for the secret used to derive secure sandbox envd access tokens. Configure the same value on every node when cross-node recovery of the same sandbox ID is required; otherwise each node uses its own managed seed. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | `/run/secrets/sandbox-access-token-hash-seed`, then auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Clustered deployments must configure the same value on the gateway and every runtime node. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index d2ad8049..a81591d6 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -257,11 +257,11 @@ Sandbox control communication settings. | Key | Type | Default | Description | |-----|------|---------|-------------| -| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive secure sandbox envd access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value when the deployment needs to recover the same sandbox ID on another node. | +| `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. -Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every node when cross-node recovery of the same sandbox is required. Nodes use their own managed seed when it is unset. +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on the gateway and every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 0d03444f..b59b1fcf 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -32,11 +32,6 @@ git clone https://github.com/kvcache-ai/AgentENV.git cd AgentENV ``` -## Configure the Access-Token Seed (Optional) - -See [Secure Sandboxes](../security/secure-sandboxes.md) -if the deployment needs future cross-node sandbox recovery. - ## Start the Cluster ```bash @@ -44,10 +39,11 @@ sudo bash scripts/docker-setup.sh make deploy-up ``` -On first startup, the runtime nodes atomically generate one API key in the +On first startup, the runtime nodes atomically generate one API key and sandbox +access-token seed in the shared `agentenv-auth` volume. The gateway mounts that volume read-only at -`/run/secrets`, so all three services use the same key. Normal -`make deploy-down` calls preserve the volume and key. +`/run/secrets`, so all three services use the same secrets. Normal +`make deploy-down` calls preserve the volume and both values. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: @@ -84,8 +80,9 @@ make deploy-logs # Stream logs from all services make deploy-down # Tear down the cluster ``` -Removing Compose volumes with `docker compose down -v` also removes the API -key. The next startup generates a new key and existing clients must be updated. +Removing Compose volumes with `docker compose down -v` also removes both +secrets. The next startup generates new values, so existing clients and sandbox +access tokens are invalidated. To provide an existing key through Docker Compose secrets, add a file-backed secret in an override file and mount it with `target: api-key` on the gateway diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 12d0097d..7f080595 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -60,18 +60,18 @@ make k8s-render make k8s-apply ``` -`make k8s-apply` generates a 256-bit API key on the first deployment and -stores it in `Secret/agentenv-auth`. Later applies reuse that key. Read it -locally when configuring clients: +`make k8s-apply` generates a 256-bit API key and sandbox access-token seed on +the first deployment and stores both in `Secret/agentenv-auth`. Later applies +reuse both values. Read the API key locally when configuring clients: ```bash kubectl -n agentenv-system get secret agentenv-auth \ -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' ``` -Set `AENV_API_KEY` when applying to supply your own key instead. A standalone -`make k8s-render` uses a temporary generated value because it does not modify -or read cluster state. +Set `AENV_API_KEY` and `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` when applying to +supply your own values. A standalone `make k8s-render` uses temporary generated +values because it does not modify or read cluster state. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: @@ -113,9 +113,8 @@ make k8s-delete A dedicated `local-dev` overlay mounts the repository's `env/` directory directly into the DaemonSet at `/workspace/env`, avoiding runtime asset copies: -This overlay also generates `agentenv-runtime-secrets` with a fixed test-only -seed so local and E2E deployments do not require production secret management. -Do not reuse that value outside local development. +The apply helper provisions the same generated `agentenv-auth` secrets used by +the default overlay. ```bash make k8s-build diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 875d95b2..1ca6821c 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -54,11 +54,13 @@ an external metrics collector needs them. ## 1. Install the runtime nodes -Generate one API key, deliver it through your normal secret-management channel, -and use the same value on every runtime node and the Gateway: +Generate one API key and one sandbox access-token seed, deliver them through +your normal secret-management channel, and use the same values on every runtime +node and the Gateway: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" ``` Run the installation on each runtime node: @@ -68,11 +70,9 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in ``` Edit `/etc/default/aenv` on each machine without removing the paths written by -the installer, and add `AENV_API_KEY=` before starting the -services. A multi-node deployment must not let each node generate an -independent managed key. - -See [Secure Sandboxes](../security/secure-sandboxes.md) if the deployment needs future cross-node sandbox recovery. +the installer, and add both shared values before starting the services. A +multi-node deployment must not let each node generate independent managed +secrets. Node A uses: @@ -116,7 +116,7 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` -Create `/etc/agentenv/auth.env` with the same key used on the runtime nodes: +Create `/etc/agentenv/auth.env` with the same values used on the runtime nodes: ```bash sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env @@ -125,6 +125,7 @@ sudoedit /etc/agentenv/auth.env ```text AENV_API_KEY= +AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= ``` If the `agentenv-control` account already exists, the `useradd` command reports diff --git a/docs/src/internals/persistence-artifact-inventory.md b/docs/src/internals/persistence-artifact-inventory.md index eefdb882..8cf25904 100644 --- a/docs/src/internals/persistence-artifact-inventory.md +++ b/docs/src/internals/persistence-artifact-inventory.md @@ -9,7 +9,7 @@ This document lists AgentENV artifacts that can remain on disk or in object stor | `home_path` | `/var/lib/aenv` | `src/cfg.rs` | Base for paths containing the literal `$AENV_HOME` placeholder. `AENV_HOME_PATH` overrides it before placeholder expansion. | | `runtime_path` | `/run/aenv` | `src/cfg.rs`, `src/sandbox/network/*` | Base for transient namespace mount points and daemon sockets. `AENV_RUNTIME_PATH` overrides it. | | `deps_path` | `$AENV_HOME/deps` | `src/cfg.rs`, `src/setup/*` | Base for downloaded runtime dependencies. `AENV_DEPS_PATH` can place these rebuildable assets outside `home_path`. | -| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure sandboxes. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted sandboxes. | | Firecracker sandbox work dirs | `$AENV_HOME/firecracker-work` with `agentenv-fc-` children | `src/sandbox/firecracker/*` | Per-sandbox runtime directories for sockets, symlinks, ublk runtime dirs, local logs, and writable OverlayBD upper layer data (`overlaybd/upper.data`, `overlaybd/upper.index`). An explicit `[firecracker].work_dir` overrides the root. | | `firecracker.serial_dir` | `$AENV_HOME/logs/serial` | `src/sandbox/firecracker/*` | Durable Firecracker stdout/stderr root, grouped by sandbox ID. An explicit `[firecracker].serial_dir` overrides the root. | | `managed_snapshot_root` | `/managed-snapshots` | `src/sandbox/firecracker/*` | In-process live snapshot artifact root used to keep captured snapshots alive until publish or drop. | @@ -37,7 +37,7 @@ Owned by `src/setup/*` and `src/cfg.rs`. | Overlaybd package downloads | `/overlaybd/downloads/*` | Temporary downloaded package archives | Setup staging for overlaybd release packages | Removed after a successful install. | | Generated overlaybd config | `$AENV_HOME/overlaybd/overlaybd-global.json`, `$AENV_HOME/overlaybd/mem-overlaybd-global.json`, `$AENV_HOME/overlaybd/convert-overlaybd-global.json`, `$AENV_HOME/overlaybd/resize-overlaybd-global.json` | Runtime global config, cache path, credentials config | Configures overlaybd runtime, memory snapshot overlaybd access, and the offline C++ tools (`overlaybd-apply`, `overlaybd-resize`), which get dedicated configs with isolated cacheDirs (`convert-blocks`, `resize-blocks`) and download disabled | Rewritten during setup/startup. | | Overlaybd runtime log | `$AENV_HOME/overlaybd/overlaybd.log` | Overlaybd runtime logs | Debugging | Appended by overlaybd runtime; no automatic GC. | -| Managed envd access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure sandboxes are persisted. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while sandboxes are persisted. | ## Firecracker Sandbox diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index b04948df..2eadf4b6 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -17,44 +17,34 @@ The API and SDKs return the sandbox's `envdAccessToken` where appropriate and at ## Access-Token Seed -A seed is a random value used to derive the access token for each sandbox. This seed is optional. When it is unset, each runtime node automatically creates and persists a node-local seed under `$AENV_HOME/secrets`. +A seed is a random value used to derive each sandbox's envd and traffic access tokens. This seed is optional for a standalone runtime. When it is unset, the runtime automatically creates and persists a seed under `$AENV_HOME/secrets`. This is sufficient for normal single-node operation and does not require additional setup. -Configure the same explicit seed on every runtime node when the deployment needs to recover the same sandbox ID on another node in the future. Generate it once and store it in the deployment's secret manager: +Configure the same explicit seed on the gateway and every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: ```bash openssl rand -hex 32 ``` -Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on every runtime node. +Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on the gateway and every runtime node. For TOML configuration, use `[sandbox].access_token_hash_seed` instead. +Container deployments may mount it at +`/run/secrets/sandbox-access-token-hash-seed`. -Preserve the seed across upgrades; changing it rotates access tokens for existing secure sandboxes. +Preserve the seed across upgrades; changing it rotates both sandbox access tokens. ### Kubernetes -The runtime DaemonSet reads the optional `agentenv-runtime-secrets` Secret. To configure a shared seed for all runtime Pods, create it before applying the runtime manifests: +`make k8s-apply` generates and preserves the seed in `Secret/agentenv-auth`, then injects it into the gateway and runtime Pods. Set `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` before applying to supply your own value. -```bash -kubectl apply -f deploy/k8s/base/namespace.yaml - -AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" -kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ - --from-literal="sandbox-access-token-hash-seed=${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" \ - --dry-run=client -o yaml | kubectl apply -f - -unset AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED -``` - -Run this once for a new cluster and preserve the existing Secret during upgrades. An external secret manager may be used instead, provided it creates the same Secret name and key: +An external secret manager may provide the same Secret and key: ```yaml apiVersion: v1 kind: Secret metadata: - name: agentenv-runtime-secrets + name: agentenv-auth namespace: agentenv-system stringData: - sandbox-access-token-hash-seed: + AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED: ``` - -If the Secret is not created, the DaemonSet still starts and each runtime Pod uses its automatically managed node-local seed. diff --git a/services/README.md b/services/README.md index ff6743d6..9bb1d066 100644 --- a/services/README.md +++ b/services/README.md @@ -73,13 +73,15 @@ Start gateway with the same API key configured on every AgentENV runtime node: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" +export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. -The gateway and runtime nodes require the same API key. The gateway reads an -explicit `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate one. +The gateway and runtime nodes require the same API key and sandbox access-token +seed. The gateway reads explicit environment values or the corresponding files +under `/run/secrets`; it does not generate either secret. Application proxy requests may additionally use the sandbox response's `trafficAccessToken` in the `e2b-traffic-access-token` header. diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index 11a8255b..2471da2c 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -25,8 +25,10 @@ import ( ) const ( - apiKeyEnv = "AENV_API_KEY" - defaultAPIKeyPath = "/run/secrets/api-key" + apiKeyEnv = "AENV_API_KEY" + accessTokenSeedEnv = "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED" + defaultAPIKeyPath = "/run/secrets/api-key" + defaultAccessTokenSeedPath = "/run/secrets/sandbox-access-token-hash-seed" ) func newSchedulerConn(addr string) (*grpc.ClientConn, error) { @@ -41,18 +43,35 @@ func loadAPIKey() (string, error) { } func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { - if value, present := lookupEnv(apiKeyEnv); present { - return validateAPIKey(value, apiKeyEnv) + return loadSecretFrom(lookupEnv, apiKeyEnv, secretPath, validateAPIKey) +} + +func loadAccessTokenSeed() (string, error) { + return loadAccessTokenSeedFrom(os.LookupEnv, defaultAccessTokenSeedPath) +} + +func loadAccessTokenSeedFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { + return loadSecretFrom(lookupEnv, accessTokenSeedEnv, secretPath, validateAccessTokenSeed) +} + +func loadSecretFrom( + lookupEnv func(string) (string, bool), + envName string, + secretPath string, + validate func(string, string) (string, error), +) (string, error) { + if value, present := lookupEnv(envName); present { + return validate(value, envName) } contents, err := os.ReadFile(secretPath) if err != nil { if os.IsNotExist(err) { - return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + return "", fmt.Errorf("%s must be set or %s must exist", envName, secretPath) } - return "", fmt.Errorf("read API key secret %s: %w", secretPath, err) + return "", fmt.Errorf("read secret %s: %w", secretPath, err) } - return validateAPIKey(string(contents), secretPath) + return validate(string(contents), secretPath) } func validateAPIKey(value, source string) (string, error) { @@ -72,6 +91,14 @@ func validateAPIKey(value, source string) (string, error) { return value, nil } +func validateAccessTokenSeed(value, source string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", fmt.Errorf("sandbox access-token seed from %s must be non-empty", source) + } + return value, nil +} + func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -84,6 +111,10 @@ func main() { if err != nil { log.Fatalf("load API key failed: %v", err) } + accessTokenSeed, err := loadAccessTokenSeed() + if err != nil { + log.Fatalf("load sandbox access-token seed failed: %v", err) + } logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { @@ -113,6 +144,7 @@ func main() { RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, APIKey: apiKey, + SandboxAccessTokenSeed: accessTokenSeed, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index 9064986b..2fc94165 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -8,6 +8,7 @@ import ( ) const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +const testAccessTokenSeed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" func TestValidateAPIKey(t *testing.T) { t.Parallel() @@ -75,3 +76,29 @@ func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") } } + +func TestLoadAccessTokenSeed(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "sandbox-access-token-hash-seed") + if err := os.WriteFile(path, []byte(testAccessTokenSeed+"\n"), 0o444); err != nil { + t.Fatal(err) + } + got, err := loadAccessTokenSeedFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAccessTokenSeedFrom() error = %v", err) + } + if got != testAccessTokenSeed { + t.Fatalf("loadAccessTokenSeedFrom() = %q, want %q", got, testAccessTokenSeed) + } +} + +func TestLoadAccessTokenSeedRejectsExplicitEmptyEnvironment(t *testing.T) { + if _, err := loadAccessTokenSeedFrom( + func(name string) (string, bool) { return "", name == accessTokenSeedEnv }, + filepath.Join(t.TempDir(), "missing"), + ); err == nil { + t.Fatal("loadAccessTokenSeedFrom() unexpectedly accepted an empty environment value") + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 70cf02aa..739237d6 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -5,7 +5,7 @@ import ( "context" "crypto/hmac" "crypto/sha256" - "encoding/base64" + "encoding/hex" "encoding/json" "errors" "io" @@ -28,8 +28,7 @@ const ( headerAPIKey = "X-API-Key" headerTrafficToken = "e2b-traffic-access-token" headerEnvdAccessToken = "X-Access-Token" - trafficTokenPrefix = "aenv_trf_" - trafficTokenContext = "agentenv-sandbox-traffic-v1\x00" + trafficTokenPrefix = "sandbox-traffic" envdControlPlanePort = 49983 headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" @@ -51,6 +50,7 @@ const ( type ServerOptions struct { APIKey string + SandboxAccessTokenSeed string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -64,6 +64,7 @@ type Server struct { queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client apiKey []byte + accessTokenSeed []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -78,6 +79,10 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, if apiKey == "" { return nil, errors.New("API key is required") } + accessTokenSeed := strings.TrimSpace(options.SandboxAccessTokenSeed) + if accessTokenSeed == "" { + return nil, errors.New("sandbox access-token seed is required") + } sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { @@ -97,6 +102,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, apiKey: []byte(apiKey), + accessTokenSeed: []byte(accessTokenSeed), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -832,11 +838,10 @@ func singleHeaderMatches(headers http.Header, name string, expected []byte) bool return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) } -func trafficAccessToken(apiKey []byte, sandboxID string) string { - mac := hmac.New(sha256.New, apiKey) - _, _ = mac.Write([]byte(trafficTokenContext)) - _, _ = mac.Write([]byte(sandboxID)) - return trafficTokenPrefix + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +func trafficAccessToken(seed []byte, sandboxID string) string { + mac := hmac.New(sha256.New, seed) + _, _ = mac.Write([]byte(trafficTokenPrefix + "-" + sandboxID)) + return hex.EncodeToString(mac.Sum(nil)) } func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { @@ -895,7 +900,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) if !authorized && dataPlane { if sandboxID, ok := s.sandboxIDForDataPlaneAuth(r); ok { - expected := trafficAccessToken(s.apiKey, sandboxID) + expected := trafficAccessToken(s.accessTokenSeed, sandboxID) authorized = singleHeaderMatches(r.Header, headerTrafficToken, []byte(expected)) } } diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index fa8358d0..30c76fb8 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,7 +148,10 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } -const testAPIKey = "test-api-key" +const ( + testAPIKey = "test-api-key" + testAccessTokenSeed = "test-access-token-seed" +) type testServerOption func(*ServerOptions) @@ -156,9 +159,10 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti t.Helper() options := ServerOptions{ - RequestTimeout: timeout, - MaxResponseSize: maxRespSize, - APIKey: testAPIKey, + RequestTimeout: timeout, + MaxResponseSize: maxRespSize, + APIKey: testAPIKey, + SandboxAccessTokenSeed: testAccessTokenSeed, } for _, opt := range opts { opt(&options) @@ -189,6 +193,17 @@ func TestNewServerRejectsEmptyAPIKey(t *testing.T) { } } +func TestNewServerRejectsEmptySandboxAccessTokenSeed(t *testing.T) { + _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ + APIKey: testAPIKey, + RequestTimeout: time.Second, + MaxResponseSize: 1024, + }) + if err == nil { + t.Fatal("NewServer accepted an empty sandbox access-token seed") + } +} + func TestGatewayRequiresExactAPIKey(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) handler := server.Handler() @@ -262,7 +277,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { @@ -272,7 +287,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), "another-sandbox")) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), "another-sandbox")) recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { @@ -282,7 +297,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "8080") - req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAPIKey), sandboxID)) + req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { @@ -291,7 +306,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAPIKey), sandboxID)) + req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) @@ -302,8 +317,8 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { func TestTrafficAccessTokenVector(t *testing.T) { const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" - const want = "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" - if got := trafficAccessToken([]byte("test-key"), sandboxID); got != want { + const want = "f5457a589b09265b169392dd49506ec70458f685cf2ba7fc2c5b4763c42a5b17" + if got := trafficAccessToken([]byte("test-seed"), sandboxID); got != want { t.Fatalf("trafficAccessToken() = %q, want %q", got, want) } } diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 6d525724..9084f9f1 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -1,3 +1,4 @@ +use agentenv_http_server::apis; use async_trait::async_trait; use axum::{ body::Body, @@ -6,20 +7,13 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -use agentenv_http_server::apis; use super::{ApiImpl, Claims}; -use crate::api::proxy; +use crate::{api::proxy, types::SandboxId}; pub(crate) const API_KEY_HEADER: &str = "x-api-key"; pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; -const TRAFFIC_TOKEN_PREFIX: &str = "aenv_trf_"; -const TRAFFIC_TOKEN_CONTEXT: &[u8] = b"agentenv-sandbox-traffic-v1\0"; fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> bool { let mut values = headers.get_all(name).iter(); @@ -33,29 +27,26 @@ fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> boo value.as_bytes() == expected.as_bytes() } -fn derive_traffic_access_token(api_key: &[u8], sandbox_id: &str) -> String { - let mut mac = - Hmac::::new_from_slice(api_key).expect("HMAC accepts API keys of any length"); - mac.update(TRAFFIC_TOKEN_CONTEXT); - mac.update(sandbox_id.as_bytes()); - format!( - "{TRAFFIC_TOKEN_PREFIX}{}", - URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()) - ) -} - impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { single_header_matches(headers, API_KEY_HEADER, &self.api_key) } - pub(crate) fn traffic_access_token(&self, sandbox_id: &str) -> String { - derive_traffic_access_token(self.api_key.as_bytes(), sandbox_id) + pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { + self.orchestrator.traffic_access_token(sandbox_id) } - fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: &str) -> bool { - let expected = self.traffic_access_token(sandbox_id); - single_header_matches(headers, TRAFFIC_ACCESS_TOKEN_HEADER, &expected) + fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: SandboxId) -> bool { + let mut values = headers.get_all(TRAFFIC_ACCESS_TOKEN_HEADER).iter(); + let Some(candidate) = values.next().and_then(|value| value.to_str().ok()) else { + return false; + }; + if values.next().is_some() { + return false; + } + + self.orchestrator + .validate_traffic_access_token(sandbox_id, candidate) } } @@ -80,7 +71,7 @@ where .is_some_and(|sandbox_id| { api_impl .as_ref() - .has_valid_traffic_access_token(request.headers(), &sandbox_id) + .has_valid_traffic_access_token(request.headers(), sandbox_id) }); } if !authorized && proxy_request { @@ -165,12 +156,4 @@ mod tests { "correct-key" )); } - - #[test] - fn traffic_access_token_matches_gateway_contract() { - assert_eq!( - derive_traffic_access_token(b"test-key", "0191f4d0-7b2a-7c11-9c2d-0123456789ab"), - "aenv_trf_PwHqhTxLa_mzUCNIGx03uiTHxZ3k995pKDOS50PaGWo" - ); - } } diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 85b91f84..48648dc8 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -214,15 +214,14 @@ impl From for models::SandboxDetail { impl ApiImpl { fn sandbox_model(&self, metadata: SandboxMetadata) -> models::Sandbox { + let traffic_access_token = self.traffic_access_token(metadata.id); let envd_access_token = self .orchestrator .get_envd_access_token(&metadata) .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; - sandbox.traffic_access_token = Some(Nullable::Present( - self.traffic_access_token(&sandbox.sandbox_id), - )); + sandbox.traffic_access_token = Some(Nullable::Present(traffic_access_token)); sandbox.domain = self .sandbox_proxy_domains() .first() diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 1cfcb015..df8d7b19 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -161,12 +161,11 @@ fn proxy_route_for_auth(request: &Request, domains: &[String]) -> Option Option { - Some( - proxy_route_for_auth(request, domains)? - .sandbox_id - .to_string(), - ) +pub(crate) fn sandbox_id_for_proxy_auth( + request: &Request, + domains: &[String], +) -> Option { + Some(proxy_route_for_auth(request, domains)?.sandbox_id) } pub(crate) fn envd_access_token_for_proxy_auth( @@ -1780,7 +1779,7 @@ mod tests { sandbox_id: &SandboxId, ) -> (axum::Router, String) { let api = build_api().await; - let access_token = api.traffic_access_token(&sandbox_id.to_string()); + let access_token = api.traffic_access_token(*sandbox_id); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -1796,7 +1795,7 @@ mod tests { domains: Vec, ) -> (axum::Router, String) { let api = build_api_with_sandbox_proxy_domains(domains).await; - let access_token = api.traffic_access_token(&sandbox_id.to_string()); + let access_token = api.traffic_access_token(*sandbox_id); api.orchestrator() .set_proxy_target_for_test( *sandbox_id, @@ -2006,7 +2005,7 @@ mod tests { async fn traffic_token_cannot_authenticate_control_plane() { let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let traffic_token = api.traffic_access_token(sandbox_id); let app = server::new(api); let response = app .oneshot( @@ -2029,7 +2028,7 @@ mod tests { async fn envd_token_cannot_authenticate_application_proxy() { let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(&sandbox_id.to_string()); + let traffic_token = api.traffic_access_token(sandbox_id); let response = server::new(api) .oneshot( Request::builder() diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index 079362ed..9f4a0e65 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -162,12 +162,12 @@ where // Restore persisted sandboxes from the previous run, keeping the paused // ones (with their state) for the paused-protection reconcile below. let persisted = persister.load_all(&factory).await?; - let managed_seed_must_exist = persisted.iter().any(|metadata| metadata.secure); + let managed_seed_must_exist = !persisted.is_empty(); let access_tokens = tokio::task::spawn_blocking(move || { SandboxAccessTokenGenerator::load_or_create(app_config, managed_seed_must_exist) }) .await - .context("join envd access-token seed loader")??; + .context("join sandbox access-token seed loader")??; let restored_paused: Vec<(SandboxId, Arc)> = persisted .iter() .filter(|metadata| metadata.state == SandboxState::Paused) @@ -741,6 +741,14 @@ where self.access_tokens.matches(sandbox_id, candidate) } + pub fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { + self.access_tokens.generate_traffic(sandbox_id) + } + + pub fn validate_traffic_access_token(&self, sandbox_id: SandboxId, candidate: &str) -> bool { + self.access_tokens.matches_traffic(sandbox_id, candidate) + } + /// Resolves the current proxyability of a sandbox without touching the sandbox mutex. #[tracing::instrument(skip(self), fields(sandbox_id = %sandbox_id))] pub async fn proxy_lookup_for(&self, sandbox_id: &SandboxId) -> Result { diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index 3a620430..f3c88dc6 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -15,9 +15,11 @@ use crate::types::SandboxId; type HmacSha256 = Hmac; const MANAGED_SEED_RELATIVE_PATH: &str = "secrets/sandbox-access-token-hash-seed"; +const EXTERNAL_SEED_PATH: &str = "/run/secrets/sandbox-access-token-hash-seed"; const MANAGED_SEED_BYTES: usize = 32; const SEED_HEX_LEN: usize = MANAGED_SEED_BYTES * 2; const MANAGED_SEED_FILE_MAX_LEN: usize = SEED_HEX_LEN + 1; +const TRAFFIC_ACCESS_TOKEN_PREFIX: &str = "sandbox-traffic"; #[derive(Clone, PartialEq, Eq)] pub struct EnvdAccessToken(String); @@ -55,6 +57,24 @@ impl SandboxAccessTokenGenerator { return Self::new(seed); } + match fs::read_to_string(EXTERNAL_SEED_PATH) { + Ok(seed) => { + let generator = + Self::new(&seed).context("invalid external sandbox access-token seed")?; + info!( + path = EXTERNAL_SEED_PATH, + "loaded sandbox access-token seed from external secret" + ); + return Ok(generator); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("read external sandbox access-token seed {EXTERNAL_SEED_PATH}") + }); + } + } + let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; @@ -63,7 +83,7 @@ impl SandboxAccessTokenGenerator { { warn!( path = %managed_seed_path.display(), - "using a node-local managed envd access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node before enabling cross-node sandbox recovery" + "using a node-local managed sandbox access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node in a clustered deployment" ); } @@ -71,18 +91,37 @@ impl SandboxAccessTokenGenerator { } pub fn generate(&self, subject: SandboxId) -> EnvdAccessToken { + EnvdAccessToken(self.generate_for(subject.to_string().as_bytes())) + } + + pub fn generate_traffic(&self, subject: SandboxId) -> String { + self.generate_for(format!("{TRAFFIC_ACCESS_TOKEN_PREFIX}-{subject}").as_bytes()) + } + + fn generate_for(&self, subject: &[u8]) -> String { let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); - mac.update(subject.to_string().as_bytes()); - EnvdAccessToken(hex::encode(mac.finalize().into_bytes())) + mac.update(subject); + hex::encode(mac.finalize().into_bytes()) } pub fn matches(&self, subject: SandboxId, candidate: &str) -> bool { + self.matches_for(subject.to_string().as_bytes(), candidate) + } + + pub fn matches_traffic(&self, subject: SandboxId, candidate: &str) -> bool { + self.matches_for( + format!("{TRAFFIC_ACCESS_TOKEN_PREFIX}-{subject}").as_bytes(), + candidate, + ) + } + + fn matches_for(&self, subject: &[u8], candidate: &str) -> bool { let mut candidate_bytes = [0_u8; 32]; let decoded = hex::decode_to_slice(candidate, &mut candidate_bytes).is_ok(); let mut mac = HmacSha256::new_from_slice(&self.seed).expect("HMAC accepts keys of any length"); - mac.update(subject.to_string().as_bytes()); + mac.update(subject); mac.verify_slice(&candidate_bytes).is_ok() & decoded } } @@ -98,7 +137,7 @@ fn validate_explicit_seed(seed: &str) -> Result<&str> { fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { let parent = managed_path .parent() - .context("managed envd access-token seed path has no parent")?; + .context("managed sandbox access-token seed path has no parent")?; match validate_managed_seed_directory(parent) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::NotFound => {} @@ -115,7 +154,7 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { return Err(error).with_context(|| { format!( - "open managed envd access-token seed {}", + "open managed sandbox access-token seed {}", managed_path.display() ) }); @@ -124,7 +163,7 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result io::Result { } fn validate_managed_seed_file(path: &Path, file: &File) -> Result { - let metadata = file - .metadata() - .with_context(|| format!("inspect managed envd access-token seed {}", path.display()))?; + let metadata = file.metadata().with_context(|| { + format!( + "inspect managed sandbox access-token seed {}", + path.display() + ) + })?; if !metadata.is_file() { bail!( - "managed envd access-token seed {} must be a regular file", + "managed sandbox access-token seed {} must be a regular file", path.display() ); } @@ -164,14 +206,14 @@ fn validate_managed_seed_file(path: &Path, file: &File) -> Result let mode = metadata.permissions().mode() & 0o777; if mode != 0o600 { bail!( - "managed envd access-token seed {} must have permissions 0600, found {mode:04o}", + "managed sandbox access-token seed {} must have permissions 0600, found {mode:04o}", path.display() ); } let expected_uid = nix::unistd::Uid::effective().as_raw(); if metadata.uid() != expected_uid { bail!( - "managed envd access-token seed {} must be owned by uid {expected_uid}, found uid {}", + "managed sandbox access-token seed {} must be owned by uid {expected_uid}, found uid {}", path.display(), metadata.uid() ); @@ -186,7 +228,7 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { if metadata.len() > MANAGED_SEED_FILE_MAX_LEN as u64 { bail!( - "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", + "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", path.display() ); } @@ -195,17 +237,17 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { Read::by_ref(&mut file) .take((MANAGED_SEED_FILE_MAX_LEN + 1) as u64) .read_to_string(&mut contents) - .with_context(|| format!("read managed envd access-token seed {}", path.display()))?; + .with_context(|| format!("read managed sandbox access-token seed {}", path.display()))?; if contents.len() > MANAGED_SEED_FILE_MAX_LEN { bail!( - "managed envd access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", + "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", path.display() ); } let seed = contents.strip_suffix('\n').unwrap_or(&contents); if !is_valid_managed_seed(seed) { bail!( - "managed envd access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", + "managed sandbox access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", path.display() ); } @@ -216,7 +258,7 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { fn create_managed_seed(path: &Path) -> Result { let parent = path .parent() - .context("managed envd access-token seed path has no parent")?; + .context("managed sandbox access-token seed path has no parent")?; fs::create_dir_all(parent) .with_context(|| format!("create managed secret directory {}", parent.display()))?; validate_managed_seed_directory_identity(parent).with_context(|| { @@ -232,7 +274,7 @@ fn create_managed_seed(path: &Path) -> Result { let mut random = [0_u8; MANAGED_SEED_BYTES]; SysRng .try_fill_bytes(&mut random) - .context("generate managed envd access-token seed")?; + .context("generate managed sandbox access-token seed")?; let seed = hex::encode(random); let mut temporary = tempfile::NamedTempFile::new_in(parent) @@ -250,17 +292,21 @@ fn create_managed_seed(path: &Path) -> Result { fs::File::open(parent) .and_then(|directory| directory.sync_all()) .with_context(|| format!("sync managed secret directory {}", parent.display()))?; - info!(path = %path.display(), "generated managed envd access-token seed"); + info!(path = %path.display(), "generated managed sandbox access-token seed"); Ok(seed) } Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { let file = open_managed_seed(path).with_context(|| { - format!("open managed envd access-token seed {}", path.display()) + format!("open managed sandbox access-token seed {}", path.display()) })?; read_managed_seed(path, file) } - Err(error) => Err(error.error) - .with_context(|| format!("persist managed envd access-token seed {}", path.display())), + Err(error) => Err(error.error).with_context(|| { + format!( + "persist managed sandbox access-token seed {}", + path.display() + ) + }), } } @@ -350,22 +396,31 @@ mod tests { } #[test] - fn generates_lowercase_hex_hmac_sha256() { + fn generates_e2b_compatible_access_tokens() { let generator = SandboxAccessTokenGenerator::new("test-seed").unwrap(); let subject = SandboxId::try_from("01936f8e-72f5-7000-8000-000000000001").unwrap(); - let token = generator.generate(subject); + let envd_token = generator.generate(subject); + let traffic_token = generator.generate_traffic(subject); - assert_eq!(token.expose().len(), 64); assert_eq!( - token.expose(), + envd_token.expose(), "4f00f2a93a87c37161ae01c59b6d4f84506668113441277e9f6272dd4bfae1a7" ); - assert!(token.expose().bytes().all(|byte| byte.is_ascii_hexdigit())); - assert_eq!(token.expose(), token.expose().to_ascii_lowercase()); - assert!(generator.matches(subject, token.expose())); + assert_eq!( + traffic_token, + "586547d7c10facb0f4871297fdbfd9d2b4376f4b02b2e1487646c1c87a293bd8" + ); + assert!(envd_token + .expose() + .bytes() + .all(|byte| byte.is_ascii_hexdigit())); + assert!(traffic_token.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert!(generator.matches(subject, envd_token.expose())); + assert!(generator.matches_traffic(subject, &traffic_token)); assert!(!generator.matches(subject, "not-a-token")); assert!(!generator.matches(subject, &"0".repeat(64))); + assert!(!generator.matches_traffic(subject, envd_token.expose())); } #[test] @@ -511,7 +566,7 @@ mod tests { assert!(error .to_string() - .contains("open managed envd access-token seed")); + .contains("open managed sandbox access-token seed")); Ok(()) } @@ -562,15 +617,13 @@ mod tests { } #[test] - fn missing_managed_seed_is_not_recreated_for_secure_state() -> Result<()> { + fn missing_managed_seed_is_not_recreated_for_persisted_state() -> Result<()> { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); let error = resolve_seed(&managed_path, true).unwrap_err(); - assert!(error - .to_string() - .contains("persisted secure sandboxes exist")); + assert!(error.to_string().contains("persisted sandboxes exist")); assert!(!managed_path.exists()); Ok(()) } From f7b670692ac3aef9bc5932cb50d6a0c471956db0 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:25:03 +0000 Subject: [PATCH 4/6] refactor: simplify shared authentication secrets --- deploy/k8s/base/agentenv-daemonset.yaml | 5 +- deploy/k8s/base/gateway-deployment.yaml | 5 - deploy/k8s/base/kustomization.yaml | 1 - .../k8s/overlays/local-dev/kustomization.yaml | 8 ++ deploy/k8s/run.sh | 35 +----- docs/src/configuration/authentication.md | 20 ++-- docs/src/configuration/env-vars.md | 2 +- docs/src/configuration/reference.md | 4 +- docs/src/deployment/docker-compose.md | 7 +- docs/src/deployment/kubernetes.md | 19 ++-- docs/src/deployment/static-multi-node.md | 9 +- docs/src/security/secure-sandboxes.md | 28 +++-- services/README.md | 6 +- services/gateway/cmd/main.go | 59 +++------- services/gateway/cmd/main_test.go | 27 ----- services/gateway/internal/server.go | 58 +--------- services/gateway/internal/server_test.go | 51 +++------ src/api/impls/auth.rs | 106 +++++------------- src/api/proxy.rs | 56 ++------- src/sandbox/access.rs | 19 ---- 20 files changed, 142 insertions(+), 383 deletions(-) diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index e95fefb9..470b2a9c 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -31,8 +31,9 @@ spec: - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED valueFrom: secretKeyRef: - name: agentenv-auth - key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED + name: agentenv-runtime-secrets + key: sandbox-access-token-hash-seed + optional: true - name: AENV_VIRTUALIZATION_MODE value: "kvm" - name: API_ADDR diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 7f7f8644..08536fc0 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -32,11 +32,6 @@ spec: secretKeyRef: name: agentenv-auth key: AENV_API_KEY - - name: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - valueFrom: - secretKeyRef: - name: agentenv-auth - key: AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED - name: GATEWAY_SANDBOX_PROXY_DOMAINS valueFrom: configMapKeyRef: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 157d8d62..556f0d70 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -34,7 +34,6 @@ secretGenerator: - name: agentenv-auth literals: - AENV_API_KEY= - - AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= images: - name: agentenv-gateway diff --git a/deploy/k8s/overlays/local-dev/kustomization.yaml b/deploy/k8s/overlays/local-dev/kustomization.yaml index ab182ed5..2a71de5e 100644 --- a/deploy/k8s/overlays/local-dev/kustomization.yaml +++ b/deploy/k8s/overlays/local-dev/kustomization.yaml @@ -6,6 +6,14 @@ namespace: agentenv-system resources: - ../../base +secretGenerator: + - name: agentenv-runtime-secrets + literals: + - sandbox-access-token-hash-seed=agentenv-local-dev-access-token-hash-seed + +generatorOptions: + disableNameSuffixHash: true + patches: - target: kind: DaemonSet diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 49f6aec9..e1e2ba89 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -38,17 +38,15 @@ if [[ "${MODE}" == "apply" ]]; then fi fi -read_existing_secret() { - local secret="$1" - local key="$2" +read_existing_api_key() { local encoded_value="" if [[ -z "${namespace_name}" ]]; then return 0 fi - if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret "${secret}" \ - --ignore-not-found -o "go-template={{index .data \"${key}\"}}")"; then - echo "failed to read ${key} from Secret ${NAMESPACE}/${secret}" >&2 + if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY"}}')"; then + echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 return 1 fi if [[ -n "${encoded_value}" ]]; then @@ -60,27 +58,7 @@ if [[ "${MODE}" != "delete" ]]; then API_KEY_VALUE="" if [[ "${AENV_API_KEY+x}" == "x" ]]; then API_KEY_VALUE="${AENV_API_KEY}" - elif ! API_KEY_VALUE="$(read_existing_secret agentenv-auth AENV_API_KEY)"; then - exit 1 - fi - - ACCESS_TOKEN_SEED_VALUE="" - if [[ "${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED+x}" == "x" ]]; then - ACCESS_TOKEN_SEED_VALUE="${AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED}" - elif ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-auth AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED)"; then - exit 1 - fi - if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then - if ! ACCESS_TOKEN_SEED_VALUE="$(read_existing_secret agentenv-runtime-secrets sandbox-access-token-hash-seed)"; then - exit 1 - fi - fi - - if [[ -z "${ACCESS_TOKEN_SEED_VALUE}" ]]; then - ACCESS_TOKEN_SEED_VALUE="$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" - fi - if [[ ! "${ACCESS_TOKEN_SEED_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then - echo "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED must contain at least 32 URL-safe characters" >&2 + elif ! API_KEY_VALUE="$(read_existing_api_key)"; then exit 1 fi @@ -95,9 +73,6 @@ if [[ "${MODE}" != "delete" ]]; then sed_in_place \ "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ "${TEMP_DIR}/k8s/base/kustomization.yaml" - sed_in_place \ - "s#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=.*#- AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED=${ACCESS_TOKEN_SEED_VALUE}#" \ - "${TEMP_DIR}/k8s/base/kustomization.yaml" fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index c04773f7..98d5aa68 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -39,10 +39,9 @@ generates a 256-bit key and atomically stores it in the managed path with `0600` permissions. It reuses that key on later starts. Dependency and host setup modes do not create a key. -The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`. It also reads the -sandbox seed from `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` or -`/run/secrets/sandbox-access-token-hash-seed`. The gateway never generates -either value because it must share them with every runtime node. +The gateway uses `AENV_API_KEY` or `/run/secrets/api-key`; it never generates a +key. Runtime nodes validate sandbox-scoped tokens, so the gateway does not need +the sandbox seed. ## Installation Methods @@ -70,8 +69,7 @@ container replacements. The checked-in Compose deployment mounts one named volume read-write on both runtime nodes and read-only at `/run/secrets` on the gateway. Concurrent node startup is safe: atomic creation makes both nodes converge on the same key and -sandbox seed. -Read it with: +sandbox seed. The gateway reads only the API key from that volume. Read it with: ```bash docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ @@ -81,8 +79,8 @@ docker compose -f deploy/docker-compose.yml exec -T agentenv-a \ `docker compose down` preserves the key. `docker compose down -v` removes the auth volume, so the next startup generates a new key. -`make k8s-apply` creates `Secret/agentenv-auth` with an API key and sandbox -seed on the first apply, then reuses both values. Read the API key with: +`make k8s-apply` creates `Secret/agentenv-auth` with an API key on the first +apply, then reuses it. Read the key with: ```bash kubectl -n agentenv-system get secret agentenv-auth \ @@ -120,7 +118,7 @@ network, use a VPN, or terminate HTTPS at a reverse proxy or load balancer. ## Rotation Changing `AENV_API_KEY` invalidates existing client API credentials without -changing sandbox credentials. Changing +changing sandbox credentials; apply it to the gateway and every runtime node +together. Changing `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` rotates both `trafficAccessToken` and -`envdAccessToken` values. Apply either change to the gateway and every runtime -node together, then restart them. +`envdAccessToken` values and must be changed on every runtime node together. diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index bb458564..a5f8c47c 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -24,7 +24,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_OBSERVABILITY_SCHEDULER_ENDPOINT` | unset | Override scheduler heartbeat reporting endpoint | | `AENV_OBSERVABILITY_REPORT_INTERVAL_SECS` | `5` | Override heartbeat reporting interval in seconds | | `AENV_CUSTOM_EXTENSION_URL` | unset | Override `[custom_extension].url`, the HTTP base URL of the custom extension service | -| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | `/run/secrets/sandbox-access-token-hash-seed`, then auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Clustered deployments must configure the same value on the gateway and every runtime node. | +| `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` | auto-generated under `$AENV_HOME/secrets` | Optional runtime override for the secret used to derive sandbox envd and traffic access tokens. Configure the same value on every runtime node in clustered deployments. | | `AENV_SANDBOX_PROXY_DOMAINS` | from config | Comma-separated DNS domains that enable server-side host-based sandbox proxy URLs like `{port}-{sandboxID}.{domain}` and populate the sandbox response `domain` field. Empty or unset keeps `[sandbox_proxy].domains`. | | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index a81591d6..6f646576 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -259,9 +259,9 @@ Sandbox control communication settings. |-----|------|---------|-------------| | `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | -The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates access tokens for existing secure sandboxes. +The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. -Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on the gateway and every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. +Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. ## `[orchestrator]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index b59b1fcf..3c0f2fec 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -40,10 +40,9 @@ make deploy-up ``` On first startup, the runtime nodes atomically generate one API key and sandbox -access-token seed in the -shared `agentenv-auth` volume. The gateway mounts that volume read-only at -`/run/secrets`, so all three services use the same secrets. Normal -`make deploy-down` calls preserve the volume and both values. +access-token seed in the shared `agentenv-auth` volume. The gateway mounts that +volume read-only and reads the API key; sandbox tokens are validated by the +runtime nodes. Normal `make deploy-down` calls preserve both values. To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when starting the stack: diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 7f080595..6d7ec4b0 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -60,18 +60,20 @@ make k8s-render make k8s-apply ``` -`make k8s-apply` generates a 256-bit API key and sandbox access-token seed on -the first deployment and stores both in `Secret/agentenv-auth`. Later applies -reuse both values. Read the API key locally when configuring clients: +`make k8s-apply` generates a 256-bit API key on the first deployment and stores +it in `Secret/agentenv-auth`. Later applies reuse it. Read the key locally when +configuring clients: ```bash kubectl -n agentenv-system get secret agentenv-auth \ -o go-template='{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' ``` -Set `AENV_API_KEY` and `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` when applying to -supply your own values. A standalone `make k8s-render` uses temporary generated -values because it does not modify or read cluster state. +Set `AENV_API_KEY` when applying to supply your own value. A standalone +`make k8s-render` uses a temporary generated value because it does not modify or +read cluster state. The optional runtime seed keeps its existing +`agentenv-runtime-secrets` contract described in +[Secure Sandboxes](../security/secure-sandboxes.md). To enable host-based sandbox data-plane URLs, set the shared sandbox proxy domain variable when rendering or applying manifests: @@ -113,8 +115,9 @@ make k8s-delete A dedicated `local-dev` overlay mounts the repository's `env/` directory directly into the DaemonSet at `/workspace/env`, avoiding runtime asset copies: -The apply helper provisions the same generated `agentenv-auth` secrets used by -the default overlay. +The apply helper provisions the same generated API key used by the default +overlay. The local development overlay retains its fixed test-only runtime seed; +do not reuse that seed outside local development. ```bash make k8s-build diff --git a/docs/src/deployment/static-multi-node.md b/docs/src/deployment/static-multi-node.md index 1ca6821c..b727983b 100644 --- a/docs/src/deployment/static-multi-node.md +++ b/docs/src/deployment/static-multi-node.md @@ -54,9 +54,9 @@ an external metrics collector needs them. ## 1. Install the runtime nodes -Generate one API key and one sandbox access-token seed, deliver them through -your normal secret-management channel, and use the same values on every runtime -node and the Gateway: +Generate one API key and one sandbox access-token seed through your normal +secret-management channel. Use the API key on the gateway and every runtime +node; use the seed only on runtime nodes: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" @@ -116,7 +116,7 @@ sudo useradd --system --no-create-home --shell /usr/sbin/nologin agentenv-contro sudo install -d -o root -g agentenv-control -m 0750 /etc/agentenv ``` -Create `/etc/agentenv/auth.env` with the same values used on the runtime nodes: +Create `/etc/agentenv/auth.env` with the API key used on the runtime nodes: ```bash sudo install -o root -g agentenv-control -m 0640 /dev/null /etc/agentenv/auth.env @@ -125,7 +125,6 @@ sudoedit /etc/agentenv/auth.env ```text AENV_API_KEY= -AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED= ``` If the `agentenv-control` account already exists, the `useradd` command reports diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 2eadf4b6..32e38750 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -20,31 +20,43 @@ The API and SDKs return the sandbox's `envdAccessToken` where appropriate and at A seed is a random value used to derive each sandbox's envd and traffic access tokens. This seed is optional for a standalone runtime. When it is unset, the runtime automatically creates and persists a seed under `$AENV_HOME/secrets`. This is sufficient for normal single-node operation and does not require additional setup. -Configure the same explicit seed on the gateway and every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: +Configure the same explicit seed on every runtime node in a clustered deployment. Generate it once and store it in the deployment's secret manager: ```bash openssl rand -hex 32 ``` -Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on the gateway and every runtime node. +Set the value as `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` on every runtime node. For TOML configuration, use `[sandbox].access_token_hash_seed` instead. -Container deployments may mount it at -`/run/secrets/sandbox-access-token-hash-seed`. Preserve the seed across upgrades; changing it rotates both sandbox access tokens. ### Kubernetes -`make k8s-apply` generates and preserves the seed in `Secret/agentenv-auth`, then injects it into the gateway and runtime Pods. Set `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` before applying to supply your own value. +The runtime DaemonSet retains the existing optional `agentenv-runtime-secrets` +contract. Create one shared seed before applying the runtime manifests: -An external secret manager may provide the same Secret and key: +```bash +kubectl apply -f deploy/k8s/base/namespace.yaml + +AENV_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" +kubectl -n agentenv-system create secret generic agentenv-runtime-secrets \ + --from-literal="sandbox-access-token-hash-seed=${AENV_ACCESS_TOKEN_HASH_SEED}" \ + --dry-run=client -o yaml | kubectl apply -f - +unset AENV_ACCESS_TOKEN_HASH_SEED +``` + +Preserve this Secret during upgrades. An external secret manager may provide +the same name and key: ```yaml apiVersion: v1 kind: Secret metadata: - name: agentenv-auth + name: agentenv-runtime-secrets namespace: agentenv-system stringData: - AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED: + sandbox-access-token-hash-seed: ``` + +If the Secret is absent, each runtime Pod uses its managed node-local seed. diff --git a/services/README.md b/services/README.md index 9bb1d066..a5fe1869 100644 --- a/services/README.md +++ b/services/README.md @@ -73,15 +73,13 @@ Start gateway with the same API key configured on every AgentENV runtime node: ```bash export AENV_API_KEY="e2b_$(openssl rand -hex 32)" -export AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED="$(openssl rand -hex 32)" make run-gateway ``` The default local config uses `127.0.0.1:9090` for the scheduler. -The gateway and runtime nodes require the same API key and sandbox access-token -seed. The gateway reads explicit environment values or the corresponding files -under `/run/secrets`; it does not generate either secret. +The gateway and runtime nodes require the same API key. The gateway reads +`AENV_API_KEY` or `/run/secrets/api-key`; it does not generate a key. Application proxy requests may additionally use the sandbox response's `trafficAccessToken` in the `e2b-traffic-access-token` header. diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index 2471da2c..389920ff 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -25,10 +25,8 @@ import ( ) const ( - apiKeyEnv = "AENV_API_KEY" - accessTokenSeedEnv = "AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED" - defaultAPIKeyPath = "/run/secrets/api-key" - defaultAccessTokenSeedPath = "/run/secrets/sandbox-access-token-hash-seed" + apiKeyEnv = "AENV_API_KEY" + defaultAPIKeyPath = "/run/secrets/api-key" ) func newSchedulerConn(addr string) (*grpc.ClientConn, error) { @@ -43,35 +41,20 @@ func loadAPIKey() (string, error) { } func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { - return loadSecretFrom(lookupEnv, apiKeyEnv, secretPath, validateAPIKey) -} - -func loadAccessTokenSeed() (string, error) { - return loadAccessTokenSeedFrom(os.LookupEnv, defaultAccessTokenSeedPath) -} - -func loadAccessTokenSeedFrom(lookupEnv func(string) (string, bool), secretPath string) (string, error) { - return loadSecretFrom(lookupEnv, accessTokenSeedEnv, secretPath, validateAccessTokenSeed) -} - -func loadSecretFrom( - lookupEnv func(string) (string, bool), - envName string, - secretPath string, - validate func(string, string) (string, error), -) (string, error) { - if value, present := lookupEnv(envName); present { - return validate(value, envName) - } - - contents, err := os.ReadFile(secretPath) - if err != nil { - if os.IsNotExist(err) { - return "", fmt.Errorf("%s must be set or %s must exist", envName, secretPath) + value, source := "", apiKeyEnv + if explicit, present := lookupEnv(apiKeyEnv); present { + value = explicit + } else { + contents, err := os.ReadFile(secretPath) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) + } + return "", fmt.Errorf("read secret %s: %w", secretPath, err) } - return "", fmt.Errorf("read secret %s: %w", secretPath, err) + value, source = string(contents), secretPath } - return validate(string(contents), secretPath) + return validateAPIKey(value, source) } func validateAPIKey(value, source string) (string, error) { @@ -91,14 +74,6 @@ func validateAPIKey(value, source string) (string, error) { return value, nil } -func validateAccessTokenSeed(value, source string) (string, error) { - value = strings.TrimSpace(value) - if value == "" { - return "", fmt.Errorf("sandbox access-token seed from %s must be non-empty", source) - } - return value, nil -} - func main() { configPath := flag.String("config", "", "path to JSON config file") flag.Parse() @@ -111,11 +86,6 @@ func main() { if err != nil { log.Fatalf("load API key failed: %v", err) } - accessTokenSeed, err := loadAccessTokenSeed() - if err != nil { - log.Fatalf("load sandbox access-token seed failed: %v", err) - } - logger, err := logging.New(cfg.LogLevel, cfg.LogFormat) if err != nil { log.Fatalf("init logger failed: %v", err) @@ -144,7 +114,6 @@ func main() { RequestTimeout: cfg.Gateway.RequestTimeout, MaxResponseSize: cfg.Gateway.ForwardResponseSize, APIKey: apiKey, - SandboxAccessTokenSeed: accessTokenSeed, DebugMode: cfg.Gateway.DebugMode, SandboxProxyDomains: cfg.Gateway.SandboxProxyDomains, QueryOnlySchedulerClient: queryOnlySchedulerClient, diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index 2fc94165..9064986b 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -8,7 +8,6 @@ import ( ) const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" -const testAccessTokenSeed = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" func TestValidateAPIKey(t *testing.T) { t.Parallel() @@ -76,29 +75,3 @@ func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") } } - -func TestLoadAccessTokenSeed(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - path := filepath.Join(dir, "sandbox-access-token-hash-seed") - if err := os.WriteFile(path, []byte(testAccessTokenSeed+"\n"), 0o444); err != nil { - t.Fatal(err) - } - got, err := loadAccessTokenSeedFrom(func(string) (string, bool) { return "", false }, path) - if err != nil { - t.Fatalf("loadAccessTokenSeedFrom() error = %v", err) - } - if got != testAccessTokenSeed { - t.Fatalf("loadAccessTokenSeedFrom() = %q, want %q", got, testAccessTokenSeed) - } -} - -func TestLoadAccessTokenSeedRejectsExplicitEmptyEnvironment(t *testing.T) { - if _, err := loadAccessTokenSeedFrom( - func(name string) (string, bool) { return "", name == accessTokenSeedEnv }, - filepath.Join(t.TempDir(), "missing"), - ); err == nil { - t.Fatal("loadAccessTokenSeedFrom() unexpectedly accepted an empty environment value") - } -} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 739237d6..310efada 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,9 +3,6 @@ package gateway import ( "bytes" "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "io" @@ -28,8 +25,6 @@ const ( headerAPIKey = "X-API-Key" headerTrafficToken = "e2b-traffic-access-token" headerEnvdAccessToken = "X-Access-Token" - trafficTokenPrefix = "sandbox-traffic" - envdControlPlanePort = 49983 headerSandboxID = "x-agentenv-sandbox-id" headerE2BSandboxID = "e2b-sandbox-id" headerTargetPort = "x-agentenv-target-port" @@ -50,7 +45,6 @@ const ( type ServerOptions struct { APIKey string - SandboxAccessTokenSeed string RequestTimeout time.Duration MaxResponseSize int64 DebugMode bool @@ -64,7 +58,6 @@ type Server struct { queryOnlyScheduler schedulerv1.SchedulerClient httpClient *http.Client apiKey []byte - accessTokenSeed []byte requestTimeout time.Duration maxRespSize int64 // debugMode, when true, enables debug-only behaviors such as exposing @@ -79,11 +72,6 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, if apiKey == "" { return nil, errors.New("API key is required") } - accessTokenSeed := strings.TrimSpace(options.SandboxAccessTokenSeed) - if accessTokenSeed == "" { - return nil, errors.New("sandbox access-token seed is required") - } - sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) if err != nil { return nil, err @@ -102,7 +90,6 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, apiKey: []byte(apiKey), - accessTokenSeed: []byte(accessTokenSeed), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -838,12 +825,6 @@ func singleHeaderMatches(headers http.Header, name string, expected []byte) bool return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) } -func trafficAccessToken(seed []byte, sandboxID string) string { - mac := hmac.New(sha256.New, seed) - _, _ = mac.Write([]byte(trafficTokenPrefix + "-" + sandboxID)) - return hex.EncodeToString(mac.Sum(nil)) -} - func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { if strings.TrimRight(r.URL.Path, "/") == "/proxy" || strings.HasPrefix(r.URL.Path, "/proxy/") { return true @@ -857,33 +838,6 @@ func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { return !isSandboxControlPlaneRequest(r) && hasProxyRoutingHeaders(r.Header) } -func (s *Server) sandboxIDForDataPlaneAuth(r *http.Request) (string, bool) { - hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) - if err != nil { - return "", false - } - if hostRoute != nil { - return hostRoute.sandboxID, true - } - return sandboxIDFromHeaders(r.Header) -} - -func (s *Server) isEnvdDataPlaneRequest(r *http.Request) bool { - hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) - if err != nil { - return false - } - if hostRoute != nil { - return hostRoute.targetPort == envdControlPlanePort - } - targetPort, ok := targetPortFromHeaders(r.Header) - if !ok { - return false - } - port, err := strconv.Atoi(targetPort) - return err == nil && port == envdControlPlanePort -} - func hasSingleNonEmptyHeader(headers http.Header, name string) bool { values := headers.Values(name) return len(values) == 1 && strings.TrimSpace(values[0]) != "" @@ -899,15 +853,9 @@ func (s *Server) authenticate(next http.Handler) http.Handler { authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) if !authorized && dataPlane { - if sandboxID, ok := s.sandboxIDForDataPlaneAuth(r); ok { - expected := trafficAccessToken(s.accessTokenSeed, sandboxID) - authorized = singleHeaderMatches(r.Header, headerTrafficToken, []byte(expected)) - } - } - if !authorized && dataPlane && s.isEnvdDataPlaneRequest(r) { - // The runtime node owns the envd token seed and performs the definitive - // sandbox-scoped validation before forwarding the request to envd. - authorized = hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) + // Runtime nodes perform the definitive sandbox-scoped token validation. + authorized = hasSingleNonEmptyHeader(r.Header, headerTrafficToken) || + hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) } if !authorized { w.WriteHeader(http.StatusUnauthorized) diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index 30c76fb8..e192e873 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -148,10 +148,7 @@ func (s stubSchedulerClient) UnregisterNode(ctx context.Context, req *schedulerv return s.unregisterNodeFunc(ctx, req, opts...) } -const ( - testAPIKey = "test-api-key" - testAccessTokenSeed = "test-access-token-seed" -) +const testAPIKey = "test-api-key" type testServerOption func(*ServerOptions) @@ -159,10 +156,9 @@ func newTestServer(t *testing.T, schedulerClient schedulerv1.SchedulerClient, ti t.Helper() options := ServerOptions{ - RequestTimeout: timeout, - MaxResponseSize: maxRespSize, - APIKey: testAPIKey, - SandboxAccessTokenSeed: testAccessTokenSeed, + RequestTimeout: timeout, + MaxResponseSize: maxRespSize, + APIKey: testAPIKey, } for _, opt := range opts { opt(&options) @@ -193,17 +189,6 @@ func TestNewServerRejectsEmptyAPIKey(t *testing.T) { } } -func TestNewServerRejectsEmptySandboxAccessTokenSeed(t *testing.T) { - _, err := NewServer(zap.NewNop(), stubSchedulerClient{}, ServerOptions{ - APIKey: testAPIKey, - RequestTimeout: time.Second, - MaxResponseSize: 1024, - }) - if err == nil { - t.Fatal("NewServer accepted an empty sandbox access-token seed") - } -} - func TestGatewayRequiresExactAPIKey(t *testing.T) { server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) handler := server.Handler() @@ -263,7 +248,7 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { } } -func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { +func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" lookupCalls := 0 server := newTestServer(t, stubSchedulerClient{ @@ -277,7 +262,7 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { @@ -287,42 +272,34 @@ func TestGatewayAcceptsSandboxScopedTrafficToken(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), "another-sandbox")) + req.Header.Set(headerTrafficToken, "wrong-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { - t.Fatalf("wrong scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 2 { + t.Fatalf("runtime-scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "8080") - req.Header.Set("X-Access-Token", trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) + req.Header.Set("X-Access-Token", "runtime-validates-this-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { - t.Fatalf("envd token authorized application proxy: status=%d lookup calls=%d", recorder.Code, lookupCalls) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 3 { + t.Fatalf("runtime-scoped envd token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } req = httptest.NewRequest(http.MethodPost, "/sandboxes/"+sandboxID+"/pause", nil) req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerTrafficToken, trafficAccessToken([]byte(testAccessTokenSeed), sandboxID)) + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 1 { + if recorder.Code != http.StatusUnauthorized || lookupCalls != 3 { t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) } } -func TestTrafficAccessTokenVector(t *testing.T) { - const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" - const want = "f5457a589b09265b169392dd49506ec70458f685cf2ba7fc2c5b4763c42a5b17" - if got := trafficAccessToken([]byte("test-seed"), sandboxID); got != want { - t.Fatalf("trafficAccessToken() = %q, want %q", got, want) - } -} - func withSandboxProxyDomains(domains ...string) testServerOption { return func(options *ServerOptions) { options.SandboxProxyDomains = domains diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 9084f9f1..6b197c61 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use axum::{ body::Body, extract::{Request, State}, - http::{header::HeaderMap, StatusCode}, + http::{header::HeaderMap, HeaderValue, StatusCode}, middleware::Next, response::{IntoResponse, Response}, }; @@ -15,21 +15,16 @@ pub(crate) const API_KEY_HEADER: &str = "x-api-key"; pub(crate) const TRAFFIC_ACCESS_TOKEN_HEADER: &str = "e2b-traffic-access-token"; pub(crate) const ENVD_ACCESS_TOKEN_HEADER: &str = "x-access-token"; -fn single_header_matches(headers: &HeaderMap, name: &str, expected: &str) -> bool { +fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a HeaderValue> { let mut values = headers.get_all(name).iter(); - let Some(value) = values.next() else { - return false; - }; - if values.next().is_some() { - return false; - } - - value.as_bytes() == expected.as_bytes() + let value = values.next()?; + values.next().is_none().then_some(value) } impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { - single_header_matches(headers, API_KEY_HEADER, &self.api_key) + single_header(headers, API_KEY_HEADER) + .is_some_and(|value| value.as_bytes() == self.api_key.as_bytes()) } pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { @@ -37,16 +32,12 @@ impl ApiImpl { } fn has_valid_traffic_access_token(&self, headers: &HeaderMap, sandbox_id: SandboxId) -> bool { - let mut values = headers.get_all(TRAFFIC_ACCESS_TOKEN_HEADER).iter(); - let Some(candidate) = values.next().and_then(|value| value.to_str().ok()) else { - return false; - }; - if values.next().is_some() { - return false; - } - - self.orchestrator - .validate_traffic_access_token(sandbox_id, candidate) + single_header(headers, TRAFFIC_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| { + self.orchestrator + .validate_traffic_access_token(sandbox_id, candidate) + }) } } @@ -64,28 +55,26 @@ where return next.run(request).await; } - let mut authorized = api_impl.as_ref().has_valid_api_key(request.headers()); - if !authorized && proxy_request { - authorized = - proxy::sandbox_id_for_proxy_auth(&request, api_impl.as_ref().sandbox_proxy_domains()) - .is_some_and(|sandbox_id| { - api_impl - .as_ref() - .has_valid_traffic_access_token(request.headers(), sandbox_id) - }); - } + let api_impl = api_impl.as_ref(); + let mut authorized = api_impl.has_valid_api_key(request.headers()); if !authorized && proxy_request { - if let Some((sandbox_id, target_port, candidate)) = proxy::envd_access_token_for_proxy_auth( - &request, - api_impl.as_ref().sandbox_proxy_domains(), - ) { - authorized = proxy::has_valid_envd_access_token( - api_impl.as_ref(), - sandbox_id, - target_port, - candidate, - ) - .await; + if let Some((sandbox_id, target_port)) = + proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) + { + authorized = api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); + let envd_candidate = single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()); + if !authorized { + if let Some(candidate) = envd_candidate { + authorized = proxy::has_valid_envd_access_token( + api_impl, + sandbox_id, + target_port, + candidate, + ) + .await; + } + } } } @@ -124,36 +113,3 @@ impl apis::ApiAuthBasic for ApiImpl { self.has_valid_api_key(headers).then_some(Claims) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn header_match_requires_one_exact_value() { - let mut headers = HeaderMap::new(); - assert!(!single_header_matches( - &headers, - API_KEY_HEADER, - "correct-key" - )); - headers.insert(API_KEY_HEADER, "correct-key".parse().unwrap()); - assert!(single_header_matches( - &headers, - API_KEY_HEADER, - "correct-key" - )); - assert!(!single_header_matches( - &headers, - API_KEY_HEADER, - "wrong-key" - )); - - headers.append(API_KEY_HEADER, "correct-key".parse().unwrap()); - assert!(!single_header_matches( - &headers, - API_KEY_HEADER, - "correct-key" - )); - } -} diff --git a/src/api/proxy.rs b/src/api/proxy.rs index df8d7b19..76a8fdac 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -148,49 +148,24 @@ where .with_state(api_impl) } -fn proxy_route_for_auth(request: &Request, domains: &[String]) -> Option { +pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { match parse_host_proxy_route(request_host(request), domains) { - Ok(Some(route)) => return Some(route), + Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), Err(_) => return None, Ok(None) => {} } - Some(HostProxyRoute { - sandbox_id: parse_sandbox_id_header(request.headers()).ok()?, - target_port: parse_target_port_header(request.headers()).ok()?, - }) -} - -pub(crate) fn sandbox_id_for_proxy_auth( - request: &Request, - domains: &[String], -) -> Option { - Some(proxy_route_for_auth(request, domains)?.sandbox_id) -} - -pub(crate) fn envd_access_token_for_proxy_auth( - request: &Request, - domains: &[String], -) -> Option<(SandboxId, u16, String)> { - let route = proxy_route_for_auth(request, domains)?; - let candidate = { - let mut candidates = request.headers().get_all(ENVD_ACCESS_TOKEN_HEADER).iter(); - let candidate = candidates.next()?; - if candidates.next().is_some() { - return None; - } - let candidate = candidate.to_str().ok()?; - candidate.to_owned() - }; - - Some((route.sandbox_id, route.target_port, candidate)) + Some(( + parse_sandbox_id_header(request.headers()).ok()?, + parse_target_port_header(request.headers()).ok()?, + )) } pub(crate) async fn has_valid_envd_access_token( api_impl: &ApiImpl, sandbox_id: SandboxId, target_port: u16, - candidate: String, + candidate: &str, ) -> bool { let Ok(Some(metadata)) = api_impl.orchestrator().get_sandbox(&sandbox_id).await else { return false; @@ -201,7 +176,7 @@ pub(crate) async fn has_valid_envd_access_token( api_impl .orchestrator() - .validate_envd_access_token(sandbox_id, &candidate) + .validate_envd_access_token(sandbox_id, candidate) } pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { @@ -245,17 +220,10 @@ where return next.run(request).await; } - let host = request - .headers() - .get(header::HOST) - .and_then(|host| host.to_str().ok()) - .or_else(|| { - request - .uri() - .authority() - .map(|authority| authority.as_str()) - }); - let host_route = match parse_host_proxy_route(host, api_impl.as_ref().sandbox_proxy_domains()) { + let host_route = match parse_host_proxy_route( + request_host(&request), + api_impl.as_ref().sandbox_proxy_domains(), + ) { Ok(Some(route)) => route, Ok(None) => { return next.run(request).await; diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index f3c88dc6..abf13788 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -15,7 +15,6 @@ use crate::types::SandboxId; type HmacSha256 = Hmac; const MANAGED_SEED_RELATIVE_PATH: &str = "secrets/sandbox-access-token-hash-seed"; -const EXTERNAL_SEED_PATH: &str = "/run/secrets/sandbox-access-token-hash-seed"; const MANAGED_SEED_BYTES: usize = 32; const SEED_HEX_LEN: usize = MANAGED_SEED_BYTES * 2; const MANAGED_SEED_FILE_MAX_LEN: usize = SEED_HEX_LEN + 1; @@ -57,24 +56,6 @@ impl SandboxAccessTokenGenerator { return Self::new(seed); } - match fs::read_to_string(EXTERNAL_SEED_PATH) { - Ok(seed) => { - let generator = - Self::new(&seed).context("invalid external sandbox access-token seed")?; - info!( - path = EXTERNAL_SEED_PATH, - "loaded sandbox access-token seed from external secret" - ); - return Ok(generator); - } - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!("read external sandbox access-token seed {EXTERNAL_SEED_PATH}") - }); - } - } - let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; From 31eda91c1dd5a12b34e431d243107e0f711f5928 Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:47:30 +0000 Subject: [PATCH 5/6] fix: harden managed secret loading --- Cargo.lock | 1 + Cargo.toml | 1 + deploy/k8s/run.sh | 52 ++++- docs/src/configuration/authentication.md | 2 +- scripts/tests/e2e/lib/runtime.sh | 2 +- scripts/tests/e2e/suites/09_e2b_compat.sh | 1 + services/gateway/cmd/main.go | 20 +- services/gateway/cmd/main_test.go | 4 +- src/api/impls/auth.rs | 37 ++-- src/api/proxy.rs | 51 ++--- src/api_key.rs | 92 ++++---- src/lib.rs | 1 + src/managed_secret.rs | 233 ++++++++++++++++++++ src/sandbox/access.rs | 251 ++++------------------ 14 files changed, 419 insertions(+), 329 deletions(-) create mode 100644 src/managed_secret.rs diff --git a/Cargo.lock b/Cargo.lock index bead1f50..ab005734 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,7 @@ dependencies = [ "sha2 0.10.9", "shell-util", "storage-util", + "subtle", "tar", "tempfile", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index 62083f75..b596b9aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ io-uring = "0.7.9" tokio-tungstenite = "0.28" sha2 = "0.10" hmac = "0.12" +subtle = "2.6" hex = "0.4" semver = "1" iroh = { version = "=1.0.0-rc.0" } diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index e1e2ba89..93d9cdd6 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -11,6 +11,10 @@ shift KUBECTL_BIN="${KUBECTL:-kubectl}" OVERLAY_NAME="${K8S_OVERLAY:-default}" NAMESPACE="${K8S_NAMESPACE:-agentenv-system}" +if [[ ${#NAMESPACE} -gt 63 || ! "${NAMESPACE}" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "K8S_NAMESPACE must be a valid Kubernetes namespace name" >&2 + exit 1 +fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" @@ -27,8 +31,36 @@ sed_in_place() { fi } +render_api_key() { + local file="$1" + local rendered_file + + rendered_file="$(mktemp "${TEMP_DIR}/api-key.XXXXXX")" + if ! { + printf '%s\n' "${API_KEY_VALUE}" + cat "${file}" + } | awk ' + NR == 1 { api_key = $0; next } + /^ - AENV_API_KEY=/ { print " - AENV_API_KEY=" api_key; replaced = 1; next } + { print } + END { if (!replaced) exit 1 } + ' >"${rendered_file}"; then + return 1 + fi + mv "${rendered_file}" "${file}" +} + cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" +OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" +if [[ ! -d "${OVERLAY_PATH}" ]]; then + echo "unknown overlay: ${OVERLAY_NAME}" >&2 + exit 1 +fi +sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/kustomization.yaml" +sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${OVERLAY_PATH}/kustomization.yaml" +sed_in_place "s# name: agentenv-system# name: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/namespace.yaml" +sed_in_place "s#\"namespace\": \"agentenv-system\"#\"namespace\": \"${NAMESPACE}\"#" "${TEMP_DIR}/k8s/base/config/scheduler.json" namespace_name="" if [[ "${MODE}" == "apply" ]]; then @@ -55,6 +87,11 @@ read_existing_api_key() { } if [[ "${MODE}" != "delete" ]]; then + restore_xtrace=0 + if [[ $- == *x* ]]; then + restore_xtrace=1 + set +x + fi API_KEY_VALUE="" if [[ "${AENV_API_KEY+x}" == "x" ]]; then API_KEY_VALUE="${AENV_API_KEY}" @@ -65,14 +102,13 @@ if [[ "${MODE}" != "delete" ]]; then if [[ -z "${API_KEY_VALUE}" ]]; then API_KEY_VALUE="e2b_$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" fi - if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then - echo "AENV_API_KEY must contain at least 32 URL-safe characters" >&2 + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then + echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 exit 1 fi - sed_in_place \ - "s#- AENV_API_KEY=.*#- AENV_API_KEY=${API_KEY_VALUE}#" \ - "${TEMP_DIR}/k8s/base/kustomization.yaml" + render_api_key "${TEMP_DIR}/k8s/base/kustomization.yaml" + [[ "${restore_xtrace}" == "0" ]] || set -x fi if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then @@ -82,12 +118,6 @@ if [[ "${SANDBOX_PROXY_DOMAINS+x}" == "x" ]]; then sed_in_place "s#- SANDBOX_PROXY_DOMAINS=.*#- SANDBOX_PROXY_DOMAINS=${ESCAPED_SANDBOX_PROXY_DOMAINS}#" "${TEMP_DIR}/k8s/base/kustomization.yaml" fi -OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" -if [[ ! -d "${OVERLAY_PATH}" ]]; then - echo "unknown overlay: ${OVERLAY_NAME}" >&2 - exit 1 -fi - if [[ "${OVERLAY_NAME}" == "local-dev" ]]; then REPO_ENV_PATH="${AENV_LOCAL_REPO_ENV_PATH:-${REPO_ROOT}/env}" if [[ ! -d "${REPO_ENV_PATH}" ]]; then diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index 98d5aa68..4d4349f5 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -96,7 +96,7 @@ export AENV_API_KEY="e2b_$(openssl rand -hex 32)" make start-server ``` -Custom keys must contain at least 32 URL-safe characters. In a multi-node +Custom keys must contain between 32 and 4096 URL-safe characters. In a multi-node deployment, use exactly the same value for the gateway and every runtime node. The generated keys use `e2b_` followed by hexadecimal characters so they pass the E2B SDK default API-key validation. Use that format for custom keys when diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index e22d2a90..4eb3e1e9 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -400,7 +400,7 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || die "Failed to read the Compose deployment API key" - [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,}$ ]] || + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || die "Compose deployment returned an invalid API key" export AENV_API_KEY diff --git a/scripts/tests/e2e/suites/09_e2b_compat.sh b/scripts/tests/e2e/suites/09_e2b_compat.sh index 611b09ad..b4898eda 100755 --- a/scripts/tests/e2e/suites/09_e2b_compat.sh +++ b/scripts/tests/e2e/suites/09_e2b_compat.sh @@ -11,6 +11,7 @@ log "Suite: E2B Compatibility" export E2B_API_URL="${AENV_URL}" export E2B_SANDBOX_URL="${AENV_PROXY_URL}" export E2B_API_KEY="${AENV_API_KEY}" +unset E2B_ACCESS_TOKEN export E2B_COMPAT_USER_IMAGE="${E2B_COMPAT_USER_IMAGE:-${E2E_TEMPLATE_USER_IMAGE:-ghcr.io/linuxserver/baseimage-ubuntu:noble}}" cli_available=0 diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index 389920ff..a0cfacbb 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "log" "net/http" "os" @@ -27,6 +28,8 @@ import ( const ( apiKeyEnv = "AENV_API_KEY" defaultAPIKeyPath = "/run/secrets/api-key" + maxAPIKeyLen = 4096 + maxAPIKeyFileLen = maxAPIKeyLen + 2 ) func newSchedulerConn(addr string) (*grpc.ClientConn, error) { @@ -45,22 +48,27 @@ func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (s if explicit, present := lookupEnv(apiKeyEnv); present { value = explicit } else { - contents, err := os.ReadFile(secretPath) + file, err := os.Open(secretPath) if err != nil { if os.IsNotExist(err) { return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) } return "", fmt.Errorf("read secret %s: %w", secretPath, err) } - value, source = string(contents), secretPath + defer file.Close() + contents, err := io.ReadAll(io.LimitReader(file, maxAPIKeyFileLen+1)) + if err != nil { + return "", fmt.Errorf("read secret %s: %w", secretPath, err) + } + value = strings.TrimSuffix(strings.TrimSuffix(string(contents), "\n"), "\r") + source = secretPath } return validateAPIKey(value, source) } func validateAPIKey(value, source string) (string, error) { - value = strings.TrimSpace(value) - if len(value) < 32 { - return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + if len(value) < 32 || len(value) > maxAPIKeyLen { + return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) } for _, char := range []byte(value) { if (char >= 'a' && char <= 'z') || @@ -69,7 +77,7 @@ func validateAPIKey(value, source string) (string, error) { char == '.' || char == '_' || char == '~' || char == '-' { continue } - return "", fmt.Errorf("API key from %s must contain at least 32 URL-safe characters", source) + return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) } return value, nil } diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index 9064986b..bc88faeb 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -12,7 +12,7 @@ const testAPIKey = "e2b_0123456789abcdef0123456789abcdef0123456789abcdef01234567 func TestValidateAPIKey(t *testing.T) { t.Parallel() - got, err := validateAPIKey(" "+testAPIKey+"\n", "test") + got, err := validateAPIKey(testAPIKey, "test") if err != nil { t.Fatalf("validateAPIKey() error = %v", err) } @@ -20,7 +20,7 @@ func TestValidateAPIKey(t *testing.T) { t.Fatalf("validateAPIKey() = %q, want %q", got, testAPIKey) } - for _, invalid := range []string{"", "too-short", strings.Repeat("a", 31), strings.Repeat("a", 31) + "!"} { + for _, invalid := range []string{"", "too-short", " " + testAPIKey, testAPIKey + "\n", strings.Repeat("a", 31), strings.Repeat("a", maxAPIKeyLen+1), strings.Repeat("a", 31) + "!"} { if _, err := validateAPIKey(invalid, "test"); err == nil { t.Errorf("validateAPIKey(%q) unexpectedly succeeded", invalid) } diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 6b197c61..435ea118 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -7,6 +7,7 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; +use subtle::ConstantTimeEq; use super::{ApiImpl, Claims}; use crate::{api::proxy, types::SandboxId}; @@ -23,8 +24,11 @@ fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a HeaderVal impl ApiImpl { pub(crate) fn has_valid_api_key(&self, headers: &HeaderMap) -> bool { - single_header(headers, API_KEY_HEADER) - .is_some_and(|value| value.as_bytes() == self.api_key.as_bytes()) + single_header(headers, API_KEY_HEADER).is_some_and(|value| { + let candidate = value.as_bytes(); + let expected = self.api_key.as_bytes(); + candidate.len() == expected.len() && bool::from(candidate.ct_eq(expected)) + }) } pub(crate) fn traffic_access_token(&self, sandbox_id: SandboxId) -> String { @@ -43,7 +47,7 @@ impl ApiImpl { pub(crate) async fn require_auth( State(api_impl): State, - request: Request, + mut request: Request, next: Next, ) -> Response where @@ -57,23 +61,23 @@ where let api_impl = api_impl.as_ref(); let mut authorized = api_impl.has_valid_api_key(request.headers()); - if !authorized && proxy_request { + let mut envd_authorized = false; + if proxy_request { if let Some((sandbox_id, target_port)) = proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) { - authorized = api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); + authorized |= api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); let envd_candidate = single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) .and_then(|value| value.to_str().ok()); - if !authorized { - if let Some(candidate) = envd_candidate { - authorized = proxy::has_valid_envd_access_token( - api_impl, - sandbox_id, - target_port, - candidate, - ) - .await; - } + if let Some(candidate) = envd_candidate { + envd_authorized = proxy::has_valid_envd_access_token( + api_impl, + sandbox_id, + target_port, + candidate, + ) + .await; + authorized |= envd_authorized; } } } @@ -81,6 +85,9 @@ where if !authorized { return StatusCode::UNAUTHORIZED.into_response(); } + if !envd_authorized { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + } next.run(request).await } diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 76a8fdac..d275ed4d 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -56,8 +56,6 @@ struct ResolvedProxyRequest { sandbox_id: SandboxId, upstream_uri: Uri, original_host: Option, - target_port: u16, - envd_port: u16, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -149,10 +147,10 @@ where } pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { - match parse_host_proxy_route(request_host(request), domains) { - Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), - Err(_) => return None, - Ok(None) => {} + if !has_proxy_prefix(request.uri().path()) { + if let Ok(Some(route)) = parse_host_proxy_route(request_host(request), domains) { + return Some((route.sandbox_id, route.target_port)); + } } Some(( @@ -181,7 +179,7 @@ pub(crate) async fn has_valid_envd_access_token( pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { let path = request.uri().path(); - if path == PROXY_ROUTE || path.starts_with("/proxy/") { + if has_proxy_prefix(path) { return true; } @@ -216,7 +214,7 @@ where I: AsRef + Clone + Send + Sync + 'static, { let path = request.uri().path(); - if path == PROXY_ROUTE || path.starts_with("/proxy/") { + if has_proxy_prefix(path) { return next.run(request).await; } @@ -338,6 +336,10 @@ fn strip_proxy_prefix(path: &str) -> &str { path.strip_prefix(PROXY_ROUTE).unwrap_or("") } +fn has_proxy_prefix(path: &str) -> bool { + path == PROXY_ROUTE || path.starts_with("/proxy/") +} + fn parse_host_proxy_route( raw_host: Option<&str>, domains: &[String], @@ -437,11 +439,9 @@ async fn proxy_http_request( sandbox_id, upstream_uri, original_host, - target_port, - envd_port, } = resolved; - sanitize_request_headers(&mut parts.headers, target_port, envd_port); + sanitize_request_headers(&mut parts.headers); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -622,11 +622,9 @@ async fn proxy_websocket_request( sandbox_id, upstream_uri, original_host, - target_port, - envd_port, } = resolved; - sanitize_websocket_request_headers(&mut parts.headers, target_port, envd_port); + sanitize_websocket_request_headers(&mut parts.headers); inject_forwarded_headers( &mut parts.headers, original_host.as_ref(), @@ -807,14 +805,6 @@ async fn resolve_proxy_request( } }; - let metadata = api_impl - .orchestrator() - .get_sandbox(&sandbox_id) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; - let envd_port = effective_envd_port(&metadata); - let upstream_uri = if is_websocket_request { build_upstream_uri_with_scheme("ws", &target, target_port, proxy_path, parts.uri.query()) } else { @@ -826,8 +816,6 @@ async fn resolve_proxy_request( sandbox_id, upstream_uri, original_host: parts.headers.get(header::HOST).cloned(), - target_port, - envd_port, }) } @@ -1045,7 +1033,7 @@ fn build_upstream_uri_with_scheme( .map_err(|_| StatusCode::BAD_REQUEST) } -fn sanitize_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { +fn sanitize_request_headers(headers: &mut HeaderMap) { // These headers are only for the control-plane hop between the client and // AgentENV. Upstream sandbox services should not see them. headers.remove(SANDBOX_ID_HEADER); @@ -1055,14 +1043,11 @@ fn sanitize_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port headers.remove(API_KEY_HEADER); headers.remove(TRAFFIC_ACCESS_TOKEN_HEADER); headers.remove(header::HOST); - if target_port != envd_port { - headers.remove(ENVD_ACCESS_TOKEN_HEADER); - } remove_hop_by_hop_headers(headers); } -fn sanitize_websocket_request_headers(headers: &mut HeaderMap, target_port: u16, envd_port: u16) { - sanitize_request_headers(headers, target_port, envd_port); +fn sanitize_websocket_request_headers(headers: &mut HeaderMap) { + sanitize_request_headers(headers); headers.remove(header::SEC_WEBSOCKET_ACCEPT); headers.remove(header::SEC_WEBSOCKET_EXTENSIONS); headers.remove(header::SEC_WEBSOCKET_KEY); @@ -1841,11 +1826,7 @@ mod tests { HeaderValue::from_static("keep"), ); - sanitize_request_headers( - &mut headers, - 8080, - ConfigManager::global_config().tools.control_plane_port, - ); + sanitize_request_headers(&mut headers); assert!(headers.get(SANDBOX_ID_HEADER).is_none()); assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); diff --git a/src/api_key.rs b/src/api_key.rs index ec59037e..96ddc704 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -1,6 +1,6 @@ use std::ffi::OsStr; use std::fs::{self, File}; -use std::io::{self, Write}; +use std::io::{self, Read}; use std::path::Path; use anyhow::{bail, Context, Result}; @@ -8,10 +8,13 @@ use rand::{rngs::SysRng, TryRng}; use tracing::info; use crate::cfg::AppConfig; +use crate::managed_secret::{self, CreateOutcome}; const API_KEY_ENV: &str = "AENV_API_KEY"; const EXTERNAL_API_KEY_PATH: &str = "/run/secrets/api-key"; const MANAGED_API_KEY_RELATIVE_PATH: &str = "secrets/api-key"; +const API_KEY_MAX_LEN: usize = 4096; +const API_KEY_FILE_MAX_LEN: usize = API_KEY_MAX_LEN + 2; const GENERATED_API_KEY_PREFIX: &str = "e2b_"; pub fn resolve(config: &AppConfig) -> Result { @@ -36,7 +39,7 @@ fn resolve_from( .context("invalid AENV_API_KEY"); } - match read(external_path) { + match read_external(external_path) { Ok(key) => { info!(path = %external_path.display(), "loaded API key from external secret"); return Ok(key); @@ -46,80 +49,77 @@ fn resolve_from( } let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); - match read(&managed_path) { - Ok(key) => return Ok(key), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error).context("load managed API key"), + match fs::symlink_metadata(&managed_path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return create(&managed_path), + Err(error) => { + return Err(error) + .with_context(|| format!("inspect managed API key {}", managed_path.display())); + } + Ok(_) => {} + } + if let Some(value) = + managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN).context("load managed API key")? + { + return validate_file_contents(&value).context("invalid managed API key"); } create(&managed_path) } -fn read(path: &Path) -> Result { - let value = fs::read_to_string(path)?; - validate(&value).map_err(io::Error::other) +fn read_external(path: &Path) -> Result { + let value = read_bounded(File::open(path)?)?; + validate_file_contents(&value).map_err(io::Error::other) +} + +fn read_bounded(file: File) -> Result { + let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); + file.take((API_KEY_FILE_MAX_LEN + 1) as u64) + .read_to_string(&mut value)?; + if value.len() > API_KEY_FILE_MAX_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("API key file must be at most {API_KEY_FILE_MAX_LEN} bytes"), + )); + } + Ok(value) } fn validate(value: &str) -> Result { - let value = value.trim(); - if value.len() < 32 + if !(32..=API_KEY_MAX_LEN).contains(&value.len()) || !value .bytes() .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-')) { - bail!("API key must contain at least 32 URL-safe characters"); + bail!("API key must contain between 32 and {API_KEY_MAX_LEN} URL-safe characters"); } Ok(value.to_owned()) } -fn create(path: &Path) -> Result { - let parent = path - .parent() - .context("managed API key path has no parent")?; - fs::create_dir_all(parent) - .with_context(|| format!("create managed secret directory {}", parent.display()))?; - set_permissions(parent, 0o700)?; +fn validate_file_contents(value: &str) -> Result { + let value = value.strip_suffix('\n').unwrap_or(value); + validate(value.strip_suffix('\r').unwrap_or(value)) +} +fn create(path: &Path) -> Result { let mut random = [0_u8; 32]; SysRng .try_fill_bytes(&mut random) .context("generate managed API key")?; let key = format!("{GENERATED_API_KEY_PREFIX}{}", hex::encode(random)); - let mut temporary = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("create temporary API key in {}", parent.display()))?; - set_permissions(temporary.path(), 0o600)?; - writeln!(temporary, "{key}")?; - temporary.as_file().sync_all()?; - - match temporary.persist_noclobber(path) { - Ok(_) => { - File::open(parent)?.sync_all()?; + match managed_secret::create(path, format!("{key}\n").as_bytes())? { + CreateOutcome::Created => { info!(path = %path.display(), "generated managed API key"); Ok(key) } - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { - read(path).context("load concurrently generated API key") - } - Err(error) => { - Err(error.error).with_context(|| format!("persist managed API key {}", path.display())) + CreateOutcome::Existing(file) => { + let value = managed_secret::read_file(path, file, API_KEY_FILE_MAX_LEN) + .context("load concurrently generated API key")?; + validate_file_contents(&value).context("invalid concurrently generated API key") } } } -#[cfg(unix)] -fn set_permissions(path: &Path, mode: u32) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 14d02f7e..9223e1bf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod identity; pub mod image; mod local_store; pub mod logging; +mod managed_secret; pub mod observability; pub mod orchestrator; pub mod overlaybd; diff --git a/src/managed_secret.rs b/src/managed_secret.rs new file mode 100644 index 00000000..4c9f7b95 --- /dev/null +++ b/src/managed_secret.rs @@ -0,0 +1,233 @@ +#[cfg(unix)] +use std::fs::OpenOptions; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::Path; + +use anyhow::{bail, Context, Result}; + +pub(crate) enum CreateOutcome { + Created, + Existing(File), +} + +pub(crate) fn read(path: &Path, max_len: usize) -> Result> { + let parent = path.parent().context("managed secret path has no parent")?; + match validate_directory(parent) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| { + format!("validate managed secret directory {}", parent.display()) + }); + } + } + + match open(path) { + Ok(file) => read_file(path, file, max_len).map(Some), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).with_context(|| format!("open managed secret {}", path.display())), + } +} + +pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result { + let metadata = file + .metadata() + .with_context(|| format!("inspect managed secret {}", path.display()))?; + if !metadata.is_file() { + bail!("managed secret {} must be a regular file", path.display()); + } + + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let mode = metadata.permissions().mode() & 0o777; + if mode != 0o600 { + bail!( + "managed secret {} must have permissions 0600, found {mode:04o}", + path.display() + ); + } + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + bail!( + "managed secret {} must be owned by uid {expected_uid}, found uid {}", + path.display(), + metadata.uid() + ); + } + } + + if metadata.len() > max_len as u64 { + bail!( + "managed secret {} must be at most {max_len} bytes", + path.display() + ); + } + + let mut contents = String::with_capacity(max_len); + Read::by_ref(&mut file) + .take((max_len + 1) as u64) + .read_to_string(&mut contents) + .with_context(|| format!("read managed secret {}", path.display()))?; + if contents.len() > max_len { + bail!( + "managed secret {} must be at most {max_len} bytes", + path.display() + ); + } + Ok(contents) +} + +pub(crate) fn create(path: &Path, contents: &[u8]) -> Result { + ensure_supported()?; + let parent = path.parent().context("managed secret path has no parent")?; + create_directory(parent)?; + validate_directory_identity(parent).with_context(|| { + format!( + "validate managed secret directory ownership {}", + parent.display() + ) + })?; + set_permissions(parent, 0o700)?; + validate_directory(parent) + .with_context(|| format!("validate managed secret directory {}", parent.display()))?; + + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temporary secret in {}", parent.display()))?; + set_permissions(temporary.path(), 0o600)?; + temporary + .write_all(contents) + .with_context(|| format!("write temporary secret in {}", parent.display()))?; + temporary + .as_file() + .sync_all() + .with_context(|| format!("sync temporary secret in {}", parent.display()))?; + + match temporary.persist_noclobber(path) { + Ok(_) => { + File::open(parent) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("sync managed secret directory {}", parent.display()))?; + Ok(CreateOutcome::Created) + } + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { + validate_directory(parent).with_context(|| { + format!("validate managed secret directory {}", parent.display()) + })?; + open(path) + .map(CreateOutcome::Existing) + .with_context(|| format!("open managed secret {}", path.display())) + } + Err(error) => { + Err(error.error).with_context(|| format!("persist managed secret {}", path.display())) + } + } +} + +fn create_directory(path: &Path) -> Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + + builder.mode(0o700); + } + + builder + .create(path) + .with_context(|| format!("create managed secret directory {}", path.display())) +} + +#[cfg(unix)] +fn open(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + + use std::os::unix::fs::OpenOptionsExt; + + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + + options.open(path) +} + +#[cfg(not(unix))] +fn open(_path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "managed secrets require Unix no-follow file semantics", + )) +} + +#[cfg(unix)] +fn ensure_supported() -> Result<()> { + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_supported() -> Result<()> { + bail!("managed secrets require Unix no-follow file semantics") +} + +fn validate_directory(path: &Path) -> io::Result<()> { + let metadata = validate_directory_identity(path)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = metadata.permissions().mode() & 0o777; + if mode != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("must have permissions 0700, found {mode:04o}"), + )); + } + } + + Ok(()) +} + +fn validate_directory_identity(path: &Path) -> io::Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "must be a directory and not a symbolic link", + )); + } + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + + let expected_uid = nix::unistd::Uid::effective().as_raw(); + if metadata.uid() != expected_uid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "must be owned by uid {expected_uid}, found uid {}", + metadata.uid() + ), + )); + } + } + + Ok(metadata) +} + +#[cfg(unix)] +fn set_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .with_context(|| format!("set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index abf13788..b6de8b0d 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -1,6 +1,4 @@ use std::fmt; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Read, Write}; use std::path::Path; use anyhow::{bail, Context, Result}; @@ -10,6 +8,7 @@ use sha2::Sha256; use tracing::{info, warn}; use crate::cfg::AppConfig; +use crate::managed_secret::{self, CreateOutcome}; use crate::types::SandboxId; type HmacSha256 = Hmac; @@ -116,30 +115,8 @@ fn validate_explicit_seed(seed: &str) -> Result<&str> { } fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result { - let parent = managed_path - .parent() - .context("managed sandbox access-token seed path has no parent")?; - match validate_managed_seed_directory(parent) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!("validate managed secret directory {}", parent.display()) - }); - } - } - - match open_managed_seed(managed_path) { - Ok(file) => return read_managed_seed(managed_path, file), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).with_context(|| { - format!( - "open managed sandbox access-token seed {}", - managed_path.display() - ) - }); - } + if let Some(contents) = managed_secret::read(managed_path, MANAGED_SEED_FILE_MAX_LEN)? { + return validate_managed_seed(managed_path, &contents); } if managed_seed_must_exist { @@ -152,80 +129,8 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result io::Result { - let mut options = OpenOptions::new(); - options.read(true); - - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - - options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); - } - - options.open(path) -} - -fn validate_managed_seed_file(path: &Path, file: &File) -> Result { - let metadata = file.metadata().with_context(|| { - format!( - "inspect managed sandbox access-token seed {}", - path.display() - ) - })?; - if !metadata.is_file() { - bail!( - "managed sandbox access-token seed {} must be a regular file", - path.display() - ); - } - - #[cfg(unix)] - { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let mode = metadata.permissions().mode() & 0o777; - if mode != 0o600 { - bail!( - "managed sandbox access-token seed {} must have permissions 0600, found {mode:04o}", - path.display() - ); - } - let expected_uid = nix::unistd::Uid::effective().as_raw(); - if metadata.uid() != expected_uid { - bail!( - "managed sandbox access-token seed {} must be owned by uid {expected_uid}, found uid {}", - path.display(), - metadata.uid() - ); - } - } - - Ok(metadata) -} - -fn read_managed_seed(path: &Path, mut file: File) -> Result { - let metadata = validate_managed_seed_file(path, &file)?; - - if metadata.len() > MANAGED_SEED_FILE_MAX_LEN as u64 { - bail!( - "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", - path.display() - ); - } - - let mut contents = String::with_capacity(MANAGED_SEED_FILE_MAX_LEN); - Read::by_ref(&mut file) - .take((MANAGED_SEED_FILE_MAX_LEN + 1) as u64) - .read_to_string(&mut contents) - .with_context(|| format!("read managed sandbox access-token seed {}", path.display()))?; - if contents.len() > MANAGED_SEED_FILE_MAX_LEN { - bail!( - "managed sandbox access-token seed {} must be at most {MANAGED_SEED_FILE_MAX_LEN} bytes", - path.display() - ); - } - let seed = contents.strip_suffix('\n').unwrap_or(&contents); +fn validate_managed_seed(path: &Path, contents: &str) -> Result { + let seed = contents.strip_suffix('\n').unwrap_or(contents); if !is_valid_managed_seed(seed) { bail!( "managed sandbox access-token seed {} must contain exactly {SEED_HEX_LEN} lowercase hexadecimal characters, optionally followed by a newline", @@ -237,57 +142,21 @@ fn read_managed_seed(path: &Path, mut file: File) -> Result { } fn create_managed_seed(path: &Path) -> Result { - let parent = path - .parent() - .context("managed sandbox access-token seed path has no parent")?; - fs::create_dir_all(parent) - .with_context(|| format!("create managed secret directory {}", parent.display()))?; - validate_managed_seed_directory_identity(parent).with_context(|| { - format!( - "validate managed secret directory ownership {}", - parent.display() - ) - })?; - set_permissions(parent, 0o700)?; - validate_managed_seed_directory(parent) - .with_context(|| format!("validate managed secret directory {}", parent.display()))?; - let mut random = [0_u8; MANAGED_SEED_BYTES]; SysRng .try_fill_bytes(&mut random) .context("generate managed sandbox access-token seed")?; let seed = hex::encode(random); - let mut temporary = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("create temporary seed file in {}", parent.display()))?; - set_permissions(temporary.path(), 0o600)?; - writeln!(temporary, "{seed}") - .with_context(|| format!("write temporary seed file in {}", parent.display()))?; - temporary - .as_file() - .sync_all() - .with_context(|| format!("sync temporary seed file in {}", parent.display()))?; - - match temporary.persist_noclobber(path) { - Ok(_) => { - fs::File::open(parent) - .and_then(|directory| directory.sync_all()) - .with_context(|| format!("sync managed secret directory {}", parent.display()))?; + match managed_secret::create(path, format!("{seed}\n").as_bytes())? { + CreateOutcome::Created => { info!(path = %path.display(), "generated managed sandbox access-token seed"); Ok(seed) } - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { - let file = open_managed_seed(path).with_context(|| { - format!("open managed sandbox access-token seed {}", path.display()) - })?; - read_managed_seed(path, file) - } - Err(error) => Err(error.error).with_context(|| { - format!( - "persist managed sandbox access-token seed {}", - path.display() - ) - }), + CreateOutcome::Existing(file) => validate_managed_seed( + path, + &managed_secret::read_file(path, file, MANAGED_SEED_FILE_MAX_LEN)?, + ), } } @@ -298,66 +167,6 @@ fn is_valid_managed_seed(seed: &str) -> bool { .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } -fn validate_managed_seed_directory(path: &Path) -> io::Result<()> { - let metadata = validate_managed_seed_directory_identity(path)?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - let mode = metadata.permissions().mode() & 0o777; - if mode != 0o700 { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - format!("must have permissions 0700, found {mode:04o}"), - )); - } - } - - Ok(()) -} - -fn validate_managed_seed_directory_identity(path: &Path) -> io::Result { - let metadata = fs::symlink_metadata(path)?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "must be a directory and not a symbolic link", - )); - } - - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - - let expected_uid = nix::unistd::Uid::effective().as_raw(); - if metadata.uid() != expected_uid { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - format!( - "must be owned by uid {expected_uid}, found uid {}", - metadata.uid() - ), - )); - } - } - - Ok(metadata) -} - -#[cfg(unix)] -fn set_permissions(path: &Path, mode: u32) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { - Ok(()) -} - impl fmt::Debug for SandboxAccessTokenGenerator { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("SandboxAccessTokenGenerator()") @@ -367,13 +176,20 @@ impl fmt::Debug for SandboxAccessTokenGenerator { #[cfg(test)] mod tests { use super::*; + use std::fs; use std::sync::{Arc, Barrier}; use tempfile::TempDir; fn create_private_managed_seed_directory(path: &Path) -> Result<()> { fs::create_dir_all(path)?; - set_permissions(path, 0o700) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + } + Ok(()) } #[test] @@ -505,7 +321,7 @@ mod tests { for contents in ["", "invalid\n"] { fs::write(&managed_path, contents)?; - set_permissions(&managed_path, 0o600)?; + set_test_permissions(&managed_path, 0o600)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -522,7 +338,7 @@ mod tests { let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); create_private_managed_seed_directory(managed_path.parent().unwrap())?; fs::write(&managed_path, format!("{}\n", "a".repeat(64)))?; - set_permissions(&managed_path, 0o640)?; + set_test_permissions(&managed_path, 0o640)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -540,14 +356,12 @@ mod tests { let target_path = temp.path().join("seed-target"); create_private_managed_seed_directory(managed_path.parent().unwrap())?; fs::write(&target_path, format!("{}\n", "a".repeat(64)))?; - set_permissions(&target_path, 0o600)?; + set_test_permissions(&target_path, 0o600)?; symlink(&target_path, &managed_path)?; let error = resolve_seed(&managed_path, false).unwrap_err(); - assert!(error - .to_string() - .contains("open managed sandbox access-token seed")); + assert!(error.to_string().contains("open managed secret")); Ok(()) } @@ -556,9 +370,9 @@ mod tests { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); create_private_managed_seed_directory(managed_path.parent().unwrap())?; - let file = File::create(&managed_path)?; + let file = fs::File::create(&managed_path)?; file.set_len(1024 * 1024)?; - set_permissions(&managed_path, 0o600)?; + set_test_permissions(&managed_path, 0o600)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -572,7 +386,7 @@ mod tests { let temp = TempDir::new()?; let managed_path = temp.path().join(MANAGED_SEED_RELATIVE_PATH); fs::create_dir_all(managed_path.parent().unwrap())?; - set_permissions(managed_path.parent().unwrap(), 0o770)?; + set_test_permissions(managed_path.parent().unwrap(), 0o770)?; let error = resolve_seed(&managed_path, false).unwrap_err(); @@ -608,4 +422,17 @@ mod tests { assert!(!managed_path.exists()); Ok(()) } + + #[cfg(unix)] + fn set_test_permissions(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(mode))?; + Ok(()) + } + + #[cfg(not(unix))] + fn set_test_permissions(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) + } } From 32bf340ad0c4ea9dd19a9020684ee3d2e3d5d89f Mon Sep 17 00:00:00 2001 From: Yingdi Shan <5491399+yingdi-shan@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:02:40 +0000 Subject: [PATCH 6/6] fix: enforce authentication boundaries --- crates/aenv/src/client/files.rs | 21 +- crates/aenv/src/client/mod.rs | 2 +- crates/aenv/src/grpc/mod.rs | 22 +- deploy/docker-compose.yml | 2 + deploy/k8s/base/agentenv-daemonset.yaml | 2 + deploy/k8s/base/gateway-deployment.yaml | 2 + deploy/k8s/run.sh | 110 +++-- docs/src/concepts/proxy.md | 17 + docs/src/configuration/authentication.md | 21 +- docs/src/configuration/reference.md | 2 +- docs/src/deployment/kubernetes.md | 4 +- docs/src/integration/e2b.md | 13 +- .../persistence-artifact-inventory.md | 4 +- docs/src/internals/proxy-design.md | 24 + docs/src/security/secure-sandboxes.md | 2 +- scripts/install.sh | 2 +- scripts/tests/e2e/lib/helpers.sh | 42 +- scripts/tests/e2e/lib/runtime.sh | 35 +- scripts/tests/e2e/suites/06_proxy.sh | 16 +- scripts/tests/e2e/suites/08_auth.sh | 100 +++- scripts/tests/e2e/suites/11_node_metrics.sh | 22 +- services/README.md | 11 +- services/gateway/cmd/main.go | 24 +- services/gateway/cmd/main_test.go | 34 ++ services/gateway/internal/server.go | 54 ++- services/gateway/internal/server_test.go | 66 ++- src/api/generated/src/models.rs | 23 +- src/api/impls/auth.rs | 68 ++- src/api/impls/sandbox.rs | 42 +- src/api/openapi.yml | 10 +- src/api/proxy.rs | 430 ++++++++++++++---- src/api_key.rs | 62 ++- src/bin/server.rs | 4 +- src/managed_secret.rs | 365 +++++++++++---- src/orchestrator/service.rs | 21 +- src/orchestrator/tests.rs | 16 + src/sandbox/access.rs | 6 +- src/sandbox/firecracker/sandbox.rs | 22 +- src/sandbox/network/policy.rs | 40 +- 39 files changed, 1359 insertions(+), 404 deletions(-) diff --git a/crates/aenv/src/client/files.rs b/crates/aenv/src/client/files.rs index ac5173ae..39be8490 100644 --- a/crates/aenv/src/client/files.rs +++ b/crates/aenv/src/client/files.rs @@ -17,7 +17,6 @@ use super::Client; use crate::grpc::{RpcError, Transport, ENVD_PORT_STR}; use crate::progress::TransferProgress; -const API_KEY_HEADER: &str = "X-API-Key"; const SANDBOX_ID_HEADER: &str = "x-agentenv-sandbox-id"; const TARGET_PORT_HEADER: &str = "x-agentenv-target-port"; const ACCESS_TOKEN_HEADER: &str = "X-Access-Token"; @@ -32,17 +31,8 @@ pub struct EnvdFilesClient { } impl EnvdFilesClient { - fn new( - base_url: &str, - api_key: &str, - sandbox_id: &str, - envd_access_token: Option<&str>, - ) -> Result { + fn new(base_url: &str, sandbox_id: &str, envd_access_token: Option<&str>) -> Result { let mut headers = HeaderMap::new(); - headers.insert( - API_KEY_HEADER, - HeaderValue::from_str(api_key).context("invalid API key header value")?, - ); headers.insert( SANDBOX_ID_HEADER, HeaderValue::from_str(sandbox_id).context("invalid sandbox ID header value")?, @@ -72,7 +62,7 @@ impl EnvdFilesClient { Ok(Self { base_url: base_url.trim_end_matches('/').to_string(), http: client, - transport: Transport::new(base_url, api_key, sandbox_id, envd_access_token)?, + transport: Transport::new(base_url, sandbox_id, envd_access_token)?, }) } @@ -311,12 +301,7 @@ fn format_envd_response_error(status: reqwest::StatusCode, content: &str) -> any impl Client { pub fn files(&self, sandbox_id: &str) -> Result { let sandbox = self.get_sandbox(sandbox_id)?; - EnvdFilesClient::new( - &self.base, - &self.api_key, - sandbox_id, - sandbox.envd_access_token.as_deref(), - ) + EnvdFilesClient::new(&self.base, sandbox_id, sandbox.envd_access_token.as_deref()) } } diff --git a/crates/aenv/src/client/mod.rs b/crates/aenv/src/client/mod.rs index 55dba491..c384e964 100644 --- a/crates/aenv/src/client/mod.rs +++ b/crates/aenv/src/client/mod.rs @@ -40,7 +40,7 @@ impl Client { sandbox_id: &str, envd_access_token: Option<&str>, ) -> Result { - Transport::new(&self.base, &self.api_key, sandbox_id, envd_access_token) + Transport::new(&self.base, sandbox_id, envd_access_token) } fn url(&self, path: &str) -> String { diff --git a/crates/aenv/src/grpc/mod.rs b/crates/aenv/src/grpc/mod.rs index 64336c33..cdc24e83 100644 --- a/crates/aenv/src/grpc/mod.rs +++ b/crates/aenv/src/grpc/mod.rs @@ -54,23 +54,16 @@ impl std::error::Error for RpcError {} pub struct Transport { http: HttpClient, base_url: String, - api_key: String, sandbox_id: String, envd_access_token: Option, } impl Transport { - pub fn new( - base_url: &str, - api_key: &str, - sandbox_id: &str, - envd_access_token: Option<&str>, - ) -> Result { + pub fn new(base_url: &str, sandbox_id: &str, envd_access_token: Option<&str>) -> Result { let http = Self::http_client(base_url).context("building Connect-RPC HTTP client")?; Ok(Self { http, base_url: base_url.trim_end_matches('/').to_string(), - api_key: api_key.to_string(), sandbox_id: sandbox_id.to_string(), envd_access_token: envd_access_token.map(str::to_owned), }) @@ -102,7 +95,6 @@ impl Transport { fn auth(&self, builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { let builder = builder - .header("X-API-Key", &self.api_key) .header("x-agentenv-sandbox-id", &self.sandbox_id) .header("x-agentenv-target-port", ENVD_PORT_STR) .header("Connect-Protocol-Version", "1"); @@ -508,7 +500,7 @@ mod tests { #[test] fn unary_user_is_sent_as_basic_auth() { - let transport = Transport::new("http://127.0.0.1", "api-key", "sandbox-id", None).unwrap(); + let transport = Transport::new("http://127.0.0.1", "sandbox-id", None).unwrap(); let request = transport .unary_request("filesystem.Filesystem", "Stat", Some("app")) .build() @@ -518,6 +510,7 @@ mod tests { request.headers().get(AUTHORIZATION).unwrap(), "Basic YXBwOg==" ); + assert!(!request.headers().contains_key("x-api-key")); let request = transport .unary_request("filesystem.Filesystem", "Stat", None) @@ -528,13 +521,8 @@ mod tests { #[test] fn envd_access_token_is_sent_on_connect_requests() { - let transport = Transport::new( - "http://127.0.0.1", - "api-key", - "sandbox-id", - Some("envd-token"), - ) - .unwrap(); + let transport = + Transport::new("http://127.0.0.1", "sandbox-id", Some("envd-token")).unwrap(); let request = transport .unary_request("process.Process", "List", None) diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index fccc7127..453be6ca 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -3,6 +3,7 @@ x-agentenv-base: &agentenv-base init: true working_dir: /workspace environment: &agentenv-environment + AENV_API_KEY: AENV_CONFIG_PATH: /workspace/config/default.toml AENV_VIRTUALIZATION_MODE: ${AENV_VIRTUALIZATION_MODE:-kvm} API_ADDR: 0.0.0.0:8000 @@ -74,6 +75,7 @@ services: - ./docker/config/default.json:/config/default.json:ro - agentenv-auth:/run/secrets:ro environment: + AENV_API_KEY: GATEWAY_HTTP_LISTEN_ADDR: :8080 GATEWAY_SCHEDULER_ADDR: scheduler:9090 GATEWAY_SANDBOX_PROXY_DOMAINS: ${SANDBOX_PROXY_DOMAINS:-} diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index 470b2a9c..1489880a 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -10,6 +10,8 @@ spec: app.kubernetes.io/name: agentenv-node template: metadata: + annotations: + agentenv.dev/api-key-checksum: "" labels: app.kubernetes.io/name: agentenv-node spec: diff --git a/deploy/k8s/base/gateway-deployment.yaml b/deploy/k8s/base/gateway-deployment.yaml index 08536fc0..5991d9c8 100644 --- a/deploy/k8s/base/gateway-deployment.yaml +++ b/deploy/k8s/base/gateway-deployment.yaml @@ -11,6 +11,8 @@ spec: app.kubernetes.io/name: agentenv-gateway template: metadata: + annotations: + agentenv.dev/api-key-checksum: "" labels: app.kubernetes.io/name: agentenv-gateway spec: diff --git a/deploy/k8s/run.sh b/deploy/k8s/run.sh index 93d9cdd6..f640b288 100644 --- a/deploy/k8s/run.sh +++ b/deploy/k8s/run.sh @@ -50,6 +50,26 @@ render_api_key() { mv "${rendered_file}" "${file}" } +sha256_text() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 | awk '{print $1}' + else + echo "sha256sum or shasum is required" >&2 + return 1 + fi +} + +render_api_key_checksum() { + local checksum="$1" + local file="$2" + + sed_in_place \ + "s#agentenv.dev/api-key-checksum: \"\"#agentenv.dev/api-key-checksum: \"${checksum}\"#" \ + "${file}" +} + cp -R "${SCRIPT_DIR}" "${TEMP_DIR}/k8s" cp "${REPO_ROOT}/config/default.toml" "${TEMP_DIR}/k8s/base/config/agentenv.toml" OVERLAY_PATH="${TEMP_DIR}/k8s/overlays/${OVERLAY_NAME}" @@ -62,27 +82,52 @@ sed_in_place "s#^namespace: agentenv-system#namespace: ${NAMESPACE}#" "${OVERLAY sed_in_place "s# name: agentenv-system# name: ${NAMESPACE}#" "${TEMP_DIR}/k8s/base/namespace.yaml" sed_in_place "s#\"namespace\": \"agentenv-system\"#\"namespace\": \"${NAMESPACE}\"#" "${TEMP_DIR}/k8s/base/config/scheduler.json" -namespace_name="" -if [[ "${MODE}" == "apply" ]]; then - if ! namespace_name="$("${KUBECTL_BIN}" get namespace "${NAMESPACE}" --ignore-not-found -o name)"; then - echo "failed to check namespace ${NAMESPACE}" >&2 - exit 1 +read_existing_api_key() { + local value="" + + if ! value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ + --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY" | base64decode}}')"; then + echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 + return 1 fi -fi + printf '%s' "${value}" +} -read_existing_api_key() { - local encoded_value="" +ensure_namespace() { + if ! "${KUBECTL_BIN}" create namespace "${NAMESPACE}" \ + --dry-run=client -o yaml | "${KUBECTL_BIN}" apply -f - >/dev/null; then + echo "failed to create or verify namespace ${NAMESPACE}" >&2 + return 1 + fi +} + +bootstrap_api_key() { + local create_error secret_file - if [[ -z "${namespace_name}" ]]; then + if ! API_KEY_VALUE="$(read_existing_api_key)"; then + return 1 + fi + if [[ -n "${API_KEY_VALUE}" ]]; then return 0 fi - if ! encoded_value="$("${KUBECTL_BIN}" -n "${NAMESPACE}" get secret agentenv-auth \ - --ignore-not-found -o 'go-template={{index .data "AENV_API_KEY"}}')"; then - echo "failed to read AENV_API_KEY from Secret ${NAMESPACE}/agentenv-auth" >&2 + + secret_file="${TEMP_DIR}/bootstrap-api-key" + create_error="${TEMP_DIR}/bootstrap-api-key.err" + printf 'e2b_%s' "$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" >"${secret_file}" + chmod 0600 "${secret_file}" + + # A concurrent apply can win this create. The persisted reread below is + # authoritative whether this command succeeds or reports AlreadyExists. + "${KUBECTL_BIN}" -n "${NAMESPACE}" create secret generic agentenv-auth \ + --from-file="AENV_API_KEY=${secret_file}" >/dev/null 2>"${create_error}" || true + + if ! API_KEY_VALUE="$(read_existing_api_key)"; then return 1 fi - if [[ -n "${encoded_value}" ]]; then - printf '%s' "${encoded_value}" | base64 -d + if [[ -z "${API_KEY_VALUE}" ]]; then + cat "${create_error}" >&2 + echo "failed to bootstrap Secret ${NAMESPACE}/agentenv-auth" >&2 + return 1 fi } @@ -93,21 +138,29 @@ if [[ "${MODE}" != "delete" ]]; then set +x fi API_KEY_VALUE="" - if [[ "${AENV_API_KEY+x}" == "x" ]]; then - API_KEY_VALUE="${AENV_API_KEY}" - elif ! API_KEY_VALUE="$(read_existing_api_key)"; then - exit 1 - fi - - if [[ -z "${API_KEY_VALUE}" ]]; then - API_KEY_VALUE="e2b_$(od -An -N32 -tx1 /dev/urandom | tr -d '[:space:]')" - fi - if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then - echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 - exit 1 + if [[ "${MODE}" == "render" ]]; then + API_KEY_VALUE="REDACTED" + else + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + if [[ -z "${AENV_API_KEY}" ]]; then + echo "AENV_API_KEY must not be empty" >&2 + exit 1 + fi + API_KEY_VALUE="${AENV_API_KEY}" + else + ensure_namespace || exit 1 + bootstrap_api_key || exit 1 + fi + if [[ ! "${API_KEY_VALUE}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]]; then + echo "AENV_API_KEY must contain between 32 and 4096 URL-safe characters" >&2 + exit 1 + fi fi render_api_key "${TEMP_DIR}/k8s/base/kustomization.yaml" + API_KEY_CHECKSUM="$(printf '%s' "${API_KEY_VALUE}" | sha256_text)" + render_api_key_checksum "${API_KEY_CHECKSUM}" "${TEMP_DIR}/k8s/base/agentenv-daemonset.yaml" + render_api_key_checksum "${API_KEY_CHECKSUM}" "${TEMP_DIR}/k8s/base/gateway-deployment.yaml" [[ "${restore_xtrace}" == "0" ]] || set -x fi @@ -143,7 +196,10 @@ case "${MODE}" in "${KUBECTL_BIN}" apply -k "${OVERLAY_PATH}" "$@" echo "AgentENV API key stored in Secret ${NAMESPACE}/agentenv-auth." >&2 echo "Read it with:" >&2 - echo " ${KUBECTL_BIN} -n ${NAMESPACE} get secret agentenv-auth -o go-template='{{index .data \"AENV_API_KEY\" | base64decode}}{{\"\\n\"}}'" >&2 + printf " %s -n %s get secret agentenv-auth -o go-template='%s'\n" \ + "${KUBECTL_BIN}" \ + "${NAMESPACE}" \ + '{{index .data "AENV_API_KEY" | base64decode}}{{"\n"}}' >&2 ;; delete) "${KUBECTL_BIN}" delete --ignore-not-found -k "${OVERLAY_PATH}" "$@" diff --git a/docs/src/concepts/proxy.md b/docs/src/concepts/proxy.md index 20ebff43..f0368b77 100644 --- a/docs/src/concepts/proxy.md +++ b/docs/src/concepts/proxy.md @@ -33,6 +33,23 @@ E2B-compatible aliases are also accepted: These routing headers are stripped before the request is forwarded to the sandbox. +## Access Control + +Proxy authentication is independent from AgentENV API authentication: + +- Public application ingress (`allowPublicTraffic: true`, the default) requires + no AgentENV credential. +- Private application ingress (`allowPublicTraffic: false`) requires the + sandbox's `trafficAccessToken` in `e2b-traffic-access-token`. +- Secure envd traffic requires the sandbox's `envdAccessToken` in + `X-Access-Token`. Insecure envd traffic has no envd token. + +`X-API-Key` authenticates AgentENV control-plane APIs only. It does not grant +access to private application ingress or secure envd, and it is stripped before +proxy requests reach the sandbox. AgentENV also strips the traffic token before +forwarding application requests, and forwards `X-Access-Token` only to the +matching secure envd port. + Host-based proxy requests derive both values from `Host`, for example `http://8080-.sandbox.example.com/health` targets port `8080`. The configured domain must route to the AgentENV server in single-node mode or diff --git a/docs/src/configuration/authentication.md b/docs/src/configuration/authentication.md index 4d4349f5..fbdfc5b9 100644 --- a/docs/src/configuration/authentication.md +++ b/docs/src/configuration/authentication.md @@ -12,17 +12,26 @@ X-API-Key: `Authorization`, `X-Admin-Token`, and `X-Team-ID` do not authenticate AgentENV. The `Authorization` header is left unchanged when a request is proxied into a sandbox, so applications inside a sandbox can use it normally. -`GET /health` is public for load balancer and container health checks. - -E2B SDK users set `E2B_API_KEY` to the same value. Sandbox create responses -include an independent `trafficAccessToken`; send it as -`e2b-traffic-access-token` on application proxy requests. The token is scoped to -the sandbox and is not accepted for control-plane API calls. +`GET /health` is public for load balancer and container health checks. The +node Prometheus `GET /metrics` endpoint and the gateway's separate Prometheus +listener are also outside application API-key auth; protect them with the +authentication and network controls used by your Prometheus deployment. These +endpoints are distinct from E2B's authenticated sandbox metrics API. + +E2B SDK users set `E2B_API_KEY` to the same value. Sandboxes created with +`network.allowPublicTraffic: false` receive an independent +`trafficAccessToken`; send it as `e2b-traffic-access-token` on application proxy +requests. Public application ingress does not require an AgentENV credential. +The token is scoped to the sandbox and is not accepted for control-plane API +calls or envd communication. For secure sandboxes, `envdAccessToken` is a separate credential for envd control traffic and must be sent as `X-Access-Token` only when targeting the envd control-plane port. It is absent for insecure sandboxes. +`X-API-Key` is not a sandbox proxy credential and is stripped before a proxied +request reaches the sandbox. + Both sandbox credentials are derived from the sandbox ID and one independent `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED`. They are not derived from the API key. diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index 6f646576..7c4b609d 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -259,7 +259,7 @@ Sandbox control communication settings. |-----|------|---------|-------------| | `access_token_hash_seed` | string | auto-generated | Optional override for the secret used to derive sandbox envd and traffic access tokens. When unset, normal server startup creates and reuses `$AENV_HOME/secrets/sandbox-access-token-hash-seed`. Configure an explicit shared value for clustered deployments. | -The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. +The managed seed is node-local persistent state and must be included in backups of `$AENV_HOME`. AgentENV refuses to generate a replacement when persisted secure or private-ingress sandboxes exist. An explicit environment or TOML value takes precedence over the managed file; changing that effective value invalidates existing sandbox access tokens. Configure `AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED` with the same value on every runtime node in a clustered deployment. Standalone runtime nodes use their managed seed when it is unset. diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 6d7ec4b0..747e3de2 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -70,8 +70,8 @@ kubectl -n agentenv-system get secret agentenv-auth \ ``` Set `AENV_API_KEY` when applying to supply your own value. A standalone -`make k8s-render` uses a temporary generated value because it does not modify or -read cluster state. The optional runtime seed keeps its existing +`make k8s-render` uses an invalid `REDACTED` placeholder so preview output never +contains a deployable API key. The optional runtime seed keeps its existing `agentenv-runtime-secrets` contract described in [Secure Sandboxes](../security/secure-sandboxes.md). diff --git a/docs/src/integration/e2b.md b/docs/src/integration/e2b.md index 653f3037..619bd958 100644 --- a/docs/src/integration/e2b.md +++ b/docs/src/integration/e2b.md @@ -15,12 +15,13 @@ export E2B_SANDBOX_URL=${E2B_API_URL} export E2B_API_KEY=${AENV_API_KEY} ``` -No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` for -application proxy traffic and (for secure sandboxes) `envdAccessToken` for envd -control traffic. These credentials have different headers and trust boundaries: -use `e2b-traffic-access-token` for application routes and `X-Access-Token` only -for envd. This is transport data, not the deprecated user-supplied -`E2B_ACCESS_TOKEN`. +No `E2B_ACCESS_TOKEN` is needed. AgentENV returns `trafficAccessToken` when +`network.allowPublicTraffic` is false and (for secure sandboxes) +`envdAccessToken` for envd control traffic. These credentials have different +headers and trust boundaries: use `e2b-traffic-access-token` for private +application routes and `X-Access-Token` only for envd. Public application +routes require neither token. This is transport data, not the deprecated +user-supplied `E2B_ACCESS_TOKEN`. ### TypeScript SDK diff --git a/docs/src/internals/persistence-artifact-inventory.md b/docs/src/internals/persistence-artifact-inventory.md index 8cf25904..e1291be9 100644 --- a/docs/src/internals/persistence-artifact-inventory.md +++ b/docs/src/internals/persistence-artifact-inventory.md @@ -9,7 +9,7 @@ This document lists AgentENV artifacts that can remain on disk or in object stor | `home_path` | `/var/lib/aenv` | `src/cfg.rs` | Base for paths containing the literal `$AENV_HOME` placeholder. `AENV_HOME_PATH` overrides it before placeholder expansion. | | `runtime_path` | `/run/aenv` | `src/cfg.rs`, `src/sandbox/network/*` | Base for transient namespace mount points and daemon sockets. `AENV_RUNTIME_PATH` overrides it. | | `deps_path` | `$AENV_HOME/deps` | `src/cfg.rs`, `src/setup/*` | Base for downloaded runtime dependencies. `AENV_DEPS_PATH` can place these rebuildable assets outside `home_path`. | -| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted sandboxes. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | `src/sandbox/access.rs` | Node-local secret used to derive envd and traffic tokens when `[sandbox].access_token_hash_seed` is unset. It must be preserved with persisted secure or private-ingress sandboxes. | | Firecracker sandbox work dirs | `$AENV_HOME/firecracker-work` with `agentenv-fc-` children | `src/sandbox/firecracker/*` | Per-sandbox runtime directories for sockets, symlinks, ublk runtime dirs, local logs, and writable OverlayBD upper layer data (`overlaybd/upper.data`, `overlaybd/upper.index`). An explicit `[firecracker].work_dir` overrides the root. | | `firecracker.serial_dir` | `$AENV_HOME/logs/serial` | `src/sandbox/firecracker/*` | Durable Firecracker stdout/stderr root, grouped by sandbox ID. An explicit `[firecracker].serial_dir` overrides the root. | | `managed_snapshot_root` | `/managed-snapshots` | `src/sandbox/firecracker/*` | In-process live snapshot artifact root used to keep captured snapshots alive until publish or drop. | @@ -37,7 +37,7 @@ Owned by `src/setup/*` and `src/cfg.rs`. | Overlaybd package downloads | `/overlaybd/downloads/*` | Temporary downloaded package archives | Setup staging for overlaybd release packages | Removed after a successful install. | | Generated overlaybd config | `$AENV_HOME/overlaybd/overlaybd-global.json`, `$AENV_HOME/overlaybd/mem-overlaybd-global.json`, `$AENV_HOME/overlaybd/convert-overlaybd-global.json`, `$AENV_HOME/overlaybd/resize-overlaybd-global.json` | Runtime global config, cache path, credentials config | Configures overlaybd runtime, memory snapshot overlaybd access, and the offline C++ tools (`overlaybd-apply`, `overlaybd-resize`), which get dedicated configs with isolated cacheDirs (`convert-blocks`, `resize-blocks`) and download disabled | Rewritten during setup/startup. | | Overlaybd runtime log | `$AENV_HOME/overlaybd/overlaybd.log` | Overlaybd runtime logs | Debugging | Appended by overlaybd runtime; no automatic GC. | -| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while sandboxes are persisted. | +| Managed sandbox access-token seed | `$AENV_HOME/secrets/sandbox-access-token-hash-seed` | 32 random bytes encoded as lowercase hexadecimal | Derives stable per-sandbox envd and traffic access tokens when no explicit seed is configured | Atomically created with mode `0600` during normal startup and reused thereafter. Must not be deleted while secure or private-ingress sandboxes are persisted. | ## Firecracker Sandbox diff --git a/docs/src/internals/proxy-design.md b/docs/src/internals/proxy-design.md index 3fff71da..ac96d2f1 100644 --- a/docs/src/internals/proxy-design.md +++ b/docs/src/internals/proxy-design.md @@ -43,6 +43,23 @@ Validation: - Sandbox ID must be a valid UUID format. - Target port must parse as `u16` and be greater than `0`. +Authorization is evaluated by the owning runtime after route parsing: + +- Control-plane routes require the deployment `X-API-Key` and do not accept + sandbox credentials. +- Node Prometheus `/metrics` and health `/health` are public to application + auth and should be protected separately at the deployment boundary. +- Non-envd application routes require `e2b-traffic-access-token` only when the + sandbox has private ingress (`allowPublicTraffic: false`). +- The envd port requires `X-Access-Token` only for secure sandboxes. +- `X-API-Key` is never a data-plane credential. + +The distributed gateway deliberately does not make sandbox authorization +decisions. It routes data-plane requests, including public ingress and insecure +envd requests with no credential, to the owning runtime. The runtime has the +sandbox metadata needed to apply the policy and performs the authoritative +token validation. + Host-based routing derives both fields from `Host`. The configured domain must match exactly after lowercase normalization and optional trailing-dot removal. Sandbox IDs in host routes must be valid UUIDs and the target port must fit in @@ -135,6 +152,13 @@ Control-plane routing headers are stripped before forwarding upstream: - `e2b-sandbox-id` - `e2b-sandbox-port` +Sandbox credential handling: + +- `e2b-traffic-access-token` is stripped before forwarding. +- A successfully validated secure-envd `X-Access-Token` is forwarded to envd; + otherwise that header is stripped. +- `X-API-Key` is stripped because it is a platform control-plane credential. + Hop-by-hop headers are stripped on both request and response paths, including: - Standard hop-by-hop headers (`Connection`, `Upgrade`, `TE`, `Trailer`, `Transfer-Encoding`, `Proxy-Authenticate`, `Proxy-Authorization`, `Keep-Alive`) diff --git a/docs/src/security/secure-sandboxes.md b/docs/src/security/secure-sandboxes.md index 32e38750..764a8771 100644 --- a/docs/src/security/secure-sandboxes.md +++ b/docs/src/security/secure-sandboxes.md @@ -13,7 +13,7 @@ Set `secure: true` when creating a sandbox through API or E2B-compatible SDKs to aenv start --secure ``` -The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. The application proxy credential is independent and is sent as `e2b-traffic-access-token`. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. +The API and SDKs return the sandbox's `envdAccessToken` where appropriate and attach it to envd requests automatically. Private application ingress has an independent `trafficAccessToken`, sent as `e2b-traffic-access-token`; public application ingress has no AgentENV credential. Forked sandboxes get independent credentials. Secure mode is preserved across pause, restart, and resume; legacy sandboxes remain non-secure unless created with `secure: true`. ## Access-Token Seed diff --git a/scripts/install.sh b/scripts/install.sh index 9ccff8e0..254bd17c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -371,7 +371,7 @@ echo " CLI : ${INSTALL_DIR}/aenv" echo " Server : ${INSTALL_DIR}/server" echo " Data : ${DATA_DIR}" echo " Config : ${CONFIG_PATH}" -echo " API key: generated on first server start in ${DATA_DIR}/secrets/api-key" +echo " API key: ${DATA_DIR}/secrets/api-key (auto-generated when no API key is configured)" echo " Mode : ${VIRTUALIZATION_MODE}" if [[ -d /run/systemd/system ]]; then if [[ "$ENV_FILE_STATUS" == "written" ]]; then diff --git a/scripts/tests/e2e/lib/helpers.sh b/scripts/tests/e2e/lib/helpers.sh index f5803b70..e5c088f4 100644 --- a/scripts/tests/e2e/lib/helpers.sh +++ b/scripts/tests/e2e/lib/helpers.sh @@ -8,6 +8,7 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" : "${AENV_TEMPLATE_ID:=ubuntu}" : "${AENV_PROXY_URL:=${AENV_URL}/proxy}" + : "${AENV_ENVD_PORT:=49983}" : "${E2E_MODE:=single-node}" : "${E2E_DEFAULT_USER_IMAGE:=ghcr.io/linuxserver/baseimage-ubuntu:noble}" : "${E2E_TEMPLATE_USER_IMAGE:=${E2E_DEFAULT_USER_IMAGE}}" @@ -120,44 +121,6 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then HTTP_HEADERS=$(<"$_E2E_HEADERS") } - api_admin_get() { - local path="$1" - _curl_do -s \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${AENV_URL}${path}" - } - - api_admin_get_at() { - local base_url="$1" - local path="$2" - _curl_do -s \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${base_url}${path}" - } - - api_admin_get_with_headers() { - local path="$1" - [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) - curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${AENV_URL}${path}" > "$_E2E_STATUS" 2>/dev/null || true - HTTP_STATUS=$(<"$_E2E_STATUS") - HTTP_BODY=$(<"$_E2E_BODY") - HTTP_HEADERS=$(<"$_E2E_HEADERS") - } - - api_admin_get_with_headers_at() { - local base_url="$1" - local path="$2" - [[ -z "$_E2E_HEADERS" ]] && _E2E_HEADERS=$(mktemp) - curl -s -o "$_E2E_BODY" -D "$_E2E_HEADERS" -w '%{http_code}' \ - -H "X-API-Key: ${AENV_API_KEY}" \ - "${base_url}${path}" > "$_E2E_STATUS" 2>/dev/null || true - HTTP_STATUS=$(<"$_E2E_STATUS") - HTTP_BODY=$(<"$_E2E_BODY") - HTTP_HEADERS=$(<"$_E2E_HEADERS") - } - api_post() { local path="$1" local body="${2:-}" @@ -208,9 +171,8 @@ if [[ -z "${E2E_HELPERS_SH_LOADED:-}" ]]; then proxy_get_with_sandbox() { local sandbox_id="$1" local path="${2:-/health}" - local target_port="${3:-49983}" + local target_port="${3:-${AENV_ENVD_PORT}}" _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${sandbox_id}" \ -H "x-agentenv-target-port: ${target_port}" \ "${AENV_PROXY_URL}${path}" diff --git a/scripts/tests/e2e/lib/runtime.sh b/scripts/tests/e2e/lib/runtime.sh index 4eb3e1e9..a065cb10 100644 --- a/scripts/tests/e2e/lib/runtime.sh +++ b/scripts/tests/e2e/lib/runtime.sh @@ -7,11 +7,19 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then E2E_RUNTIME_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" E2E_REPO_ROOT="$(cd "${E2E_RUNTIME_DIR}/../../../.." && pwd)" + _E2E_API_KEY_FROM_USER=0 + if [[ "${AENV_API_KEY+x}" == "x" ]]; then + if [[ -z "${AENV_API_KEY}" ]]; then + echo "AENV_API_KEY must not be empty" >&2 + return 1 + fi + _E2E_API_KEY_FROM_USER=1 + fi + # shellcheck source=/dev/null source "${E2E_RUNTIME_DIR}/server.sh" : "${E2E_MODE:=single-node}" - : "${AENV_API_KEY:=e2b_0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" export AENV_API_KEY : "${E2E_COMPOSE_FILE:=deploy/docker-compose.yml}" : "${E2E_COMPOSE_OVERRIDE_FILE:=scripts/tests/e2e/docker-compose.e2e.yml}" @@ -61,7 +69,11 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then } _deploy_make_cmd() { - make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + if [[ "${_E2E_API_KEY_FROM_USER}" == "1" ]]; then + make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + else + env -u AENV_API_KEY make --no-print-directory -C "${E2E_REPO_ROOT}" "$@" + fi } _run_deploy_target() { @@ -156,7 +168,12 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _run_k8s_target() { local target="${1:?usage: _run_k8s_target }" [[ -n "${target}" && "${target}" != "none" ]] || return 0 - make --no-print-directory -C "${E2E_REPO_ROOT}" "${target}" + _deploy_make_cmd "${target}" + } + + _read_k8s_api_key() { + kubectl -n "${E2E_K8S_NAMESPACE}" get secret agentenv-auth \ + -o 'go-template={{index .data "AENV_API_KEY" | base64decode}}' } _resolve_runtime_path() { @@ -398,8 +415,10 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then _wait_for_health_url "agentenv-b" "${AENV_NODE_B_URL}" "${timeout}" || die "agentenv-b failed to become ready within ${timeout}s" - AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || - die "Failed to read the Compose deployment API key" + if [[ "${_E2E_API_KEY_FROM_USER}" != "1" ]]; then + AENV_API_KEY="$(_compose_cmd exec -T agentenv-a cat /workspace/env/secrets/api-key)" || + die "Failed to read the Compose deployment API key" + fi [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || die "Compose deployment returned an invalid API key" export AENV_API_KEY @@ -440,6 +459,12 @@ if [[ -z "${E2E_RUNTIME_SH_LOADED:-}" ]]; then die "${label} failed to become ready within ${timeout}s" done < <(printf '%s\n' "${AENV_NODE_URLS}" | tr ' ' '\n') + AENV_API_KEY="$(_read_k8s_api_key)" || + die "Failed to read the Kubernetes deployment API key" + [[ "${AENV_API_KEY}" =~ ^[A-Za-z0-9._~-]{32,4096}$ ]] || + die "Kubernetes deployment returned an invalid API key" + export AENV_API_KEY + expected_nodes="$(_runtime_node_count)" [[ "${expected_nodes}" -gt 0 ]] || expected_nodes=1 _wait_for_scheduler_ready_nodes "${timeout}" "${expected_nodes}" || diff --git a/scripts/tests/e2e/suites/06_proxy.sh b/scripts/tests/e2e/suites/06_proxy.sh index da9903e8..e500e67c 100755 --- a/scripts/tests/e2e/suites/06_proxy.sh +++ b/scripts/tests/e2e/suites/06_proxy.sh @@ -12,24 +12,23 @@ log "Suite: Proxy Routing" sandbox_id=$(create_sandbox); _sync_http assert_status "$HTTP_STATUS" "201" "create sandbox for proxy test" assert_not_empty "$sandbox_id" "sandboxID present" +assert_json_field "$HTTP_BODY" '.trafficAccessToken' "null" "public sandbox omits traffic token" track_sandbox "$sandbox_id" wait_for_sandbox_state "$sandbox_id" "running" 30 # -- Proxy request using AgentENV headers -- -# The envd health endpoint runs on port 49983 inside the sandbox. +# The envd health endpoint runs on the configured control-plane port. _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${sandbox_id}" \ - -H "x-agentenv-target-port: 49983" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (agentenv headers) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "proxy with agentenv headers" # -- Proxy request using E2B compat headers -- _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "e2b-sandbox-id: ${sandbox_id}" \ - -H "e2b-sandbox-port: 49983" \ + -H "e2b-sandbox-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (e2b headers) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "proxy with e2b headers" @@ -38,7 +37,6 @@ assert_status "$HTTP_STATUS" "204" "proxy with e2b headers" # Use the explicit /proxy path so both single-node and compose reach the # node-local proxy entrypoint before header validation. _curl_do -s --max-time 5 \ - -H "X-API-Key: ${AENV_API_KEY}" \ "${AENV_URL}/proxy/health" log "Proxy (no sandbox header) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "400" "proxy without sandbox header" @@ -59,9 +57,8 @@ else fi _curl_do -s --max-time 10 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "x-agentenv-sandbox-id: ${paused_no_resume_id}" \ - -H "x-agentenv-target-port: 49983" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (paused + auto-resume disabled) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "410" "paused sandbox without auto-resume returns 410" @@ -83,9 +80,8 @@ fi # Auto-resume may wait up to 60s in non-test runtime. Keep client timeout above that. _curl_do -s --max-time 75 \ - -H "X-API-Key: ${AENV_API_KEY}" \ -H "e2b-sandbox-id: ${paused_auto_resume_id}" \ - -H "e2b-sandbox-port: 49983" \ + -H "e2b-sandbox-port: ${AENV_ENVD_PORT}" \ "${AENV_PROXY_URL}/health" log "Proxy (paused + auto-resume enabled) returned HTTP ${HTTP_STATUS}" assert_status "$HTTP_STATUS" "204" "paused sandbox auto-resumes on proxy request" diff --git a/scripts/tests/e2e/suites/08_auth.sh b/scripts/tests/e2e/suites/08_auth.sh index 47837517..666dbcac 100755 --- a/scripts/tests/e2e/suites/08_auth.sh +++ b/scripts/tests/e2e/suites/08_auth.sh @@ -13,7 +13,7 @@ api_get_no_auth "/sandboxes" assert_status "$HTTP_STATUS" "401" "no auth header returns 401" # -- Alternative and malformed credentials are rejected -- -_curl_do -s -H "X-API-Key: wrong-key" "${AENV_URL}/sandboxes" +_curl_do -s -H "X-API-Key: ${AENV_API_KEY}x" "${AENV_URL}/sandboxes" assert_status "$HTTP_STATUS" "401" "wrong API key returns 401" _curl_do -s -H "Authorization: Bearer ${AENV_API_KEY}" "${AENV_URL}/sandboxes" @@ -27,16 +27,108 @@ assert_status "$HTTP_STATUS" "401" "legacy team key does not authenticate AgentE _curl_do -s \ -H "X-API-Key: ${AENV_API_KEY}" \ - -H "X-API-Key: ${AENV_API_KEY}" \ + -H "X-API-Key: ${AENV_API_KEY}x" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "valid and invalid API key headers return 401" + +_curl_do -s \ + -H "X-API-Key: ${AENV_API_KEY}x" \ + -H "X-API-Key: ${AENV_API_KEY}y" \ "${AENV_URL}/sandboxes" -assert_status "$HTTP_STATUS" "401" "duplicate API key headers return 401" +assert_status "$HTTP_STATUS" "401" "conflicting invalid API key headers return 401" # -- Request with valid API key succeeds -- api_get "/sandboxes" assert_not_eq "$HTTP_STATUS" "401" "valid API key does not return 401" -# -- Health endpoint works without auth -- +# -- Sandbox-scoped credentials cannot authenticate the control plane -- +secure_sandbox_id=$(create_sandbox "$AENV_TEMPLATE_ID" 60 \ + '{"secure":true,"network":{"allowPublicTraffic":false}}'); _sync_http +if [[ -n "$secure_sandbox_id" ]]; then + track_sandbox "$secure_sandbox_id" +fi +assert_status "$HTTP_STATUS" "201" "create private secure sandbox" +assert_not_empty "$secure_sandbox_id" "private secure sandbox ID present" + +if [[ "$HTTP_STATUS" != "201" || -z "$secure_sandbox_id" ]]; then + suite_summary "08_auth" || true + exit 1 +fi + +traffic_access_token=$(echo "$HTTP_BODY" | jq -r '.trafficAccessToken // empty') +envd_access_token=$(echo "$HTTP_BODY" | jq -r '.envdAccessToken // empty') +assert_not_empty "$traffic_access_token" "private sandbox returns traffic token" +assert_not_empty "$envd_access_token" "secure sandbox returns envd token" +if [[ -z "$traffic_access_token" || -z "$envd_access_token" ]]; then + suite_summary "08_auth" || true + exit 1 +fi +wait_for_sandbox_state "$secure_sandbox_id" "running" 30 + +_curl_do -s \ + -H "e2b-traffic-access-token: ${traffic_access_token}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "traffic token does not authenticate control plane" + +_curl_do -s \ + -H "X-Access-Token: ${envd_access_token}" \ + "${AENV_URL}/sandboxes" +assert_status "$HTTP_STATUS" "401" "envd token does not authenticate control plane" + +# -- Secure envd accepts only its envd access token -- +_curl_do -s --max-time 5 \ + -H "x-agentenv-sandbox-id: ${secure_sandbox_id}" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ + "${AENV_PROXY_URL}/health" +assert_status "$HTTP_STATUS" "401" "secure envd rejects missing token" + +_curl_do -s --max-time 5 \ + -H "X-API-Key: ${AENV_API_KEY}" \ + -H "x-agentenv-sandbox-id: ${secure_sandbox_id}" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ + "${AENV_PROXY_URL}/health" +assert_status "$HTTP_STATUS" "401" "secure envd rejects API key" + +_curl_do -s --max-time 5 \ + -H "e2b-traffic-access-token: ${traffic_access_token}" \ + -H "x-agentenv-sandbox-id: ${secure_sandbox_id}" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ + "${AENV_PROXY_URL}/health" +assert_status "$HTTP_STATUS" "401" "secure envd rejects traffic token" + +_curl_do -s --max-time 5 \ + -H "X-Access-Token: ${envd_access_token}" \ + -H "x-agentenv-sandbox-id: ${secure_sandbox_id}" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ + "${AENV_PROXY_URL}/health" +assert_status "$HTTP_STATUS" "204" "secure envd accepts envd token" + +# -- Envd tokens are scoped to one sandbox -- +other_secure_sandbox_id=$(create_sandbox "$AENV_TEMPLATE_ID" 60 \ + '{"secure":true,"network":{"allowPublicTraffic":false}}'); _sync_http +if [[ -n "$other_secure_sandbox_id" ]]; then + track_sandbox "$other_secure_sandbox_id" +fi +assert_status "$HTTP_STATUS" "201" "create second private secure sandbox" +assert_not_empty "$other_secure_sandbox_id" "second private secure sandbox ID present" +if [[ "$HTTP_STATUS" != "201" || -z "$other_secure_sandbox_id" ]]; then + suite_summary "08_auth" || true + exit 1 +fi +wait_for_sandbox_state "$other_secure_sandbox_id" "running" 30 + +_curl_do -s --max-time 5 \ + -H "X-Access-Token: ${envd_access_token}" \ + -H "x-agentenv-sandbox-id: ${other_secure_sandbox_id}" \ + -H "x-agentenv-target-port: ${AENV_ENVD_PORT}" \ + "${AENV_PROXY_URL}/health" +assert_status "$HTTP_STATUS" "401" "envd token cannot authenticate another sandbox" + +# -- Health and Prometheus metrics work without application auth -- api_get_no_auth "/health" assert_status "$HTTP_STATUS" "204" "/health works without auth" +api_get_no_auth "/metrics" +assert_not_eq "$HTTP_STATUS" "401" "/metrics does not require API key" + suite_summary "08_auth" diff --git a/scripts/tests/e2e/suites/11_node_metrics.sh b/scripts/tests/e2e/suites/11_node_metrics.sh index d547ed48..d2df68c7 100644 --- a/scripts/tests/e2e/suites/11_node_metrics.sh +++ b/scripts/tests/e2e/suites/11_node_metrics.sh @@ -11,7 +11,7 @@ log "Suite: Node Metrics" readonly NODE_METRICS_SANDBOX_TIMEOUT_SECONDS=60 readonly SCHEDULER_BINDING_CLEANUP_TIMEOUT_SECONDS=75 readonly PROXY_HEALTH_PATH="/health" -readonly PROXY_HEALTH_PORT=49983 +readonly PROXY_HEALTH_PORT="${AENV_ENVD_PORT}" wait_for_global_sandboxes_quiesced() { local timeout="${1:-20}" @@ -55,7 +55,7 @@ wait_for_node_runtime_allocations_quiesced() { local attempt for ((attempt = 0; attempt < timeout * 2; attempt++)); do - api_admin_get "/nodes" + api_get "/nodes" if [[ "${HTTP_STATUS}" == "200" ]] && echo "${HTTP_BODY}" | jq -e 'all(.[]; (.sandboxCount == 0 and .metrics.allocatedCPU == 0 and .metrics.allocatedMemoryBytes == 0))' >/dev/null 2>&1; then return 0 @@ -97,7 +97,7 @@ quiesce_runtime_state_before_baseline() { fetch_admin_nodes() { local base_url="${1:-${AENV_URL}}" - api_admin_get_at "${base_url}" "/nodes" + api_get_at "${base_url}" "/nodes" [[ "${HTTP_STATUS}" == "200" ]] || return 1 printf '%s\n' "${HTTP_BODY}" } @@ -138,7 +138,7 @@ wait_for_node_snapshot() { local create_successes for ((attempt = 0; attempt < timeout * 2; attempt++)); do - api_admin_get "/nodes" + api_get "/nodes" if [[ "${HTTP_STATUS}" == "200" ]]; then body="${HTTP_BODY}" sandbox_count="$(echo "${body}" | jq -r --arg id "${node_id}" '.[] | select(.id == $id) | .sandboxCount // empty' 2>/dev/null || true)" @@ -164,7 +164,7 @@ wait_for_node_snapshot() { node_detail_sandbox_count() { local base_url="$1" local node_id="$2" - api_admin_get_at "${base_url}" "/nodes/${node_id}" + api_get_at "${base_url}" "/nodes/${node_id}" [[ "${HTTP_STATUS}" == "200" ]] || return 1 echo "${HTTP_BODY}" | jq '.sandboxCount' } @@ -192,7 +192,7 @@ quiesce_runtime_state_before_baseline || wait_for_admin_nodes_count "${AENV_URL}" "${expected_nodes}" 60 || die "Timed out waiting for gateway/admin nodes endpoint" -api_admin_get "/nodes" +api_get "/nodes" assert_status "${HTTP_STATUS}" "200" "admin /nodes returns 200" baseline_nodes_json="${HTTP_BODY}" @@ -206,7 +206,7 @@ while IFS=$'\t' read -r node_id sandbox_count allocated_cpu allocated_memory cre BASELINE_ALLOCATED_MEMORY["${node_id}"]="${allocated_memory}" BASELINE_CREATE_SUCCESSES["${node_id}"]="${create_successes}" - api_admin_get "/nodes/${node_id}" + api_get "/nodes/${node_id}" assert_status "${HTTP_STATUS}" "200" "admin /nodes/${node_id} returns 200" BASELINE_DETAIL_SANDBOX_COUNT["${node_id}"]="$(echo "${HTTP_BODY}" | jq '.sandboxCount')" done < <(echo "${baseline_nodes_json}" | jq -r '.[] | [ @@ -222,7 +222,7 @@ if e2e_mode_is_clustered; then [[ -n "${node_url}" ]] || continue wait_for_admin_nodes_count "${node_url}" 1 45 || die "Timed out waiting for ${node_url}/nodes" - api_admin_get_at "${node_url}" "/nodes" + api_get_at "${node_url}" "/nodes" assert_status "${HTTP_STATUS}" "200" "node-local admin /nodes returns 200 for $(node_label_for_url "${node_url}")" local_node_id="$(echo "${HTTP_BODY}" | jq -r '.[0].id')" assert_not_empty "${local_node_id}" "node-local admin /nodes exposes node id for $(node_label_for_url "${node_url}")" @@ -316,14 +316,14 @@ for node_id in "${!BASELINE_SANDBOX_COUNT[@]}"; do 30; then _pass "gateway /nodes metrics converge for ${node_id}" else - api_admin_get "/nodes" + api_get "/nodes" _fail \ "gateway /nodes metrics converge for ${node_id}" \ "sandboxCount=${expected_sandbox_count}, allocatedCPU=${expected_allocated_cpu}, allocatedMemoryBytes=${expected_allocated_memory}, createSuccesses>=${expected_create_successes_min}" \ "${HTTP_BODY}" fi - api_admin_get "/nodes/${node_id}" + api_get "/nodes/${node_id}" assert_status "${HTTP_STATUS}" "200" "gateway /nodes/${node_id} returns 200 after workload" detail_sandboxes="$(echo "${HTTP_BODY}" | jq '.sandboxCount')" expected_detail_sandboxes=$((BASELINE_DETAIL_SANDBOX_COUNT["${node_id}"] + owned_count)) @@ -370,7 +370,7 @@ for node_id in "${!BASELINE_SANDBOX_COUNT[@]}"; do 30; then _pass "gateway /nodes runtime allocation resets for ${node_id} after cleanup" else - api_admin_get "/nodes" + api_get "/nodes" _fail \ "gateway /nodes runtime allocation resets for ${node_id} after cleanup" \ "sandboxCount=${BASELINE_SANDBOX_COUNT[${node_id}]}, allocatedCPU=${BASELINE_ALLOCATED_CPU[${node_id}]}, allocatedMemoryBytes=${BASELINE_ALLOCATED_MEMORY[${node_id}]}, createSuccesses>=${expected_create_successes_min_after_cleanup}" \ diff --git a/services/README.md b/services/README.md index a5fe1869..59fe1194 100644 --- a/services/README.md +++ b/services/README.md @@ -78,10 +78,13 @@ make run-gateway The default local config uses `127.0.0.1:9090` for the scheduler. -The gateway and runtime nodes require the same API key. The gateway reads -`AENV_API_KEY` or `/run/secrets/api-key`; it does not generate a key. -Application proxy requests may additionally use the sandbox response's -`trafficAccessToken` in the `e2b-traffic-access-token` header. +The gateway and runtime nodes require the same API key for control-plane APIs. +The gateway reads `AENV_API_KEY` or `/run/secrets/api-key`; it does not generate +a key. The gateway routes data-plane requests without authenticating them +because only the owning runtime has the sandbox policy needed to distinguish +public ingress, private ingress, and secure envd. Private application proxy +requests use the sandbox response's `trafficAccessToken` in the +`e2b-traffic-access-token` header; secure envd requests use `X-Access-Token`. ## Scheduler configuration diff --git a/services/gateway/cmd/main.go b/services/gateway/cmd/main.go index a0cfacbb..ea6448ea 100644 --- a/services/gateway/cmd/main.go +++ b/services/gateway/cmd/main.go @@ -48,7 +48,7 @@ func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (s if explicit, present := lookupEnv(apiKeyEnv); present { value = explicit } else { - file, err := os.Open(secretPath) + file, err := openSecretFile(secretPath) if err != nil { if os.IsNotExist(err) { return "", fmt.Errorf("%s must be set or %s must exist", apiKeyEnv, secretPath) @@ -66,6 +66,28 @@ func loadAPIKeyFrom(lookupEnv func(string) (string, bool), secretPath string) (s return validateAPIKey(value, source) } +func openSecretFile(path string) (*os.File, error) { + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = syscall.Close(fd) + return nil, fmt.Errorf("open returned an invalid file descriptor") + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, fmt.Errorf("must be a regular file") + } + return file, nil +} + func validateAPIKey(value, source string) (string, error) { if len(value) < 32 || len(value) > maxAPIKeyLen { return "", fmt.Errorf("API key from %s must contain between 32 and %d URL-safe characters", source, maxAPIKeyLen) diff --git a/services/gateway/cmd/main_test.go b/services/gateway/cmd/main_test.go index bc88faeb..f384460e 100644 --- a/services/gateway/cmd/main_test.go +++ b/services/gateway/cmd/main_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" ) @@ -75,3 +76,36 @@ func TestLoadAPIKeyRejectsMissingFile(t *testing.T) { t.Fatal("loadAPIKeyFrom() unexpectedly accepted a missing secret") } } + +func TestLoadAPIKeyRejectsNonRegularFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "api-key") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path); err == nil { + t.Fatal("loadAPIKeyFrom() unexpectedly accepted a FIFO") + } +} + +func TestLoadAPIKeyAllowsSymlinkedSecret(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + target := filepath.Join(dir, "..data-api-key") + path := filepath.Join(dir, "api-key") + if err := os.WriteFile(target, []byte(testAPIKey+"\n"), 0o444); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Base(target), path); err != nil { + t.Fatal(err) + } + got, err := loadAPIKeyFrom(func(string) (string, bool) { return "", false }, path) + if err != nil { + t.Fatalf("loadAPIKeyFrom() error = %v", err) + } + if got != testAPIKey { + t.Fatalf("loadAPIKeyFrom() = %q, want %q", got, testAPIKey) + } +} diff --git a/services/gateway/internal/server.go b/services/gateway/internal/server.go index 310efada..5cffa097 100644 --- a/services/gateway/internal/server.go +++ b/services/gateway/internal/server.go @@ -3,6 +3,7 @@ package gateway import ( "bytes" "context" + "crypto/subtle" "encoding/json" "errors" "io" @@ -68,8 +69,7 @@ type Server struct { } func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, options ServerOptions) (*Server, error) { - apiKey := strings.TrimSpace(options.APIKey) - if apiKey == "" { + if options.APIKey == "" { return nil, errors.New("API key is required") } sandboxProxyDomains, err := normalizeProxyDomains(options.SandboxProxyDomains) @@ -89,7 +89,7 @@ func NewServer(logger *zap.Logger, schedulerClient schedulerv1.SchedulerClient, httpClient: &http.Client{}, requestTimeout: options.RequestTimeout, maxRespSize: options.MaxResponseSize, - apiKey: []byte(apiKey), + apiKey: []byte(options.APIKey), debugMode: options.DebugMode, sandboxProxyDomains: sandboxProxyDomains, }, nil @@ -104,6 +104,15 @@ func (s *Server) Handler() http.Handler { // decoding %2F → / and issuing 301 redirects), which breaks proxy // forwarding of percent-encoded path segments such as /files/%2F. core := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if isExplicitProxyPath(r.URL.Path) && !hasCompleteProxyRouteHeaders(r.Header) { + setGatewayRouteSource(w, routeSourceHeader) + if _, hasSandbox := sandboxIDFromHeaders(r.Header); !hasSandbox { + http.Error(w, "sandbox id header required", http.StatusBadRequest) + return + } + http.Error(w, "target port header required", http.StatusBadRequest) + return + } if r.URL.Path == "/metrics" { http.NotFound(w, r) return @@ -539,6 +548,12 @@ func hasProxyRoutingHeaders(h http.Header) bool { return false } +func hasCompleteProxyRouteHeaders(h http.Header) bool { + _, hasSandbox := sandboxIDFromHeaders(h) + _, hasTargetPort := targetPortFromHeaders(h) + return hasSandbox && hasTargetPort +} + func targetPortFromHeaders(h http.Header) (string, bool) { for _, name := range []string{headerTargetPort, headerE2BTargetPort} { v := strings.TrimSpace(h.Get(name)) @@ -822,42 +837,45 @@ func extractSandboxIDsFromResponse(body []byte) []string { func singleHeaderMatches(headers http.Header, name string, expected []byte) bool { values := headers.Values(name) - return len(values) == 1 && bytes.Equal([]byte(values[0]), expected) + if len(values) != 1 || len(values[0]) != len(expected) { + return false + } + return subtle.ConstantTimeCompare([]byte(values[0]), expected) == 1 } func (s *Server) isSandboxDataPlaneRequest(r *http.Request) bool { - if strings.TrimRight(r.URL.Path, "/") == "/proxy" || strings.HasPrefix(r.URL.Path, "/proxy/") { + if isExplicitProxyPath(r.URL.Path) { + // The explicit proxy prefix cannot dispatch to a control-plane handler. + // Let the core handler return a stable 400 for incomplete routing data. return true } hostRoute, err := parseHostRoute(r.Host, s.sandboxProxyDomains) - if hostRoute != nil || err != nil { + if hostRoute != nil { return true } + if err != nil { + return false + } - return !isSandboxControlPlaneRequest(r) && hasProxyRoutingHeaders(r.Header) + return !isSandboxControlPlaneRequest(r) && hasCompleteProxyRouteHeaders(r.Header) } -func hasSingleNonEmptyHeader(headers http.Header, name string) bool { - values := headers.Values(name) - return len(values) == 1 && strings.TrimSpace(values[0]) != "" +func isExplicitProxyPath(path string) bool { + return path == "/proxy" || strings.HasPrefix(path, "/proxy/") } func (s *Server) authenticate(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { dataPlane := s.isSandboxDataPlaneRequest(r) - if r.URL.Path == "/health" && !dataPlane { + if dataPlane || r.URL.Path == "/health" || r.URL.Path == "/metrics" { + // Sandbox-scoped ingress and envd authorization depend on runtime + // metadata and are enforced by the owning runtime node. next.ServeHTTP(w, r) return } - authorized := singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) - if !authorized && dataPlane { - // Runtime nodes perform the definitive sandbox-scoped token validation. - authorized = hasSingleNonEmptyHeader(r.Header, headerTrafficToken) || - hasSingleNonEmptyHeader(r.Header, headerEnvdAccessToken) - } - if !authorized { + if !singleHeaderMatches(r.Header, headerAPIKey, s.apiKey) { w.WriteHeader(http.StatusUnauthorized) return } diff --git a/services/gateway/internal/server_test.go b/services/gateway/internal/server_test.go index e192e873..4f71bc98 100644 --- a/services/gateway/internal/server_test.go +++ b/services/gateway/internal/server_test.go @@ -221,7 +221,7 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { addHeaders: func(headers http.Header) { headers.Set(headerAPIKey, testAPIKey) }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadGateway, }, { name: "duplicate", @@ -235,7 +235,7 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req := httptest.NewRequest(http.MethodGet, "/nodes", nil) tt.addHeaders(req.Header) recorder := httptest.NewRecorder() @@ -248,7 +248,18 @@ func TestGatewayRequiresExactAPIKey(t *testing.T) { } } -func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { +func TestGatewayMetricsPathDoesNotRequireAPIKey(t *testing.T) { + server := newTestServer(t, stubSchedulerClient{}, time.Second, 1024) + recorder := httptest.NewRecorder() + + server.Handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if recorder.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusNotFound) + } +} + +func TestGatewayLeavesDataPlaneAuthorizationToRuntime(t *testing.T) { const sandboxID = "0191f4d0-7b2a-7c11-9c2d-0123456789ab" lookupCalls := 0 server := newTestServer(t, stubSchedulerClient{ @@ -261,11 +272,20 @@ func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) - req.Header.Set(headerE2BTargetPort, "49983") - req.Header.Set(headerTrafficToken, "runtime-validates-this-token") + req.Header.Set(headerE2BTargetPort, "8080") recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, req) if recorder.Code == http.StatusUnauthorized || lookupCalls != 1 { + t.Fatalf("credential-free data plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) + } + + req = httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header.Set(headerE2BSandboxID, sandboxID) + req.Header.Set(headerE2BTargetPort, "49983") + req.Header.Set(headerTrafficToken, "runtime-validates-this-token") + recorder = httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code == http.StatusUnauthorized || lookupCalls != 2 { t.Fatalf("valid scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } @@ -275,17 +295,17 @@ func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { req.Header.Set(headerTrafficToken, "wrong-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code == http.StatusUnauthorized || lookupCalls != 2 { + if recorder.Code == http.StatusUnauthorized || lookupCalls != 3 { t.Fatalf("runtime-scoped token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } req = httptest.NewRequest(http.MethodGet, "/proxy", nil) req.Header.Set(headerE2BSandboxID, sandboxID) req.Header.Set(headerE2BTargetPort, "8080") - req.Header.Set("X-Access-Token", "runtime-validates-this-token") + req.Header.Set(headerEnvdAccessToken, "runtime-validates-this-token") recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code == http.StatusUnauthorized || lookupCalls != 3 { + if recorder.Code == http.StatusUnauthorized || lookupCalls != 4 { t.Fatalf("runtime-scoped envd token: status=%d lookup calls=%d", recorder.Code, lookupCalls) } @@ -295,11 +315,39 @@ func TestGatewayForwardsSandboxTokensOnlyOnDataPlane(t *testing.T) { recorder = httptest.NewRecorder() handler.ServeHTTP(recorder, req) - if recorder.Code != http.StatusUnauthorized || lookupCalls != 3 { + if recorder.Code != http.StatusUnauthorized || lookupCalls != 4 { t.Fatalf("scoped token reached control plane: status=%d lookup calls=%d", recorder.Code, lookupCalls) } } +func TestGatewayRejectsIncompleteProxyRouteBeforeScheduling(t *testing.T) { + scheduleCalls := 0 + server := newTestServer(t, stubSchedulerClient{ + scheduleFunc: func(context.Context, *schedulerv1.ScheduleRequest, ...grpc.CallOption) (*schedulerv1.ScheduleResponse, error) { + scheduleCalls++ + return nil, fmt.Errorf("schedule reached") + }, + }, time.Second, 1024) + handler := server.Handler() + + for _, headers := range []http.Header{ + {}, + {headerE2BSandboxID: []string{"sandbox-only"}}, + {headerE2BTargetPort: []string{"8080"}}, + } { + req := httptest.NewRequest(http.MethodGet, "/proxy", nil) + req.Header = headers + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, req) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest) + } + } + if scheduleCalls != 0 { + t.Fatalf("schedule calls = %d, want 0", scheduleCalls) + } +} + func withSandboxProxyDomains(domains ...string) testServerOption { return func(options *ServerOptions) { options.SandboxProxyDomains = domains diff --git a/src/api/generated/src/models.rs b/src/api/generated/src/models.rs index 434da344..089b1093 100644 --- a/src/api/generated/src/models.rs +++ b/src/api/generated/src/models.rs @@ -5722,7 +5722,7 @@ impl std::convert::TryFrom for header::IntoHeaderValue, @@ -5910,10 +5910,15 @@ impl std::convert::TryFrom for header::IntoHeaderValue, + /// List of allowed destinations for egress traffic. Each entry can be a CIDR block (e.g. \"8.8.8.8/32\"), a bare IP address (e.g. \"8.8.8.8\"), or a domain name (e.g. \"example.com\", \"*.example.com\"). Allowed entries always take precedence over denied entries. #[serde(rename = "allowOut")] #[validate(custom(function = "check_xss_vec_string"))] @@ -5936,6 +5941,7 @@ impl SandboxNetworkUpdateConfig { #[allow(clippy::new_without_default, clippy::too_many_arguments)] pub fn new() -> SandboxNetworkUpdateConfig { SandboxNetworkUpdateConfig { + allow_public_traffic: Some(true), allow_out: None, deny_out: None, allow_internet_access: None, @@ -5949,6 +5955,15 @@ impl SandboxNetworkUpdateConfig { impl std::fmt::Display for SandboxNetworkUpdateConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let params: Vec> = vec![ + self.allow_public_traffic + .as_ref() + .map(|allow_public_traffic| { + [ + "allowPublicTraffic".to_string(), + allow_public_traffic.to_string(), + ] + .join(",") + }), self.allow_out.as_ref().map(|allow_out| { [ "allowOut".to_string(), @@ -6001,6 +6016,7 @@ impl std::str::FromStr for SandboxNetworkUpdateConfig { #[derive(Default)] #[allow(dead_code)] struct IntermediateRep { + pub allow_public_traffic: Vec, pub allow_out: Vec>, pub deny_out: Vec>, pub allow_internet_access: Vec, @@ -6025,6 +6041,8 @@ impl std::str::FromStr for SandboxNetworkUpdateConfig { if let Some(key) = key_result { #[allow(clippy::match_single_binding)] match key { + #[allow(clippy::redundant_clone)] + "allowPublicTraffic" => intermediate_rep.allow_public_traffic.push(::from_str(val).map_err(|x| x.to_string())?), "allowOut" => return std::result::Result::Err("Parsing a container in this style is not supported in SandboxNetworkUpdateConfig".to_string()), "denyOut" => return std::result::Result::Err("Parsing a container in this style is not supported in SandboxNetworkUpdateConfig".to_string()), #[allow(clippy::redundant_clone)] @@ -6039,6 +6057,7 @@ impl std::str::FromStr for SandboxNetworkUpdateConfig { // Use the intermediate representation to return the struct std::result::Result::Ok(SandboxNetworkUpdateConfig { + allow_public_traffic: intermediate_rep.allow_public_traffic.into_iter().next(), allow_out: intermediate_rep.allow_out.into_iter().next(), deny_out: intermediate_rep.deny_out.into_iter().next(), allow_internet_access: intermediate_rep.allow_internet_access.into_iter().next(), diff --git a/src/api/impls/auth.rs b/src/api/impls/auth.rs index 435ea118..715b8545 100644 --- a/src/api/impls/auth.rs +++ b/src/api/impls/auth.rs @@ -27,7 +27,9 @@ impl ApiImpl { single_header(headers, API_KEY_HEADER).is_some_and(|value| { let candidate = value.as_bytes(); let expected = self.api_key.as_bytes(); - candidate.len() == expected.len() && bool::from(candidate.ct_eq(expected)) + !expected.is_empty() + && candidate.len() == expected.len() + && bool::from(candidate.ct_eq(expected)) }) } @@ -55,32 +57,56 @@ where { let proxy_request = proxy::is_sandbox_proxy_request(&request, api_impl.as_ref().sandbox_proxy_domains()); - if request.uri().path() == "/health" && !proxy_request { + if matches!(request.uri().path(), "/health" | "/metrics") && !proxy_request { return next.run(request).await; } let api_impl = api_impl.as_ref(); - let mut authorized = api_impl.has_valid_api_key(request.headers()); - let mut envd_authorized = false; - if proxy_request { - if let Some((sandbox_id, target_port)) = - proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) + if !proxy_request { + return if api_impl.has_valid_api_key(request.headers()) { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + }; + } + + let Some((sandbox_id, target_port)) = + proxy::route_for_auth(&request, api_impl.sandbox_proxy_domains()) + else { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + return if proxy::has_proxy_prefix(request.uri().path()) + || api_impl.has_valid_api_key(request.headers()) { - authorized |= api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id); - let envd_candidate = single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) - .and_then(|value| value.to_str().ok()); - if let Some(candidate) = envd_candidate { - envd_authorized = proxy::has_valid_envd_access_token( - api_impl, - sandbox_id, - target_port, - candidate, - ) - .await; - authorized |= envd_authorized; - } + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + }; + }; + let metadata = match api_impl.orchestrator().get_sandbox(&sandbox_id).await { + Ok(Some(metadata)) => metadata, + Ok(None) => { + request.headers_mut().remove(ENVD_ACCESS_TOKEN_HEADER); + return proxy::sandbox_not_found_response(sandbox_id); } - } + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + + let envd_request = target_port == proxy::effective_envd_port(&metadata); + let envd_authorized = envd_request + && metadata.secure + && single_header(request.headers(), ENVD_ACCESS_TOKEN_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|candidate| { + api_impl + .orchestrator() + .validate_envd_access_token(sandbox_id, candidate) + }); + let authorized = if envd_request { + !metadata.secure || envd_authorized + } else { + metadata.network_policy.allow_public_traffic + || api_impl.has_valid_traffic_access_token(request.headers(), sandbox_id) + }; if !authorized { return StatusCode::UNAUTHORIZED.into_response(); diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index 48648dc8..2e7390cd 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -145,7 +145,7 @@ impl From<&SandboxNetworkPolicy> for models::SandboxNetworkConfig { fn from(policy: &SandboxNetworkPolicy) -> Self { let egress = &policy.egress; Self { - allow_public_traffic: Some(true), + allow_public_traffic: Some(policy.allow_public_traffic), allow_out: (!egress.allowed_cidrs.is_empty() || !egress.allowed_domains.is_empty()) .then(|| { egress @@ -179,10 +179,9 @@ fn allow_internet_access_from_base_policy(policy: BaseSandboxNetworkPolicy) -> N impl From for models::SandboxDetail { fn from(m: SandboxMetadata) -> Self { - let network = m - .network_policy - .has_explicit_egress_rules() - .then(|| models::SandboxNetworkConfig::from(&m.network_policy)); + let network = (!m.network_policy.allow_public_traffic + || m.network_policy.has_explicit_egress_rules()) + .then(|| models::SandboxNetworkConfig::from(&m.network_policy)); let allow_internet_access = Some(allow_internet_access_from_base_policy( m.network_policy.base_policy, )); @@ -214,14 +213,18 @@ impl From for models::SandboxDetail { impl ApiImpl { fn sandbox_model(&self, metadata: SandboxMetadata) -> models::Sandbox { - let traffic_access_token = self.traffic_access_token(metadata.id); + let traffic_access_token = (!metadata.network_policy.allow_public_traffic) + .then(|| self.traffic_access_token(metadata.id)); let envd_access_token = self .orchestrator .get_envd_access_token(&metadata) .map(|token| token.expose().to_owned()); let mut sandbox = models::Sandbox::from(metadata); sandbox.envd_access_token = envd_access_token; - sandbox.traffic_access_token = Some(Nullable::Present(traffic_access_token)); + sandbox.traffic_access_token = Some(match traffic_access_token { + Some(token) => Nullable::Present(token), + None => Nullable::Null, + }); sandbox.domain = self .sandbox_proxy_domains() .first() @@ -366,7 +369,11 @@ fn network_policy_from_create( let allow_out = network.and_then(|network| network.allow_out.clone()); let deny_out = network.and_then(|network| network.deny_out.clone()); let egress = SandboxNetworkEgressPolicy::new(allow_out, deny_out)?; - let policy = SandboxNetworkPolicy::new(base_policy, egress); + let allow_public_traffic = network + .and_then(|network| network.allow_public_traffic) + .unwrap_or(true); + let policy = SandboxNetworkPolicy::new(base_policy, egress) + .with_allow_public_traffic(allow_public_traffic); if policy.has_domain_allow_rules() { anyhow::bail!( "domain entries in allowOut are not supported until TCP egress proxy is enabled" @@ -387,7 +394,8 @@ fn network_policy_from_update( Ok(SandboxNetworkPolicy::new( base_policy_from_allow_internet_access(body.allow_internet_access), policy, - )) + ) + .with_allow_public_traffic(body.allow_public_traffic.unwrap_or(true))) } #[async_trait] @@ -1493,6 +1501,7 @@ mod tests { #[test] fn network_update_replaces_base_policy_and_egress() { let body = models::SandboxNetworkUpdateConfig { + allow_public_traffic: Some(false), allow_out: Some(vec!["8.8.8.8".to_string()]), deny_out: Some(vec!["203.0.113.0/24".to_string()]), allow_internet_access: Some(false), @@ -1501,10 +1510,25 @@ mod tests { let policy = network_policy_from_update(&body).unwrap(); assert_eq!(policy.base_policy, BaseSandboxNetworkPolicy::Deny); + assert!(!policy.allow_public_traffic); assert_eq!(policy.egress.allowed_cidrs, ["8.8.8.8/32"]); assert_eq!(policy.egress.denied_cidrs, ["203.0.113.0/24"]); } + #[test] + fn network_create_preserves_private_ingress() { + let mut network = models::SandboxNetworkConfig::new(); + network.allow_public_traffic = Some(false); + + let policy = network_policy_from_create(None, Some(&network)).unwrap(); + + assert!(!policy.allow_public_traffic); + assert_eq!( + models::SandboxNetworkConfig::from(&policy).allow_public_traffic, + Some(false) + ); + } + #[test] fn empty_network_update_clears_base_policy_and_egress() { let policy = network_policy_from_update(&models::SandboxNetworkUpdateConfig::new()) diff --git a/src/api/openapi.yml b/src/api/openapi.yml index 32a4c74b..8bbd9661 100644 --- a/src/api/openapi.yml +++ b/src/api/openapi.yml @@ -302,7 +302,7 @@ components: allowPublicTraffic: type: boolean default: true - description: Specify if the sandbox URLs should be accessible only with authentication. + description: Specify if the sandbox URLs should be accessible without a traffic access token. allowOut: type: array description: List of allowed destinations for egress traffic. Each entry can be a CIDR block (e.g. "8.8.8.8/32"), a bare IP address (e.g. "8.8.8.8"), or a domain name (e.g. "example.com", "*.example.com"). Allowed entries always take precedence over denied entries. @@ -319,8 +319,12 @@ components: SandboxNetworkUpdateConfig: type: object - description: Network configuration update for a running sandbox. Replaces the current egress rules with the provided configuration. Omitting a field clears it. + description: Network configuration update for a running sandbox. Replaces the current ingress and egress rules with the provided configuration. Omitting a field restores its default. properties: + allowPublicTraffic: + type: boolean + default: true + description: Specify if the sandbox URLs should be accessible without a traffic access token. allowOut: type: array description: List of allowed destinations for egress traffic. Each entry can be a CIDR block (e.g. "8.8.8.8/32"), a bare IP address (e.g. "8.8.8.8"), or a domain name (e.g. "example.com", "*.example.com"). Allowed entries always take precedence over denied entries. @@ -1686,7 +1690,7 @@ paths: /sandboxes/{sandboxID}/network: put: summary: Update sandbox network - description: Update the network configuration for a running sandbox. Replaces the current egress rules with the provided configuration. Omitting field clears it. + description: Update the network configuration for a running sandbox. Replaces the current ingress and egress rules with the provided configuration. Omitting a field restores its default. security: - ApiKeyAuth: [] - AuthProviderBearerAuth: [] diff --git a/src/api/proxy.rs b/src/api/proxy.rs index d275ed4d..40b1d265 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -36,7 +36,7 @@ use tracing::{debug, info, trace, warn}; use crate::{ api::{ - impls::auth::{API_KEY_HEADER, ENVD_ACCESS_TOKEN_HEADER, TRAFFIC_ACCESS_TOKEN_HEADER}, + impls::auth::{API_KEY_HEADER, TRAFFIC_ACCESS_TOKEN_HEADER}, ApiImpl, }, cfg::ConfigManager, @@ -48,6 +48,9 @@ use crate::{ types::SandboxId, }; +#[cfg(test)] +use crate::api::impls::auth::ENVD_ACCESS_TOKEN_HEADER; + /// Shared outbound HTTP client for the client-facing reverse proxy. pub(crate) type ProxyClient = Client; type UpstreamWebSocket = WebSocketStream>; @@ -148,8 +151,10 @@ where pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(SandboxId, u16)> { if !has_proxy_prefix(request.uri().path()) { - if let Ok(Some(route)) = parse_host_proxy_route(request_host(request), domains) { - return Some((route.sandbox_id, route.target_port)); + match parse_host_proxy_route(request_host(request), domains) { + Ok(Some(route)) => return Some((route.sandbox_id, route.target_port)), + Ok(None) => {} + Err(_) => return None, } } @@ -159,24 +164,6 @@ pub(crate) fn route_for_auth(request: &Request, domains: &[String]) -> Option<(S )) } -pub(crate) async fn has_valid_envd_access_token( - api_impl: &ApiImpl, - sandbox_id: SandboxId, - target_port: u16, - candidate: &str, -) -> bool { - let Ok(Some(metadata)) = api_impl.orchestrator().get_sandbox(&sandbox_id).await else { - return false; - }; - if !metadata.secure || target_port != effective_envd_port(&metadata) { - return false; - } - - api_impl - .orchestrator() - .validate_envd_access_token(sandbox_id, candidate) -} - pub(crate) fn is_sandbox_proxy_request(request: &Request, domains: &[String]) -> bool { let path = request.uri().path(); if has_proxy_prefix(path) { @@ -336,7 +323,7 @@ fn strip_proxy_prefix(path: &str) -> &str { path.strip_prefix(PROXY_ROUTE).unwrap_or("") } -fn has_proxy_prefix(path: &str) -> bool { +pub(crate) fn has_proxy_prefix(path: &str) -> bool { path == PROXY_ROUTE || path.starts_with("/proxy/") } @@ -420,7 +407,7 @@ fn has_routing_header(headers: &HeaderMap) -> bool { headers.get(SANDBOX_ID_HEADER).is_some() || headers.get(E2B_SANDBOX_ID_HEADER).is_some() } -fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { +pub(crate) fn effective_envd_port(metadata: &SandboxMetadata) -> u16 { metadata .paused_state .as_ref() @@ -750,13 +737,6 @@ async fn resolve_proxy_request( sandbox_id, ))); } - authorize_secure_envd_auto_resume( - api_impl, - sandbox_id, - target_port, - &parts.headers, - ) - .await?; try_auto_resume(api_impl, sandbox_id).await?; auto_resume_attempted = true; continue; @@ -819,45 +799,6 @@ async fn resolve_proxy_request( }) } -async fn authorize_secure_envd_auto_resume( - api_impl: &ApiImpl, - sandbox_id: SandboxId, - target_port: u16, - headers: &HeaderMap, -) -> Result<(), Response> { - let metadata = api_impl - .orchestrator() - .get_sandbox(&sandbox_id) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)))?; - if target_port != effective_envd_port(&metadata) { - return Ok(()); - } - if !metadata.secure { - return Ok(()); - } - let candidate = headers - .get(ENVD_ACCESS_TOKEN_HEADER) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default(); - if api_impl - .orchestrator() - .validate_envd_access_token(sandbox_id, candidate) - { - return Ok(()); - } - - Err(Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ) - .body(Body::from("invalid or missing envd access token")) - .expect("static unauthorized proxy response is valid")) -} - async fn try_auto_resume(api_impl: &ApiImpl, sandbox_id: SandboxId) -> Result<(), Response> { match timeout( PROXY_AUTO_RESUME_TIMEOUT, @@ -909,6 +850,10 @@ fn parse_target_port_header(headers: &HeaderMap) -> Result Response { + proxy_error_response(&ProxyRequestError::SandboxNotFound(sandbox_id)) +} + fn proxy_error_response(error: &ProxyRequestError) -> Response { let (status, message) = match error { ProxyRequestError::MissingSandboxId => { @@ -1681,6 +1626,10 @@ mod tests { } async fn build_api_with_sandbox_proxy_domains(domains: Vec) -> Arc { + build_api_with_auth(domains, "test-key").await + } + + async fn build_api_with_auth(domains: Vec, api_key: &str) -> Arc { let root = tempfile::tempdir().unwrap(); let orchestrator = Orchestrator::new( crate::orchestrator::InMemoryMetadataStore::new(), @@ -1699,7 +1648,7 @@ mod tests { image_resolver, None, domains, - "test-key".to_string(), + api_key.to_string(), )) } @@ -1740,6 +1689,10 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; + api.orchestrator() + .set_allow_public_traffic_for_test(sandbox_id, false) + .await + .unwrap(); (server::new(api), access_token) } @@ -1756,6 +1709,10 @@ mod tests { crate::orchestrator::SandboxState::Running, ) .await; + api.orchestrator() + .set_allow_public_traffic_for_test(sandbox_id, false) + .await + .unwrap(); (server::new(api), access_token) } @@ -1819,6 +1776,11 @@ mod tests { ); headers.insert(TARGET_PORT_HEADER, HeaderValue::from_static("8080")); headers.insert(E2B_TARGET_PORT_HEADER, HeaderValue::from_static("8080")); + headers.insert(API_KEY_HEADER, HeaderValue::from_static("application-key")); + headers.insert( + TRAFFIC_ACCESS_TOKEN_HEADER, + HeaderValue::from_static("traffic-token"), + ); headers.insert(HOST, HeaderValue::from_static("client.example")); headers.insert(header::CONNECTION, HeaderValue::from_static("keep-alive")); headers.insert( @@ -1832,6 +1794,8 @@ mod tests { assert!(headers.get(E2B_SANDBOX_ID_HEADER).is_none()); assert!(headers.get(TARGET_PORT_HEADER).is_none()); assert!(headers.get(E2B_TARGET_PORT_HEADER).is_none()); + assert!(headers.get(API_KEY_HEADER).is_none()); + assert!(headers.get(TRAFFIC_ACCESS_TOKEN_HEADER).is_none()); assert!(headers.get(HOST).is_none()); assert!(headers.get(header::CONNECTION).is_none()); assert_eq!(headers.get("x-extra").unwrap(), "keep"); @@ -1881,7 +1845,7 @@ mod tests { } #[tokio::test] - async fn server_requires_exact_api_key_and_leaves_health_public() { + async fn server_requires_exact_api_key_and_leaves_health_and_metrics_public() { let app = server::new(build_api().await); for request in [ @@ -1936,6 +1900,19 @@ mod tests { .unwrap(); assert_eq!(response.status(), StatusCode::NO_CONTENT); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/metrics") + .header(header::HOST, "localhost") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_ne!(response.status(), StatusCode::UNAUTHORIZED); + let response = app .oneshot( Request::builder() @@ -1951,47 +1928,321 @@ mod tests { } #[tokio::test] - async fn traffic_token_cannot_authenticate_control_plane() { + async fn empty_configured_api_key_never_authenticates() { + let app = server::new(build_api_with_auth(Vec::new(), "").await); + let response = app + .oneshot( + Request::builder() + .uri("/sandboxes") + .header(API_KEY_HEADER, "") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn unknown_sandbox_routes_cannot_fall_through_to_control_handlers() { + let sandbox_id = SandboxId::new(); + let domains = vec!["sandbox.example.invalid".to_string()]; + let app = server::new(build_api_with_sandbox_proxy_domains(domains).await); + + let control_response = app + .clone() + .oneshot( + Request::builder() + .uri("/sandboxes") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "8080") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(control_response.status(), StatusCode::UNAUTHORIZED); + + let prefix_response = app + .clone() + .oneshot( + Request::builder() + .uri("/proxy/health") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "8080") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(prefix_response.status(), StatusCode::NOT_FOUND); + + let host_response = app + .oneshot( + Request::builder() + .uri("/sandboxes") + .header( + header::HOST, + format!("8080-{sandbox_id}.sandbox.example.invalid"), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(host_response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn malformed_proxy_host_cannot_bypass_control_plane_auth() { + let app = server::new( + build_api_with_sandbox_proxy_domains(vec!["sandbox.example.invalid".to_string()]).await, + ); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/sandboxes") + .header(header::HOST, "malformed.sandbox.example.invalid") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + + let response = app + .oneshot( + Request::builder() + .uri("/sandboxes") + .header(header::HOST, "malformed.sandbox.example.invalid") + .header(API_KEY_HEADER, "test-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn sandbox_tokens_cannot_authenticate_control_plane() { let api = build_api().await; let sandbox_id = SandboxId::new(); + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + api.orchestrator() + .set_secure_for_test(&sandbox_id, true) + .await + .unwrap(); let traffic_token = api.traffic_access_token(sandbox_id); + let metadata = api + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .unwrap() + .unwrap(); + let envd_token = api.orchestrator().get_envd_access_token(&metadata).unwrap(); let app = server::new(api); + + for (header_name, token) in [ + (TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str()), + (ENVD_ACCESS_TOKEN_HEADER, envd_token.expose()), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/sandboxes/{sandbox_id}/pause")) + .header(header_name, token) + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, "80") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + } + + #[tokio::test] + async fn application_proxy_auth_respects_public_and_private_ingress() { + let upstream_addr = start_upstream_server().await; + let api = build_api().await; + let sandbox_id = SandboxId::new(); + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + + let app = server::new(Arc::clone(&api)); + let public_response = app + .clone() + .oneshot( + Request::builder() + .uri("/proxy/public") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(public_response.status(), StatusCode::OK); + + api.orchestrator() + .set_allow_public_traffic_for_test(&sandbox_id, false) + .await + .unwrap(); + api.orchestrator() + .set_secure_for_test(&sandbox_id, true) + .await + .unwrap(); + let traffic_token = api.traffic_access_token(sandbox_id); + let metadata = api + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .unwrap() + .unwrap(); + let envd_token = api.orchestrator().get_envd_access_token(&metadata).unwrap(); + + for credential in [ + None, + Some((API_KEY_HEADER, "test-key")), + Some((TRAFFIC_ACCESS_TOKEN_HEADER, "incorrect")), + Some((ENVD_ACCESS_TOKEN_HEADER, envd_token.expose())), + ] { + let mut request = Request::builder() + .uri("/proxy/private") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()); + if let Some((header_name, value)) = credential { + request = request.header(header_name, value); + } + let response = app + .clone() + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + let response = app .oneshot( Request::builder() - .method(Method::POST) - .uri(format!("/sandboxes/{sandbox_id}/pause")) + .uri("/proxy/private") + .header(API_KEY_HEADER, "application-api-key") .header(TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token) .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) - .header(TARGET_PORT_HEADER, "80") + .header(TARGET_PORT_HEADER, upstream_addr.port().to_string()) .body(Body::empty()) .unwrap(), ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let payload: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(payload["api_key_header_seen"], false); + assert_eq!(payload["traffic_token_header_seen"], false); + assert!(payload["access_token"].is_null()); } #[tokio::test] - async fn envd_token_cannot_authenticate_application_proxy() { + async fn envd_proxy_auth_depends_only_on_secure_mode_and_envd_token() { let api = build_api().await; let sandbox_id = SandboxId::new(); - let traffic_token = api.traffic_access_token(sandbox_id); - let response = server::new(api) + api.orchestrator() + .set_proxy_target_for_test( + sandbox_id, + ProxyTarget::new(Ipv4Addr::LOCALHOST), + crate::orchestrator::SandboxState::Running, + ) + .await; + let target_port = ConfigManager::global_config() + .tools + .control_plane_port + .to_string(); + let app = server::new(Arc::clone(&api)); + + let insecure_response = app + .clone() .oneshot( Request::builder() - .uri("/proxy/hello") - .header(ENVD_ACCESS_TOKEN_HEADER, traffic_token) + .uri("/proxy/health") .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) - .header(TARGET_PORT_HEADER, "80") + .header(TARGET_PORT_HEADER, &target_port) .body(Body::empty()) .unwrap(), ) .await .unwrap(); + assert_ne!(insecure_response.status(), StatusCode::UNAUTHORIZED); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + api.orchestrator() + .set_secure_for_test(&sandbox_id, true) + .await + .unwrap(); + let metadata = api + .orchestrator() + .get_sandbox(&sandbox_id) + .await + .unwrap() + .unwrap(); + let envd_token = api.orchestrator().get_envd_access_token(&metadata).unwrap(); + let traffic_token = api.traffic_access_token(sandbox_id); + + for credential in [ + None, + Some((API_KEY_HEADER, "test-key")), + Some((TRAFFIC_ACCESS_TOKEN_HEADER, traffic_token.as_str())), + Some((ENVD_ACCESS_TOKEN_HEADER, "incorrect")), + ] { + let mut request = Request::builder() + .uri("/proxy/health") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, &target_port); + if let Some((header_name, value)) = credential { + request = request.header(header_name, value); + } + let response = app + .clone() + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + let response = app + .oneshot( + Request::builder() + .uri("/proxy/health") + .header(SANDBOX_ID_HEADER, sandbox_id.to_string()) + .header(TARGET_PORT_HEADER, target_port) + .header(ENVD_ACCESS_TOKEN_HEADER, envd_token.expose()) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_ne!(response.status(), StatusCode::UNAUTHORIZED); } #[tokio::test] @@ -2003,7 +2254,6 @@ mod tests { .oneshot( Request::builder() .uri("/proxy/hello") - .header("x-api-key", "test-key") .body(Body::empty()) .unwrap(), ) @@ -2242,7 +2492,7 @@ mod tests { } #[tokio::test] - async fn proxy_forwards_request_and_strips_internal_headers() { + async fn proxy_forwards_application_headers_and_strips_internal_headers() { let upstream_addr = start_upstream_server().await; let sandbox_id = SandboxId::new(); let app = proxy_app_for_sandbox(&sandbox_id).await; @@ -2534,7 +2784,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/health?foo=bar") - .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .header( "host", format!( @@ -2567,7 +2817,7 @@ mod tests { upstream_addr.port(), sandbox_id )) - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .body(Body::empty()) .unwrap(), ) @@ -2579,7 +2829,7 @@ mod tests { let payload: Value = serde_json::from_slice(&body).unwrap(); assert_eq!(payload["path"], "/authority"); - let (app, _) = proxy_app_for_sandbox_with_domains( + let (app, access_token) = proxy_app_for_sandbox_with_domains( &sandbox_id, vec!["sandbox.example.invalid".to_string()], ) @@ -2590,7 +2840,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, &access_token) .header( "host", format!( @@ -2612,7 +2862,7 @@ mod tests { Request::builder() .method(Method::GET) .uri("/proxy/health") - .header("x-api-key", "test-key") + .header(TRAFFIC_ACCESS_TOKEN_HEADER, access_token) .header( "host", format!( diff --git a/src/api_key.rs b/src/api_key.rs index 96ddc704..310d00bf 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -1,5 +1,5 @@ use std::ffi::OsStr; -use std::fs::{self, File}; +use std::fs::{File, OpenOptions}; use std::io::{self, Read}; use std::path::Path; @@ -49,14 +49,6 @@ fn resolve_from( } let managed_path = home_path.join(MANAGED_API_KEY_RELATIVE_PATH); - match fs::symlink_metadata(&managed_path) { - Err(error) if error.kind() == io::ErrorKind::NotFound => return create(&managed_path), - Err(error) => { - return Err(error) - .with_context(|| format!("inspect managed API key {}", managed_path.display())); - } - Ok(_) => {} - } if let Some(value) = managed_secret::read(&managed_path, API_KEY_FILE_MAX_LEN).context("load managed API key")? { @@ -67,10 +59,31 @@ fn resolve_from( } fn read_external(path: &Path) -> Result { - let value = read_bounded(File::open(path)?)?; + let file = open_external(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "API key secret must be a regular file", + )); + } + let value = read_bounded(file)?; validate_file_contents(&value).map_err(io::Error::other) } +fn open_external(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.custom_flags(libc::O_NONBLOCK); + } + + options.open(path) +} + fn read_bounded(file: File) -> Result { let mut value = String::with_capacity(API_KEY_FILE_MAX_LEN); file.take((API_KEY_FILE_MAX_LEN + 1) as u64) @@ -123,6 +136,7 @@ fn create(path: &Path) -> Result { #[cfg(test)] mod tests { use super::*; + use std::fs; use std::sync::{Arc, Barrier}; use tempfile::TempDir; @@ -144,6 +158,34 @@ mod tests { Ok(()) } + #[test] + fn external_secret_must_be_a_regular_file() -> Result<()> { + let temp = TempDir::new()?; + let external_path = temp.path().join("external"); + fs::create_dir(&external_path)?; + + let error = resolve_from(None, &external_path, temp.path()).unwrap_err(); + + assert!(error.to_string().contains("load external API key")); + assert!(format!("{error:#}").contains("must be a regular file")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn external_secret_allows_kubernetes_style_symlinks() -> Result<()> { + use std::os::unix::fs::symlink; + + let temp = TempDir::new()?; + let target_path = temp.path().join("target"); + let external_path = temp.path().join("external"); + fs::write(&target_path, format!("{TEST_KEY}\n"))?; + symlink(&target_path, &external_path)?; + + assert_eq!(resolve_from(None, &external_path, temp.path())?, TEST_KEY); + Ok(()) + } + #[test] fn managed_key_is_private_and_stable() -> Result<()> { let temp = TempDir::new()?; diff --git a/src/bin/server.rs b/src/bin/server.rs index f755bad9..f9f7d4ed 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -76,11 +76,11 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } - let api_key = agentenv::api_key::resolve(config)?; - agentenv::privileges::require_runtime_capabilities()?; agentenv::privileges::clear_ambient_capabilities()?; + let api_key = agentenv::api_key::resolve(config)?; + let addr = std::env::var("API_ADDR").unwrap_or_else(|_| "0.0.0.0:8000".to_string()); let identity = NodeIdentity::from_config(&config.node_identity); let p2p_transport = agentenv::p2p::transport_from_config(config, &identity).await?; diff --git a/src/managed_secret.rs b/src/managed_secret.rs index 4c9f7b95..1d88da8a 100644 --- a/src/managed_secret.rs +++ b/src/managed_secret.rs @@ -1,8 +1,7 @@ -#[cfg(unix)] -use std::fs::OpenOptions; +use std::ffi::OsStr; use std::fs::{self, File}; use std::io::{self, Read, Write}; -use std::path::Path; +use std::path::{Component, Path}; use anyhow::{bail, Context, Result}; @@ -12,10 +11,18 @@ pub(crate) enum CreateOutcome { } pub(crate) fn read(path: &Path, max_len: usize) -> Result> { - let parent = path.parent().context("managed secret path has no parent")?; - match validate_directory(parent) { - Ok(()) => {} + ensure_supported()?; + let (parent, file_name) = secret_path_parts(path)?; + let directory = match open_directory(parent, false) { + Ok(directory) => directory, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error) + .with_context(|| format!("open managed secret directory {}", parent.display())); + } + }; + match validate_directory(&directory) { + Ok(()) => {} Err(error) => { return Err(error).with_context(|| { format!("validate managed secret directory {}", parent.display()) @@ -23,7 +30,7 @@ pub(crate) fn read(path: &Path, max_len: usize) -> Result> { } } - match open(path) { + match open_secret(&directory, file_name) { Ok(file) => read_file(path, file, max_len).map(Some), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), Err(error) => Err(error).with_context(|| format!("open managed secret {}", path.display())), @@ -42,7 +49,7 @@ pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result Result max_len { @@ -82,86 +92,204 @@ pub(crate) fn read_file(path: &Path, mut file: File, max_len: usize) -> Result Result { ensure_supported()?; - let parent = path.parent().context("managed secret path has no parent")?; - create_directory(parent)?; - validate_directory_identity(parent).with_context(|| { + let (parent, file_name) = secret_path_parts(path)?; + let directory = open_directory(parent, true) + .with_context(|| format!("create managed secret directory {}", parent.display()))?; + validate_directory_identity(&directory).with_context(|| { format!( "validate managed secret directory ownership {}", parent.display() ) })?; - set_permissions(parent, 0o700)?; - validate_directory(parent) + // This directory is the dedicated `/secrets` store used only for + // AgentENV-managed credentials. Tightening an existing volume root is part + // of establishing that store's contract. + set_directory_permissions(&directory)?; + validate_directory(&directory) .with_context(|| format!("validate managed secret directory {}", parent.display()))?; - let mut temporary = tempfile::NamedTempFile::new_in(parent) - .with_context(|| format!("create temporary secret in {}", parent.display()))?; - set_permissions(temporary.path(), 0o600)?; - temporary - .write_all(contents) - .with_context(|| format!("write temporary secret in {}", parent.display()))?; - temporary - .as_file() - .sync_all() - .with_context(|| format!("sync temporary secret in {}", parent.display()))?; - - match temporary.persist_noclobber(path) { - Ok(_) => { - File::open(parent) - .and_then(|directory| directory.sync_all()) - .with_context(|| format!("sync managed secret directory {}", parent.display()))?; - Ok(CreateOutcome::Created) - } - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => { - validate_directory(parent).with_context(|| { - format!("validate managed secret directory {}", parent.display()) - })?; - open(path) - .map(CreateOutcome::Existing) - .with_context(|| format!("open managed secret {}", path.display())) - } - Err(error) => { - Err(error.error).with_context(|| format!("persist managed secret {}", path.display())) - } - } + create_secret(&directory, file_name, path, contents) } -fn create_directory(path: &Path) -> Result<()> { - let mut builder = fs::DirBuilder::new(); - builder.recursive(true); - - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt; - - builder.mode(0o700); +fn secret_path_parts(path: &Path) -> Result<(&Path, &OsStr)> { + let parent = path.parent().context("managed secret path has no parent")?; + if parent.file_name() != Some(OsStr::new("secrets")) { + bail!( + "managed secret parent {} must be a dedicated directory named secrets", + parent.display() + ); } - - builder - .create(path) - .with_context(|| format!("create managed secret directory {}", path.display())) + let file_name = path + .file_name() + .context("managed secret path has no file name")?; + Ok((parent, file_name)) } #[cfg(unix)] -fn open(path: &Path) -> io::Result { - let mut options = OpenOptions::new(); - options.read(true); +fn open_directory(path: &Path, create: bool) -> io::Result { + use nix::errno::Errno; + use nix::fcntl::{openat, OFlag}; + use nix::sys::stat::{mkdirat, Mode}; + + let mut directory = if path.is_absolute() { + File::open("/")? + } else { + File::open(".")? + }; + let flags = OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW; + + for component in path.components() { + let name = match component { + Component::RootDir | Component::CurDir => continue, + Component::Normal(name) => name, + Component::ParentDir | Component::Prefix(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "managed secret paths must not contain parent components", + )); + } + }; + + let next = match openat(&directory, name, flags, Mode::empty()) { + Ok(next) => next, + Err(Errno::ENOENT) if create => { + match mkdirat(&directory, name, Mode::S_IRWXU) { + Ok(()) | Err(Errno::EEXIST) => {} + Err(error) => return Err(nix_io_error(error)), + } + openat(&directory, name, flags, Mode::empty()).map_err(directory_open_error)? + } + Err(error) => return Err(directory_open_error(error)), + }; + directory = File::from(next); + } - use std::os::unix::fs::OpenOptionsExt; + Ok(directory) +} - options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); +#[cfg(not(unix))] +fn open_directory(_path: &Path, _create: bool) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "managed secrets require Unix no-follow file semantics", + )) +} - options.open(path) +#[cfg(unix)] +fn open_secret(directory: &File, file_name: &OsStr) -> io::Result { + use nix::fcntl::{openat, OFlag}; + use nix::sys::stat::Mode; + + openat( + directory, + file_name, + OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK, + Mode::empty(), + ) + .map(File::from) + .map_err(nix_io_error) } #[cfg(not(unix))] -fn open(_path: &Path) -> io::Result { +fn open_secret(_directory: &File, _file_name: &OsStr) -> io::Result { Err(io::Error::new( io::ErrorKind::Unsupported, "managed secrets require Unix no-follow file semantics", )) } +#[cfg(unix)] +fn create_secret( + directory: &File, + file_name: &OsStr, + display_path: &Path, + contents: &[u8], +) -> Result { + use nix::errno::Errno; + use nix::fcntl::{openat, AtFlags, OFlag}; + use nix::sys::stat::Mode; + use nix::unistd::{linkat, unlinkat, UnlinkatFlags}; + + let temporary_name = format!(".agentenv-secret-{}", uuid::Uuid::now_v7()); + let temporary_fd = openat( + directory, + temporary_name.as_str(), + OFlag::O_WRONLY | OFlag::O_CLOEXEC | OFlag::O_CREAT | OFlag::O_EXCL | OFlag::O_NOFOLLOW, + Mode::S_IRUSR | Mode::S_IWUSR, + ) + .map_err(nix_io_error) + .with_context(|| format!("create temporary secret for {}", display_path.display()))?; + let mut temporary = File::from(temporary_fd); + + let outcome = (|| -> Result { + temporary + .write_all(contents) + .with_context(|| format!("write temporary secret for {}", display_path.display()))?; + temporary + .sync_all() + .with_context(|| format!("sync temporary secret for {}", display_path.display()))?; + + match linkat( + directory, + temporary_name.as_str(), + directory, + file_name, + AtFlags::empty(), + ) { + Ok(()) => Ok(CreateOutcome::Created), + Err(Errno::EEXIST) => open_secret(directory, file_name) + .map(CreateOutcome::Existing) + .with_context(|| format!("open managed secret {}", display_path.display())), + Err(error) => Err(nix_io_error(error)) + .with_context(|| format!("persist managed secret {}", display_path.display())), + } + })(); + + let _ = unlinkat( + directory, + temporary_name.as_str(), + UnlinkatFlags::NoRemoveDir, + ); + if matches!(&outcome, Ok(CreateOutcome::Created)) { + directory.sync_all().with_context(|| { + format!( + "sync managed secret directory for {}", + display_path.display() + ) + })?; + } + outcome +} + +#[cfg(not(unix))] +fn create_secret( + _directory: &File, + _file_name: &OsStr, + _display_path: &Path, + _contents: &[u8], +) -> Result { + bail!("managed secrets require Unix no-follow file semantics") +} + +#[cfg(unix)] +fn nix_io_error(error: nix::errno::Errno) -> io::Error { + io::Error::from_raw_os_error(error as i32) +} + +#[cfg(unix)] +fn directory_open_error(error: nix::errno::Errno) -> io::Error { + use nix::errno::Errno; + + if matches!(error, Errno::ELOOP | Errno::ENOTDIR) { + io::Error::new( + io::ErrorKind::InvalidInput, + "must be a directory and not a symbolic link", + ) + } else { + nix_io_error(error) + } +} + #[cfg(unix)] fn ensure_supported() -> Result<()> { Ok(()) @@ -172,14 +300,14 @@ fn ensure_supported() -> Result<()> { bail!("managed secrets require Unix no-follow file semantics") } -fn validate_directory(path: &Path) -> io::Result<()> { - let metadata = validate_directory_identity(path)?; +fn validate_directory(directory: &File) -> io::Result<()> { + let metadata = validate_directory_identity(directory)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let mode = metadata.permissions().mode() & 0o777; + let mode = metadata.permissions().mode() & 0o7777; if mode != 0o700 { return Err(io::Error::new( io::ErrorKind::PermissionDenied, @@ -191,12 +319,12 @@ fn validate_directory(path: &Path) -> io::Result<()> { Ok(()) } -fn validate_directory_identity(path: &Path) -> io::Result { - let metadata = fs::symlink_metadata(path)?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { +fn validate_directory_identity(directory: &File) -> io::Result { + let metadata = directory.metadata()?; + if !metadata.is_dir() { return Err(io::Error::new( io::ErrorKind::InvalidInput, - "must be a directory and not a symbolic link", + "must be a directory", )); } @@ -220,14 +348,99 @@ fn validate_directory_identity(path: &Path) -> io::Result { } #[cfg(unix)] -fn set_permissions(path: &Path, mode: u32) -> Result<()> { - use std::os::unix::fs::PermissionsExt; +fn set_directory_permissions(directory: &File) -> Result<()> { + use nix::sys::stat::{fchmod, Mode}; - fs::set_permissions(path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("set permissions on {}", path.display())) + fchmod(directory, Mode::S_IRWXU).context("set managed secret directory permissions") } #[cfg(not(unix))] -fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { +fn set_directory_permissions(_directory: &File) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_file_rejects_unrepresentable_limit() -> Result<()> { + let file = tempfile::NamedTempFile::new()?; + let error = read_file(file.path(), file.reopen()?, usize::MAX).unwrap_err(); + + assert!(error + .to_string() + .contains("managed secret size limit is too large")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn create_rejects_non_dedicated_parent_without_changing_permissions() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let parent = tempfile::tempdir()?; + fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o750))?; + let error = match create(&parent.path().join("api-key"), b"secret") { + Ok(_) => panic!("non-dedicated parent must be rejected"), + Err(error) => error, + }; + + assert!(error + .to_string() + .contains("dedicated directory named secrets")); + assert_eq!( + fs::metadata(parent.path())?.permissions().mode() & 0o7777, + 0o750 + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn validation_rejects_special_permission_bits() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir()?; + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o1700))?; + let directory_file = File::open(directory.path())?; + let directory_error = validate_directory(&directory_file).unwrap_err(); + assert!(directory_error.to_string().contains("permissions 0700")); + + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700))?; + let secret_path = directory.path().join("secret"); + fs::write(&secret_path, "secret")?; + fs::set_permissions(&secret_path, fs::Permissions::from_mode(0o4600))?; + let file_error = read_file(&secret_path, File::open(&secret_path)?, 64).unwrap_err(); + assert!(file_error.to_string().contains("permissions 0600")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn rejects_symlinked_secret_ancestor() -> Result<()> { + use std::os::unix::fs::{symlink, PermissionsExt}; + + let root = tempfile::tempdir()?; + let real = root.path().join("real"); + let link = root.path().join("link"); + let secrets = real.join("secrets"); + fs::create_dir_all(&secrets)?; + fs::set_permissions(&secrets, fs::Permissions::from_mode(0o700))?; + let existing_secret = secrets.join("existing"); + fs::write(&existing_secret, "secret")?; + fs::set_permissions(&existing_secret, fs::Permissions::from_mode(0o600))?; + symlink(&real, &link)?; + + let read_error = read(&link.join("secrets/existing"), 64) + .expect_err("reads through symlinked ancestors must be rejected"); + assert!(format!("{read_error:#}").contains("managed secret directory")); + + let error = match create(&link.join("secrets/api-key"), b"secret") { + Ok(_) => panic!("symlinked ancestors must be rejected"), + Err(error) => error, + }; + assert!(format!("{error:#}").contains("managed secret directory")); + Ok(()) + } +} diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index 9f4a0e65..8dbd96fe 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -162,7 +162,7 @@ where // Restore persisted sandboxes from the previous run, keeping the paused // ones (with their state) for the paused-protection reconcile below. let persisted = persister.load_all(&factory).await?; - let managed_seed_must_exist = !persisted.is_empty(); + let managed_seed_must_exist = persisted_sandboxes_require_managed_seed(&persisted); let access_tokens = tokio::task::spawn_blocking(move || { SandboxAccessTokenGenerator::load_or_create(app_config, managed_seed_must_exist) }) @@ -2458,6 +2458,12 @@ where } } +fn persisted_sandboxes_require_managed_seed(persisted: &[SandboxMetadata]) -> bool { + persisted + .iter() + .any(|metadata| metadata.secure || !metadata.network_policy.allow_public_traffic) +} + #[cfg(test)] impl Orchestrator where @@ -2534,6 +2540,19 @@ where Ok(()) } + pub(crate) async fn set_allow_public_traffic_for_test( + &self, + sandbox_id: &SandboxId, + allow_public_traffic: bool, + ) -> Result<()> { + let Some(mut metadata) = self.store.get(sandbox_id).await? else { + return Err(OrchestratorError::SandboxNotFound(*sandbox_id)); + }; + metadata.network_policy.allow_public_traffic = allow_public_traffic; + self.store.update(metadata).await?; + Ok(()) + } + pub(crate) async fn remove_proxy_route_for_test(&self, sandbox_id: &SandboxId) { let _ = self.proxy_routes.write().await.remove(sandbox_id); } diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index af4d24ae..b08428eb 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -677,6 +677,22 @@ async fn new_loads_persisted_sandboxes_into_store() -> Result<()> { Ok(()) } +#[test] +fn managed_seed_continuity_only_applies_to_token_protected_sandboxes() { + let public_insecure = SandboxMetadata::default(); + assert!(!persisted_sandboxes_require_managed_seed( + std::slice::from_ref(&public_insecure) + )); + + let mut secure = public_insecure.clone(); + secure.secure = true; + assert!(persisted_sandboxes_require_managed_seed(&[secure])); + + let mut private = public_insecure; + private.network_policy.allow_public_traffic = false; + assert!(persisted_sandboxes_require_managed_seed(&[private])); +} + #[tokio::test] async fn new_returns_error_when_loading_persisted_sandboxes_fails() { setup(); diff --git a/src/sandbox/access.rs b/src/sandbox/access.rs index b6de8b0d..1ddbee77 100644 --- a/src/sandbox/access.rs +++ b/src/sandbox/access.rs @@ -58,9 +58,7 @@ impl SandboxAccessTokenGenerator { let managed_seed_path = config.home_path.join(MANAGED_SEED_RELATIVE_PATH); let seed = resolve_seed(&managed_seed_path, managed_seed_must_exist)?; - if config.sandbox.access_token_hash_seed.is_none() - && config.cluster.scheduler_endpoint.is_some() - { + if config.cluster.scheduler_endpoint.is_some() { warn!( path = %managed_seed_path.display(), "using a node-local managed sandbox access-token seed; configure AENV_SANDBOX_ACCESS_TOKEN_HASH_SEED with the same value on every node in a clustered deployment" @@ -121,7 +119,7 @@ fn resolve_seed(managed_path: &Path, managed_seed_must_exist: bool) -> Result Option { - Some(self.snapshot_config.common.control_plane_port) + let port = self.snapshot_config.common.control_plane_port; + (port != 0).then_some(port) } fn encode(&self) -> Result { @@ -1918,6 +1919,25 @@ mod tests { ) } + #[test] + fn paused_state_ignores_invalid_control_plane_port() { + let mut common = fresh_config().common; + common.control_plane_port = 0; + let state = FirecrackerPausedState::new(FirecrackerSnapshotConfig { + common, + vm_state_path: "snapshot/vm_state.bin".into(), + mem_overlaybd_config: OverlaybdConfig { + image_config_path: "snapshot/mem_image.json".into(), + read_only: true, + runtime_upper_mode: overlaybd::config::UpperMode::LogStructured, + }, + mem_virtual_size: 4096, + managed_snapshot_root: None, + }); + + assert_eq!(state.control_plane_port(), None); + } + fn overlaybd_config() -> FirecrackerSandboxConfig { let mut config = fresh_config(); config.common.ublk_config = Some(crate::sandbox::ublk::UblkConfig::overlaybd( diff --git a/src/sandbox/network/policy.rs b/src/sandbox/network/policy.rs index 205578de..6c42a021 100644 --- a/src/sandbox/network/policy.rs +++ b/src/sandbox/network/policy.rs @@ -73,20 +73,42 @@ impl SandboxNetworkEgressPolicy { } } -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct SandboxNetworkPolicy { + #[serde(default = "default_allow_public_traffic")] + pub allow_public_traffic: bool, pub base_policy: BaseSandboxNetworkPolicy, pub egress: SandboxNetworkEgressPolicy, } +fn default_allow_public_traffic() -> bool { + true +} + +impl Default for SandboxNetworkPolicy { + fn default() -> Self { + Self { + allow_public_traffic: true, + base_policy: BaseSandboxNetworkPolicy::default(), + egress: SandboxNetworkEgressPolicy::default(), + } + } +} + impl SandboxNetworkPolicy { pub fn new(base_policy: BaseSandboxNetworkPolicy, egress: SandboxNetworkEgressPolicy) -> Self { Self { + allow_public_traffic: true, base_policy, egress, } } + pub fn with_allow_public_traffic(mut self, allow_public_traffic: bool) -> Self { + self.allow_public_traffic = allow_public_traffic; + self + } + pub(crate) fn runtime_policy(&self) -> Option { self.has_runtime_egress_rules().then(|| self.clone()) } @@ -335,14 +357,29 @@ mod tests { SandboxNetworkEgressPolicy::new(Some(vec!["8.8.8.8/32".to_string()]), None).unwrap(), ); + assert!(policy.allow_public_traffic); assert_eq!(policy.base_policy, BaseSandboxNetworkPolicy::Deny); assert!(policy.egress.denied_cidrs.is_empty()); assert!(policy.has_runtime_egress_rules()); } + #[test] + fn missing_ingress_policy_deserializes_as_public() { + let mut value = serde_json::to_value(SandboxNetworkPolicy::default()).unwrap(); + value + .as_object_mut() + .unwrap() + .remove("allow_public_traffic"); + + let policy: SandboxNetworkPolicy = serde_json::from_value(value).unwrap(); + + assert!(policy.allow_public_traffic); + } + #[test] fn build_rules_keeps_allow_before_deny() { let policy = SandboxNetworkPolicy { + allow_public_traffic: true, base_policy: BaseSandboxNetworkPolicy::Deny, egress: SandboxNetworkEgressPolicy { allowed_cidrs: vec!["8.8.8.8/32".to_string()], @@ -371,6 +408,7 @@ mod tests { #[test] fn build_policy_replacement_flushes_before_installing_rules() { let policy = SandboxNetworkPolicy { + allow_public_traffic: true, base_policy: BaseSandboxNetworkPolicy::Deny, egress: SandboxNetworkEgressPolicy { allowed_cidrs: vec!["8.8.8.8/32".to_string()],