diff --git a/CLAUDE.md b/CLAUDE.md index 866a3f18..03d0508d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,6 +97,11 @@ - This triggers `examples/shared/Dockerfile.dev` which builds the server binary from local source and packages it into the Docker image. - Example: `SANDBOX_AGENT_DEV=1 pnpm --filter @sandbox-agent/example-mcp start` +## Docker Test Image + +- Docker-backed Rust and TypeScript tests build `docker/test-agent/Dockerfile` directly in-process and cache the image tag only in memory (`OnceLock` in Rust, module-level variable in TypeScript). +- Do not add cross-process image-build scripts unless there is a concrete need for them. + ## Install Version References - Channel policy: diff --git a/docker/test-agent/Dockerfile b/docker/test-agent/Dockerfile new file mode 100644 index 00000000..5494a4c0 --- /dev/null +++ b/docker/test-agent/Dockerfile @@ -0,0 +1,41 @@ +FROM rust:1.88.0-bookworm AS builder +WORKDIR /build + +COPY Cargo.toml Cargo.lock ./ +COPY server/ ./server/ +COPY gigacode/ ./gigacode/ +COPY resources/agent-schemas/artifacts/ ./resources/agent-schemas/artifacts/ +COPY scripts/agent-configs/ ./scripts/agent-configs/ + +ENV SANDBOX_AGENT_SKIP_INSPECTOR=1 + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/target \ + cargo build -p sandbox-agent --release && \ + cp target/release/sandbox-agent /sandbox-agent + +FROM node:22-bookworm-slim +RUN apt-get update -qq && \ + apt-get install -y -qq --no-install-recommends \ + ca-certificates \ + bash \ + libstdc++6 \ + xvfb \ + openbox \ + xdotool \ + imagemagick \ + x11-xserver-utils \ + dbus-x11 \ + xauth \ + fonts-dejavu-core \ + xterm \ + > /dev/null 2>&1 && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=builder /sandbox-agent /usr/local/bin/sandbox-agent + +EXPOSE 3000 + +ENTRYPOINT ["/usr/local/bin/sandbox-agent"] +CMD ["server", "--host", "0.0.0.0", "--port", "3000", "--no-token"] diff --git a/docs/cli.mdx b/docs/cli.mdx index fa6aa4ee..d83be2f7 100644 --- a/docs/cli.mdx +++ b/docs/cli.mdx @@ -37,6 +37,36 @@ Notes: - Set `SANDBOX_AGENT_LOG_STDOUT=1` to force stdout/stderr logging. - Use `SANDBOX_AGENT_LOG_DIR` to override log directory. +## install + +Install first-party runtime dependencies. + +### install desktop + +Install the Linux desktop runtime packages required by `/v1/desktop/*`. + +```bash +sandbox-agent install desktop [OPTIONS] +``` + +| Option | Description | +|--------|-------------| +| `--yes` | Skip the confirmation prompt | +| `--print-only` | Print the package-manager command without executing it | +| `--package-manager ` | Override package-manager detection | +| `--no-fonts` | Skip the default DejaVu font package | + +```bash +sandbox-agent install desktop --yes +sandbox-agent install desktop --print-only +``` + +Notes: + +- Supported on Linux only. +- The command detects `apt`, `dnf`, or `apk`. +- If the host is not already running as root, the command requires `sudo`. + ## install-agent Install or reinstall a single agent. diff --git a/docs/deploy/docker.mdx b/docs/deploy/docker.mdx index 988382a5..72a2ac4c 100644 --- a/docs/deploy/docker.mdx +++ b/docs/deploy/docker.mdx @@ -21,6 +21,27 @@ docker run --rm -p 3000:3000 \ sandbox-agent server --no-token --host 0.0.0.0 --port 3000" ``` +If you also want the desktop API inside the container, install desktop dependencies before starting the server: + +```bash +docker run --rm -p 3000:3000 \ + -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ + -e OPENAI_API_KEY="$OPENAI_API_KEY" \ + node:22-bookworm-slim sh -c "\ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates bash libstdc++6 && \ + rm -rf /var/lib/apt/lists/* && \ + curl -fsSL https://releases.rivet.dev/sandbox-agent/0.3.x/install.sh | sh && \ + sandbox-agent install desktop --yes && \ + sandbox-agent server --no-token --host 0.0.0.0 --port 3000" +``` + +In a Dockerfile: + +```dockerfile +RUN sandbox-agent install desktop --yes +``` + ## TypeScript with dockerode ```typescript diff --git a/docs/inspector.mdx b/docs/inspector.mdx index 06318b2f..4829499d 100644 --- a/docs/inspector.mdx +++ b/docs/inspector.mdx @@ -34,6 +34,7 @@ console.log(url); - Event JSON inspector - Prompt testing - Request/response debugging +- Desktop panel for status, remediation, start/stop, and screenshot refresh - Process management (create, stop, kill, delete, view logs) - Interactive PTY terminal for tty processes - One-shot command execution @@ -49,3 +50,16 @@ console.log(url); The Inspector includes an embedded Ghostty-based terminal for interactive tty processes. The UI uses the SDK's high-level `connectProcessTerminal(...)` wrapper via the shared `@sandbox-agent/react` `ProcessTerminal` component. + +## Desktop panel + +The `Desktop` panel shows the current desktop runtime state, missing dependencies, +the suggested install command, last error details, process/log paths, and the +latest captured screenshot. + +Use it to: + +- Check whether desktop dependencies are installed +- Start or stop the managed desktop runtime +- Refresh desktop status +- Capture a fresh screenshot on demand diff --git a/docs/openapi.json b/docs/openapi.json index b399f745..cc71dcc4 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -654,75 +654,41 @@ } } }, - "/v1/fs/entries": { + "/v1/desktop/display/info": { "get": { "tags": [ "v1" ], - "operationId": "get_v1_fs_entries", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "Directory path", - "required": false, - "schema": { - "type": "string", - "nullable": true - } - } - ], + "summary": "Get desktop display information.", + "description": "Performs a health-gated display query against the managed desktop and\nreturns the current display identifier and resolution.", + "operationId": "get_v1_desktop_display_info", "responses": { "200": { - "description": "Directory entries", + "description": "Desktop display information", "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/FsEntry" - } + "$ref": "#/components/schemas/DesktopDisplayInfoResponse" } } } - } - } - } - }, - "/v1/fs/entry": { - "delete": { - "tags": [ - "v1" - ], - "operationId": "delete_v1_fs_entry", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "File or directory path", - "required": true, - "schema": { - "type": "string" - } }, - { - "name": "recursive", - "in": "query", - "description": "Delete directory recursively", - "required": false, - "schema": { - "type": "boolean", - "nullable": true + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } } - } - ], - "responses": { - "200": { - "description": "Delete result", + }, + "503": { + "description": "Desktop runtime health or display query failed", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FsActionResponse" + "$ref": "#/components/schemas/ProblemDetails" } } } @@ -730,51 +696,19 @@ } } }, - "/v1/fs/file": { - "get": { - "tags": [ - "v1" - ], - "operationId": "get_v1_fs_file", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "File path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "File content" - } - } - }, - "put": { + "/v1/desktop/keyboard/press": { + "post": { "tags": [ "v1" ], - "operationId": "put_v1_fs_file", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "File path", - "required": true, - "schema": { - "type": "string" - } - } - ], + "summary": "Press a desktop keyboard shortcut.", + "description": "Performs a health-gated `xdotool key` operation against the managed\ndesktop.", + "operationId": "post_v1_desktop_keyboard_press", "requestBody": { - "description": "Raw file bytes", "content": { - "text/plain": { + "application/json": { "schema": { - "type": "string" + "$ref": "#/components/schemas/DesktopKeyboardPressRequest" } } }, @@ -782,103 +716,41 @@ }, "responses": { "200": { - "description": "Write result", + "description": "Desktop keyboard action result", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FsWriteResponse" + "$ref": "#/components/schemas/DesktopActionResponse" } } } - } - } - } - }, - "/v1/fs/mkdir": { - "post": { - "tags": [ - "v1" - ], - "operationId": "post_v1_fs_mkdir", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "Directory path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Directory created", + }, + "400": { + "description": "Invalid keyboard press request", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FsActionResponse" + "$ref": "#/components/schemas/ProblemDetails" } } } - } - } - } - }, - "/v1/fs/move": { - "post": { - "tags": [ - "v1" - ], - "operationId": "post_v1_fs_move", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/FsMoveRequest" - } - } }, - "required": true - }, - "responses": { - "200": { - "description": "Move result", + "409": { + "description": "Desktop runtime is not ready", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FsMoveResponse" + "$ref": "#/components/schemas/ProblemDetails" } } } - } - } - } - }, - "/v1/fs/stat": { - "get": { - "tags": [ - "v1" - ], - "operationId": "get_v1_fs_stat", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "Path to stat", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Path metadata", + }, + "502": { + "description": "Desktop runtime health or input failed", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FsStat" + "$ref": "#/components/schemas/ProblemDetails" } } } @@ -886,30 +758,19 @@ } } }, - "/v1/fs/upload-batch": { + "/v1/desktop/keyboard/type": { "post": { "tags": [ "v1" ], - "operationId": "post_v1_fs_upload_batch", - "parameters": [ - { - "name": "path", - "in": "query", - "description": "Destination path", - "required": false, - "schema": { - "type": "string", - "nullable": true - } - } - ], + "summary": "Type desktop keyboard text.", + "description": "Performs a health-gated `xdotool type` operation against the managed\ndesktop.", + "operationId": "post_v1_desktop_keyboard_type", "requestBody": { - "description": "tar archive body", "content": { - "text/plain": { + "application/json": { "schema": { - "type": "string" + "$ref": "#/components/schemas/DesktopKeyboardTypeRequest" } } }, @@ -917,59 +778,37 @@ }, "responses": { "200": { - "description": "Upload/extract result", + "description": "Desktop keyboard action result", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FsUploadBatchResponse" + "$ref": "#/components/schemas/DesktopActionResponse" } } } - } - } - } - }, - "/v1/health": { - "get": { - "tags": [ - "v1" - ], - "operationId": "get_v1_health", - "responses": { - "200": { - "description": "Service health response", + }, + "400": { + "description": "Invalid keyboard type request", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/HealthResponse" + "$ref": "#/components/schemas/ProblemDetails" } } } - } - } - } - }, - "/v1/processes": { - "get": { - "tags": [ - "v1" - ], - "summary": "List all managed processes.", - "description": "Returns a list of all processes (running and exited) currently tracked\nby the runtime, sorted by process ID.", - "operationId": "get_v1_processes", - "responses": { - "200": { - "description": "List processes", + }, + "409": { + "description": "Desktop runtime is not ready", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessListResponse" + "$ref": "#/components/schemas/ProblemDetails" } } } }, - "501": { - "description": "Process API unsupported on this platform", + "502": { + "description": "Desktop runtime health or input failed", "content": { "application/json": { "schema": { @@ -979,19 +818,21 @@ } } } - }, + } + }, + "/v1/desktop/mouse/click": { "post": { "tags": [ "v1" ], - "summary": "Create a long-lived managed process.", - "description": "Spawns a new process with the given command and arguments. Supports both\npipe-based and PTY (tty) modes. Returns the process descriptor on success.", - "operationId": "post_v1_processes", + "summary": "Click on the desktop.", + "description": "Performs a health-gated pointer move and click against the managed desktop\nand returns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_click", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessCreateRequest" + "$ref": "#/components/schemas/DesktopMouseClickRequest" } } }, @@ -999,17 +840,17 @@ }, "responses": { "200": { - "description": "Started process", + "description": "Desktop mouse position after click", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessInfo" + "$ref": "#/components/schemas/DesktopMousePositionResponse" } } } }, "400": { - "description": "Invalid request", + "description": "Invalid mouse click request", "content": { "application/json": { "schema": { @@ -1019,7 +860,7 @@ } }, "409": { - "description": "Process limit or state conflict", + "description": "Desktop runtime is not ready", "content": { "application/json": { "schema": { @@ -1028,8 +869,8 @@ } } }, - "501": { - "description": "Process API unsupported on this platform", + "502": { + "description": "Desktop runtime health or input failed", "content": { "application/json": { "schema": { @@ -1041,27 +882,57 @@ } } }, - "/v1/processes/config": { - "get": { + "/v1/desktop/mouse/drag": { + "post": { "tags": [ "v1" ], - "summary": "Get process runtime configuration.", - "description": "Returns the current runtime configuration for the process management API,\nincluding limits for concurrency, timeouts, and buffer sizes.", - "operationId": "get_v1_processes_config", + "summary": "Drag the desktop mouse.", + "description": "Performs a health-gated drag gesture against the managed desktop and\nreturns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_drag", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMouseDragRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "Current runtime process config", + "description": "Desktop mouse position after drag", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessConfig" + "$ref": "#/components/schemas/DesktopMousePositionResponse" } } } }, - "501": { - "description": "Process API unsupported on this platform", + "400": { + "description": "Invalid mouse drag request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "502": { + "description": "Desktop runtime health or input failed", "content": { "application/json": { "schema": { @@ -1071,19 +942,21 @@ } } } - }, + } + }, + "/v1/desktop/mouse/move": { "post": { "tags": [ "v1" ], - "summary": "Update process runtime configuration.", - "description": "Replaces the runtime configuration for the process management API.\nValidates that all values are non-zero and clamps default timeout to max.", - "operationId": "post_v1_processes_config", + "summary": "Move the desktop mouse.", + "description": "Performs a health-gated absolute pointer move on the managed desktop and\nreturns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_move", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessConfig" + "$ref": "#/components/schemas/DesktopMouseMoveRequest" } } }, @@ -1091,17 +964,17 @@ }, "responses": { "200": { - "description": "Updated runtime process config", + "description": "Desktop mouse position after move", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessConfig" + "$ref": "#/components/schemas/DesktopMousePositionResponse" } } } }, "400": { - "description": "Invalid config", + "description": "Invalid mouse move request", "content": { "application/json": { "schema": { @@ -1110,8 +983,18 @@ } } }, - "501": { - "description": "Process API unsupported on this platform", + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "502": { + "description": "Desktop runtime health or input failed", "content": { "application/json": { "schema": { @@ -1123,19 +1006,61 @@ } } }, - "/v1/processes/run": { + "/v1/desktop/mouse/position": { + "get": { + "tags": [ + "v1" + ], + "summary": "Get the current desktop mouse position.", + "description": "Performs a health-gated mouse position query against the managed desktop.", + "operationId": "get_v1_desktop_mouse_position", + "responses": { + "200": { + "description": "Desktop mouse position", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMousePositionResponse" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "502": { + "description": "Desktop runtime health or input check failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/mouse/scroll": { "post": { "tags": [ "v1" ], - "summary": "Run a one-shot command.", - "description": "Executes a command to completion and returns its stdout, stderr, exit code,\nand duration. Supports configurable timeout and output size limits.", - "operationId": "post_v1_processes_run", + "summary": "Scroll the desktop mouse wheel.", + "description": "Performs a health-gated scroll gesture at the requested coordinates and\nreturns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_scroll", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessRunRequest" + "$ref": "#/components/schemas/DesktopMouseScrollRequest" } } }, @@ -1143,17 +1068,17 @@ }, "responses": { "200": { - "description": "One-off command result", + "description": "Desktop mouse position after scroll", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessRunResponse" + "$ref": "#/components/schemas/DesktopMousePositionResponse" } } } }, "400": { - "description": "Invalid request", + "description": "Invalid mouse scroll request", "content": { "application/json": { "schema": { @@ -1162,8 +1087,18 @@ } } }, - "501": { - "description": "Process API unsupported on this platform", + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "502": { + "description": "Desktop runtime health or input failed", "content": { "application/json": { "schema": { @@ -1175,38 +1110,20 @@ } } }, - "/v1/processes/{id}": { + "/v1/desktop/screenshot": { "get": { "tags": [ "v1" ], - "summary": "Get a single process by ID.", - "description": "Returns the current state of a managed process including its status,\nPID, exit code, and creation/exit timestamps.", - "operationId": "get_v1_process", - "parameters": [ - { - "name": "id", - "in": "path", - "description": "Process ID", - "required": true, - "schema": { - "type": "string" - } - } - ], + "summary": "Capture a full desktop screenshot.", + "description": "Performs a health-gated full-frame screenshot of the managed desktop and\nreturns PNG bytes.", + "operationId": "get_v1_desktop_screenshot", "responses": { "200": { - "description": "Process details", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProcessInfo" - } - } - } + "description": "Desktop screenshot as PNG bytes" }, - "404": { - "description": "Unknown process", + "409": { + "description": "Desktop runtime is not ready", "content": { "application/json": { "schema": { @@ -1215,8 +1132,8 @@ } } }, - "501": { - "description": "Process API unsupported on this platform", + "502": { + "description": "Desktop runtime health or screenshot capture failed", "content": { "application/json": { "schema": { @@ -1226,31 +1143,66 @@ } } } - }, - "delete": { + } + }, + "/v1/desktop/screenshot/region": { + "get": { "tags": [ "v1" ], - "summary": "Delete a process record.", - "description": "Removes a stopped process from the runtime. Returns 409 if the process\nis still running; stop or kill it first.", - "operationId": "delete_v1_process", + "summary": "Capture a desktop screenshot region.", + "description": "Performs a health-gated screenshot crop against the managed desktop and\nreturns the requested PNG region bytes.", + "operationId": "get_v1_desktop_screenshot_region", "parameters": [ { - "name": "id", - "in": "path", - "description": "Process ID", + "name": "x", + "in": "query", + "description": "Region x coordinate", "required": true, "schema": { - "type": "string" + "type": "integer", + "format": "int32" + } + }, + { + "name": "y", + "in": "query", + "description": "Region y coordinate", + "required": true, + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "width", + "in": "query", + "description": "Region width", + "required": true, + "schema": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + }, + { + "name": "height", + "in": "query", + "description": "Region height", + "required": true, + "schema": { + "type": "integer", + "format": "int32", + "minimum": 0 } } ], "responses": { - "204": { - "description": "Process deleted" + "200": { + "description": "Desktop screenshot region as PNG bytes" }, - "404": { - "description": "Unknown process", + "400": { + "description": "Invalid screenshot region", "content": { "application/json": { "schema": { @@ -1260,7 +1212,7 @@ } }, "409": { - "description": "Process is still running", + "description": "Desktop runtime is not ready", "content": { "application/json": { "schema": { @@ -1269,8 +1221,8 @@ } } }, - "501": { - "description": "Process API unsupported on this platform", + "502": { + "description": "Desktop runtime health or screenshot capture failed", "content": { "application/json": { "schema": { @@ -1282,30 +1234,19 @@ } } }, - "/v1/processes/{id}/input": { + "/v1/desktop/start": { "post": { "tags": [ "v1" ], - "summary": "Write input to a process.", - "description": "Sends data to a process's stdin (pipe mode) or PTY writer (tty mode).\nData can be encoded as base64, utf8, or text. Returns 413 if the decoded\npayload exceeds the configured `maxInputBytesPerRequest` limit.", - "operationId": "post_v1_process_input", - "parameters": [ - { - "name": "id", - "in": "path", - "description": "Process ID", - "required": true, - "schema": { - "type": "string" - } - } - ], + "summary": "Start the private desktop runtime.", + "description": "Lazily launches the managed Xvfb/openbox stack, validates display health,\nand returns the resulting desktop status snapshot.", + "operationId": "post_v1_desktop_start", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessInputRequest" + "$ref": "#/components/schemas/DesktopStartRequest" } } }, @@ -1313,17 +1254,17 @@ }, "responses": { "200": { - "description": "Input accepted", + "description": "Desktop runtime status after start", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessInputResponse" + "$ref": "#/components/schemas/DesktopStatusResponse" } } } }, "400": { - "description": "Invalid request", + "description": "Invalid desktop start request", "content": { "application/json": { "schema": { @@ -1333,7 +1274,7 @@ } }, "409": { - "description": "Process not writable", + "description": "Desktop runtime is already transitioning", "content": { "application/json": { "schema": { @@ -1342,8 +1283,8 @@ } } }, - "413": { - "description": "Input exceeds configured limit", + "501": { + "description": "Desktop API unsupported on this platform", "content": { "application/json": { "schema": { @@ -1352,8 +1293,8 @@ } } }, - "501": { - "description": "Process API unsupported on this platform", + "503": { + "description": "Desktop runtime could not be started", "content": { "application/json": { "schema": { @@ -1365,60 +1306,27 @@ } } }, - "/v1/processes/{id}/kill": { - "post": { + "/v1/desktop/status": { + "get": { "tags": [ "v1" ], - "summary": "Send SIGKILL to a process.", - "description": "Sends SIGKILL to the process and optionally waits up to `waitMs`\nmilliseconds for the process to exit before returning.", - "operationId": "post_v1_process_kill", - "parameters": [ - { - "name": "id", - "in": "path", - "description": "Process ID", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "waitMs", - "in": "query", - "description": "Wait up to N ms for process to exit", - "required": false, - "schema": { - "type": "integer", - "format": "int64", - "nullable": true, - "minimum": 0 - } - } - ], + "summary": "Get desktop runtime status.", + "description": "Returns the current desktop runtime state, dependency status, active\ndisplay metadata, and supervised process information.", + "operationId": "get_v1_desktop_status", "responses": { "200": { - "description": "Kill signal sent", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProcessInfo" - } - } - } - }, - "404": { - "description": "Unknown process", + "description": "Desktop runtime status", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProblemDetails" + "$ref": "#/components/schemas/DesktopStatusResponse" } } } }, - "501": { - "description": "Process API unsupported on this platform", + "401": { + "description": "Authentication required", "content": { "application/json": { "schema": { @@ -1430,85 +1338,27 @@ } } }, - "/v1/processes/{id}/logs": { - "get": { + "/v1/desktop/stop": { + "post": { "tags": [ "v1" ], - "summary": "Fetch process logs.", - "description": "Returns buffered log entries for a process. Supports filtering by stream\ntype, tail count, and sequence-based resumption. When `follow=true`,\nreturns an SSE stream that replays buffered entries then streams live output.", - "operationId": "get_v1_process_logs", - "parameters": [ - { - "name": "id", - "in": "path", - "description": "Process ID", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "stream", - "in": "query", - "description": "stdout|stderr|combined|pty", - "required": false, - "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/ProcessLogsStream" - } - ], - "nullable": true - } - }, - { - "name": "tail", - "in": "query", - "description": "Tail N entries", - "required": false, - "schema": { - "type": "integer", - "nullable": true, - "minimum": 0 - } - }, - { - "name": "follow", - "in": "query", - "description": "Follow via SSE", - "required": false, - "schema": { - "type": "boolean", - "nullable": true - } - }, - { - "name": "since", - "in": "query", - "description": "Only entries with sequence greater than this", - "required": false, - "schema": { - "type": "integer", - "format": "int64", - "nullable": true, - "minimum": 0 - } - } - ], + "summary": "Stop the private desktop runtime.", + "description": "Terminates the managed openbox/Xvfb/dbus processes owned by the desktop\nruntime and returns the resulting status snapshot.", + "operationId": "post_v1_desktop_stop", "responses": { "200": { - "description": "Process logs", + "description": "Desktop runtime status after stop", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProcessLogsResponse" + "$ref": "#/components/schemas/DesktopStatusResponse" } } } }, - "404": { - "description": "Unknown process", + "409": { + "description": "Desktop runtime is already transitioning", "content": { "application/json": { "schema": { @@ -1516,13 +1366,38 @@ } } } - }, - "501": { - "description": "Process API unsupported on this platform", + } + } + } + }, + "/v1/fs/entries": { + "get": { + "tags": [ + "v1" + ], + "operationId": "get_v1_fs_entries", + "parameters": [ + { + "name": "path", + "in": "query", + "description": "Directory path", + "required": false, + "schema": { + "type": "string", + "nullable": true + } + } + ], + "responses": { + "200": { + "description": "Directory entries", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProblemDetails" + "type": "array", + "items": { + "$ref": "#/components/schemas/FsEntry" + } } } } @@ -1530,64 +1405,40 @@ } } }, - "/v1/processes/{id}/stop": { - "post": { + "/v1/fs/entry": { + "delete": { "tags": [ "v1" ], - "summary": "Send SIGTERM to a process.", - "description": "Sends SIGTERM to the process and optionally waits up to `waitMs`\nmilliseconds for the process to exit before returning.", - "operationId": "post_v1_process_stop", + "operationId": "delete_v1_fs_entry", "parameters": [ { - "name": "id", - "in": "path", - "description": "Process ID", + "name": "path", + "in": "query", + "description": "File or directory path", "required": true, "schema": { "type": "string" } }, { - "name": "waitMs", + "name": "recursive", "in": "query", - "description": "Wait up to N ms for process to exit", + "description": "Delete directory recursively", "required": false, "schema": { - "type": "integer", - "format": "int64", - "nullable": true, - "minimum": 0 + "type": "boolean", + "nullable": true } } ], "responses": { "200": { - "description": "Stop signal sent", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProcessInfo" - } - } - } - }, - "404": { - "description": "Unknown process", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - } - }, - "501": { - "description": "Process API unsupported on this platform", + "description": "Delete result", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProblemDetails" + "$ref": "#/components/schemas/FsActionResponse" } } } @@ -1595,19 +1446,39 @@ } } }, - "/v1/processes/{id}/terminal/resize": { - "post": { + "/v1/fs/file": { + "get": { "tags": [ "v1" ], - "summary": "Resize a process terminal.", - "description": "Sets the PTY window size (columns and rows) for a tty-mode process and\nsends SIGWINCH so the child process can adapt.", - "operationId": "post_v1_process_terminal_resize", + "operationId": "get_v1_fs_file", "parameters": [ { - "name": "id", - "in": "path", - "description": "Process ID", + "name": "path", + "in": "query", + "description": "File path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "File content" + } + } + }, + "put": { + "tags": [ + "v1" + ], + "operationId": "put_v1_fs_file", + "parameters": [ + { + "name": "path", + "in": "query", + "description": "File path", "required": true, "schema": { "type": "string" @@ -1615,10 +1486,11 @@ } ], "requestBody": { + "description": "Raw file bytes", "content": { - "application/json": { + "text/plain": { "schema": { - "$ref": "#/components/schemas/ProcessTerminalResizeRequest" + "type": "string" } } }, @@ -1626,51 +1498,11 @@ }, "responses": { "200": { - "description": "Resize accepted", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProcessTerminalResizeResponse" - } - } - } - }, - "400": { - "description": "Invalid request", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - } - }, - "404": { - "description": "Unknown process", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - } - }, - "409": { - "description": "Not a terminal process", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProblemDetails" - } - } - } - }, - "501": { - "description": "Process API unsupported on this platform", + "description": "Write result", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProblemDetails" + "$ref": "#/components/schemas/FsWriteResponse" } } } @@ -1678,41 +1510,182 @@ } } }, - "/v1/processes/{id}/terminal/ws": { - "get": { + "/v1/fs/mkdir": { + "post": { "tags": [ "v1" ], - "summary": "Open an interactive WebSocket terminal session.", - "description": "Upgrades the connection to a WebSocket for bidirectional PTY I/O. Accepts\n`access_token` query param for browser-based auth (WebSocket API cannot\nsend custom headers). Streams raw PTY output as binary frames and accepts\nJSON control frames for input, resize, and close.", - "operationId": "get_v1_process_terminal_ws", + "operationId": "post_v1_fs_mkdir", "parameters": [ { - "name": "id", - "in": "path", - "description": "Process ID", + "name": "path", + "in": "query", + "description": "Directory path", "required": true, "schema": { "type": "string" } - }, - { - "name": "access_token", - "in": "query", - "description": "Bearer token alternative for WS auth", - "required": false, - "schema": { - "type": "string", - "nullable": true - } } ], "responses": { - "101": { - "description": "WebSocket upgraded" + "200": { + "description": "Directory created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FsActionResponse" + } + } + } + } + } + } + }, + "/v1/fs/move": { + "post": { + "tags": [ + "v1" + ], + "operationId": "post_v1_fs_move", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FsMoveRequest" + } + } }, - "400": { - "description": "Invalid websocket frame or upgrade request", + "required": true + }, + "responses": { + "200": { + "description": "Move result", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FsMoveResponse" + } + } + } + } + } + } + }, + "/v1/fs/stat": { + "get": { + "tags": [ + "v1" + ], + "operationId": "get_v1_fs_stat", + "parameters": [ + { + "name": "path", + "in": "query", + "description": "Path to stat", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Path metadata", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FsStat" + } + } + } + } + } + } + }, + "/v1/fs/upload-batch": { + "post": { + "tags": [ + "v1" + ], + "operationId": "post_v1_fs_upload_batch", + "parameters": [ + { + "name": "path", + "in": "query", + "description": "Destination path", + "required": false, + "schema": { + "type": "string", + "nullable": true + } + } + ], + "requestBody": { + "description": "tar archive body", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Upload/extract result", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FsUploadBatchResponse" + } + } + } + } + } + } + }, + "/v1/health": { + "get": { + "tags": [ + "v1" + ], + "operationId": "get_v1_health", + "responses": { + "200": { + "description": "Service health response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthResponse" + } + } + } + } + } + } + }, + "/v1/processes": { + "get": { + "tags": [ + "v1" + ], + "summary": "List all managed processes.", + "description": "Returns a list of all processes (running and exited) currently tracked\nby the runtime, sorted by process ID.", + "operationId": "get_v1_processes", + "responses": { + "200": { + "description": "List processes", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessListResponse" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", "content": { "application/json": { "schema": { @@ -1720,9 +1693,39 @@ } } } + } + } + }, + "post": { + "tags": [ + "v1" + ], + "summary": "Create a long-lived managed process.", + "description": "Spawns a new process with the given command and arguments. Supports both\npipe-based and PTY (tty) modes. Returns the process descriptor on success.", + "operationId": "post_v1_processes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessCreateRequest" + } + } }, - "404": { - "description": "Unknown process", + "required": true + }, + "responses": { + "200": { + "description": "Started process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessInfo" + } + } + } + }, + "400": { + "description": "Invalid request", "content": { "application/json": { "schema": { @@ -1732,7 +1735,7 @@ } }, "409": { - "description": "Not a terminal process", + "description": "Process limit or state conflict", "content": { "application/json": { "schema": { @@ -1753,276 +1756,1373 @@ } } } - } - }, - "components": { - "schemas": { - "AcpEnvelope": { - "type": "object", - "required": [ - "jsonrpc" + }, + "/v1/processes/config": { + "get": { + "tags": [ + "v1" ], - "properties": { - "error": { - "nullable": true - }, - "id": { - "nullable": true - }, - "jsonrpc": { - "type": "string" - }, - "method": { - "type": "string", - "nullable": true - }, - "params": { - "nullable": true + "summary": "Get process runtime configuration.", + "description": "Returns the current runtime configuration for the process management API,\nincluding limits for concurrency, timeouts, and buffer sizes.", + "operationId": "get_v1_processes_config", + "responses": { + "200": { + "description": "Current runtime process config", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessConfig" + } + } + } }, - "result": { - "nullable": true + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } } } }, - "AcpPostQuery": { - "type": "object", - "properties": { - "agent": { - "type": "string", - "nullable": true + "post": { + "tags": [ + "v1" + ], + "summary": "Update process runtime configuration.", + "description": "Replaces the runtime configuration for the process management API.\nValidates that all values are non-zero and clamps default timeout to max.", + "operationId": "post_v1_processes_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Updated runtime process config", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessConfig" + } + } + } + }, + "400": { + "description": "Invalid config", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/run": { + "post": { + "tags": [ + "v1" + ], + "summary": "Run a one-shot command.", + "description": "Executes a command to completion and returns its stdout, stderr, exit code,\nand duration. Supports configurable timeout and output size limits.", + "operationId": "post_v1_processes_run", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessRunRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "One-off command result", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessRunResponse" + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}": { + "get": { + "tags": [ + "v1" + ], + "summary": "Get a single process by ID.", + "description": "Returns the current state of a managed process including its status,\nPID, exit code, and creation/exit timestamps.", + "operationId": "get_v1_process", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Process details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessInfo" + } + } + } + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + }, + "delete": { + "tags": [ + "v1" + ], + "summary": "Delete a process record.", + "description": "Removes a stopped process from the runtime. Returns 409 if the process\nis still running; stop or kill it first.", + "operationId": "delete_v1_process", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Process deleted" + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Process is still running", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}/input": { + "post": { + "tags": [ + "v1" + ], + "summary": "Write input to a process.", + "description": "Sends data to a process's stdin (pipe mode) or PTY writer (tty mode).\nData can be encoded as base64, utf8, or text. Returns 413 if the decoded\npayload exceeds the configured `maxInputBytesPerRequest` limit.", + "operationId": "post_v1_process_input", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessInputRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Input accepted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessInputResponse" + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Process not writable", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "413": { + "description": "Input exceeds configured limit", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}/kill": { + "post": { + "tags": [ + "v1" + ], + "summary": "Send SIGKILL to a process.", + "description": "Sends SIGKILL to the process and optionally waits up to `waitMs`\nmilliseconds for the process to exit before returning.", + "operationId": "post_v1_process_kill", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "waitMs", + "in": "query", + "description": "Wait up to N ms for process to exit", + "required": false, + "schema": { + "type": "integer", + "format": "int64", + "nullable": true, + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Kill signal sent", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessInfo" + } + } + } + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}/logs": { + "get": { + "tags": [ + "v1" + ], + "summary": "Fetch process logs.", + "description": "Returns buffered log entries for a process. Supports filtering by stream\ntype, tail count, and sequence-based resumption. When `follow=true`,\nreturns an SSE stream that replays buffered entries then streams live output.", + "operationId": "get_v1_process_logs", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "stream", + "in": "query", + "description": "stdout|stderr|combined|pty", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ProcessLogsStream" + } + ], + "nullable": true + } + }, + { + "name": "tail", + "in": "query", + "description": "Tail N entries", + "required": false, + "schema": { + "type": "integer", + "nullable": true, + "minimum": 0 + } + }, + { + "name": "follow", + "in": "query", + "description": "Follow via SSE", + "required": false, + "schema": { + "type": "boolean", + "nullable": true + } + }, + { + "name": "since", + "in": "query", + "description": "Only entries with sequence greater than this", + "required": false, + "schema": { + "type": "integer", + "format": "int64", + "nullable": true, + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Process logs", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessLogsResponse" + } + } + } + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}/stop": { + "post": { + "tags": [ + "v1" + ], + "summary": "Send SIGTERM to a process.", + "description": "Sends SIGTERM to the process and optionally waits up to `waitMs`\nmilliseconds for the process to exit before returning.", + "operationId": "post_v1_process_stop", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "waitMs", + "in": "query", + "description": "Wait up to N ms for process to exit", + "required": false, + "schema": { + "type": "integer", + "format": "int64", + "nullable": true, + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Stop signal sent", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessInfo" + } + } + } + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}/terminal/resize": { + "post": { + "tags": [ + "v1" + ], + "summary": "Resize a process terminal.", + "description": "Sets the PTY window size (columns and rows) for a tty-mode process and\nsends SIGWINCH so the child process can adapt.", + "operationId": "post_v1_process_terminal_resize", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessTerminalResizeRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Resize accepted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProcessTerminalResizeResponse" + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Not a terminal process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/processes/{id}/terminal/ws": { + "get": { + "tags": [ + "v1" + ], + "summary": "Open an interactive WebSocket terminal session.", + "description": "Upgrades the connection to a WebSocket for bidirectional PTY I/O. Accepts\n`access_token` query param for browser-based auth (WebSocket API cannot\nsend custom headers). Streams raw PTY output as binary frames and accepts\nJSON control frames for input, resize, and close.", + "operationId": "get_v1_process_terminal_ws", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Process ID", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "access_token", + "in": "query", + "description": "Bearer token alternative for WS auth", + "required": false, + "schema": { + "type": "string", + "nullable": true + } + } + ], + "responses": { + "101": { + "description": "WebSocket upgraded" + }, + "400": { + "description": "Invalid websocket frame or upgrade request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "404": { + "description": "Unknown process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Not a terminal process", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Process API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "AcpEnvelope": { + "type": "object", + "required": [ + "jsonrpc" + ], + "properties": { + "error": { + "nullable": true + }, + "id": { + "nullable": true + }, + "jsonrpc": { + "type": "string" + }, + "method": { + "type": "string", + "nullable": true + }, + "params": { + "nullable": true + }, + "result": { + "nullable": true + } + } + }, + "AcpPostQuery": { + "type": "object", + "properties": { + "agent": { + "type": "string", + "nullable": true + } + } + }, + "AcpServerInfo": { + "type": "object", + "required": [ + "serverId", + "agent", + "createdAtMs" + ], + "properties": { + "agent": { + "type": "string" + }, + "createdAtMs": { + "type": "integer", + "format": "int64" + }, + "serverId": { + "type": "string" + } + } + }, + "AcpServerListResponse": { + "type": "object", + "required": [ + "servers" + ], + "properties": { + "servers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AcpServerInfo" + } + } + } + }, + "AgentCapabilities": { + "type": "object", + "required": [ + "planMode", + "permissions", + "questions", + "toolCalls", + "toolResults", + "textMessages", + "images", + "fileAttachments", + "sessionLifecycle", + "errorEvents", + "reasoning", + "status", + "commandExecution", + "fileChanges", + "mcpTools", + "streamingDeltas", + "itemStarted", + "sharedProcess" + ], + "properties": { + "commandExecution": { + "type": "boolean" + }, + "errorEvents": { + "type": "boolean" + }, + "fileAttachments": { + "type": "boolean" + }, + "fileChanges": { + "type": "boolean" + }, + "images": { + "type": "boolean" + }, + "itemStarted": { + "type": "boolean" + }, + "mcpTools": { + "type": "boolean" + }, + "permissions": { + "type": "boolean" + }, + "planMode": { + "type": "boolean" + }, + "questions": { + "type": "boolean" + }, + "reasoning": { + "type": "boolean" + }, + "sessionLifecycle": { + "type": "boolean" + }, + "sharedProcess": { + "type": "boolean" + }, + "status": { + "type": "boolean" + }, + "streamingDeltas": { + "type": "boolean" + }, + "textMessages": { + "type": "boolean" + }, + "toolCalls": { + "type": "boolean" + }, + "toolResults": { + "type": "boolean" } } }, - "AcpServerInfo": { + "AgentInfo": { "type": "object", "required": [ - "serverId", - "agent", - "createdAtMs" + "id", + "installed", + "credentialsAvailable", + "capabilities" ], "properties": { - "agent": { + "capabilities": { + "$ref": "#/components/schemas/AgentCapabilities" + }, + "configError": { + "type": "string", + "nullable": true + }, + "configOptions": { + "type": "array", + "items": {}, + "nullable": true + }, + "credentialsAvailable": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "installed": { + "type": "boolean" + }, + "path": { + "type": "string", + "nullable": true + }, + "serverStatus": { + "allOf": [ + { + "$ref": "#/components/schemas/ServerStatusInfo" + } + ], + "nullable": true + }, + "version": { + "type": "string", + "nullable": true + } + } + }, + "AgentInstallArtifact": { + "type": "object", + "required": [ + "kind", + "path", + "source" + ], + "properties": { + "kind": { + "type": "string" + }, + "path": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string", + "nullable": true + } + } + }, + "AgentInstallRequest": { + "type": "object", + "properties": { + "agentProcessVersion": { + "type": "string", + "nullable": true + }, + "agentVersion": { + "type": "string", + "nullable": true + }, + "reinstall": { + "type": "boolean", + "nullable": true + } + } + }, + "AgentInstallResponse": { + "type": "object", + "required": [ + "already_installed", + "artifacts" + ], + "properties": { + "already_installed": { + "type": "boolean" + }, + "artifacts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentInstallArtifact" + } + } + } + }, + "AgentListResponse": { + "type": "object", + "required": [ + "agents" + ], + "properties": { + "agents": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentInfo" + } + } + } + }, + "DesktopActionResponse": { + "type": "object", + "required": [ + "ok" + ], + "properties": { + "ok": { + "type": "boolean" + } + } + }, + "DesktopDisplayInfoResponse": { + "type": "object", + "required": [ + "display", + "resolution" + ], + "properties": { + "display": { + "type": "string" + }, + "resolution": { + "$ref": "#/components/schemas/DesktopResolution" + } + } + }, + "DesktopErrorInfo": { + "type": "object", + "required": [ + "code", + "message" + ], + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + } + }, + "DesktopKeyboardPressRequest": { + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + } + } + }, + "DesktopKeyboardTypeRequest": { + "type": "object", + "required": [ + "text" + ], + "properties": { + "delayMs": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "text": { "type": "string" + } + } + }, + "DesktopMouseButton": { + "type": "string", + "enum": [ + "left", + "middle", + "right" + ] + }, + "DesktopMouseClickRequest": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "button": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopMouseButton" + } + ], + "nullable": true + }, + "clickCount": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopMouseDragRequest": { + "type": "object", + "required": [ + "startX", + "startY", + "endX", + "endY" + ], + "properties": { + "button": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopMouseButton" + } + ], + "nullable": true + }, + "endX": { + "type": "integer", + "format": "int32" }, - "createdAtMs": { + "endY": { "type": "integer", - "format": "int64" + "format": "int32" }, - "serverId": { - "type": "string" + "startX": { + "type": "integer", + "format": "int32" + }, + "startY": { + "type": "integer", + "format": "int32" } } }, - "AcpServerListResponse": { + "DesktopMouseMoveRequest": { "type": "object", "required": [ - "servers" + "x", + "y" ], "properties": { - "servers": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AcpServerInfo" - } + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" } } }, - "AgentCapabilities": { + "DesktopMousePositionResponse": { "type": "object", "required": [ - "planMode", - "permissions", - "questions", - "toolCalls", - "toolResults", - "textMessages", - "images", - "fileAttachments", - "sessionLifecycle", - "errorEvents", - "reasoning", - "status", - "commandExecution", - "fileChanges", - "mcpTools", - "streamingDeltas", - "itemStarted", - "sharedProcess" + "x", + "y" ], "properties": { - "commandExecution": { - "type": "boolean" - }, - "errorEvents": { - "type": "boolean" - }, - "fileAttachments": { - "type": "boolean" - }, - "fileChanges": { - "type": "boolean" - }, - "images": { - "type": "boolean" - }, - "itemStarted": { - "type": "boolean" - }, - "mcpTools": { - "type": "boolean" - }, - "permissions": { - "type": "boolean" - }, - "planMode": { - "type": "boolean" - }, - "questions": { - "type": "boolean" - }, - "reasoning": { - "type": "boolean" - }, - "sessionLifecycle": { - "type": "boolean" - }, - "sharedProcess": { - "type": "boolean" - }, - "status": { - "type": "boolean" - }, - "streamingDeltas": { - "type": "boolean" + "screen": { + "type": "integer", + "format": "int32", + "nullable": true }, - "textMessages": { - "type": "boolean" + "window": { + "type": "string", + "nullable": true }, - "toolCalls": { - "type": "boolean" + "x": { + "type": "integer", + "format": "int32" }, - "toolResults": { - "type": "boolean" + "y": { + "type": "integer", + "format": "int32" } } }, - "AgentInfo": { + "DesktopMouseScrollRequest": { "type": "object", "required": [ - "id", - "installed", - "credentialsAvailable", - "capabilities" + "x", + "y" ], "properties": { - "capabilities": { - "$ref": "#/components/schemas/AgentCapabilities" - }, - "configError": { - "type": "string", + "deltaX": { + "type": "integer", + "format": "int32", "nullable": true }, - "configOptions": { - "type": "array", - "items": {}, + "deltaY": { + "type": "integer", + "format": "int32", "nullable": true }, - "credentialsAvailable": { - "type": "boolean" - }, - "id": { - "type": "string" - }, - "installed": { - "type": "boolean" + "x": { + "type": "integer", + "format": "int32" }, - "path": { + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopProcessInfo": { + "type": "object", + "required": [ + "name", + "running" + ], + "properties": { + "logPath": { "type": "string", "nullable": true }, - "serverStatus": { - "allOf": [ - { - "$ref": "#/components/schemas/ServerStatusInfo" - } - ], - "nullable": true + "name": { + "type": "string" }, - "version": { - "type": "string", - "nullable": true + "pid": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "running": { + "type": "boolean" } } }, - "AgentInstallArtifact": { + "DesktopRegionScreenshotQuery": { "type": "object", "required": [ - "kind", - "path", - "source" + "x", + "y", + "width", + "height" ], "properties": { - "kind": { - "type": "string" + "height": { + "type": "integer", + "format": "int32", + "minimum": 0 }, - "path": { - "type": "string" + "width": { + "type": "integer", + "format": "int32", + "minimum": 0 }, - "source": { - "type": "string" + "x": { + "type": "integer", + "format": "int32" }, - "version": { - "type": "string", - "nullable": true + "y": { + "type": "integer", + "format": "int32" } } }, - "AgentInstallRequest": { + "DesktopResolution": { "type": "object", + "required": [ + "width", + "height" + ], "properties": { - "agentProcessVersion": { - "type": "string", - "nullable": true + "dpi": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 }, - "agentVersion": { - "type": "string", - "nullable": true + "height": { + "type": "integer", + "format": "int32", + "minimum": 0 }, - "reinstall": { - "type": "boolean", - "nullable": true + "width": { + "type": "integer", + "format": "int32", + "minimum": 0 } } }, - "AgentInstallResponse": { + "DesktopScreenshotQuery": { + "type": "object" + }, + "DesktopStartRequest": { "type": "object", - "required": [ - "already_installed", - "artifacts" - ], "properties": { - "already_installed": { - "type": "boolean" + "dpi": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 }, - "artifacts": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AgentInstallArtifact" - } + "height": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "width": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 } } }, - "AgentListResponse": { + "DesktopState": { + "type": "string", + "enum": [ + "inactive", + "install_required", + "starting", + "active", + "stopping", + "failed" + ] + }, + "DesktopStatusResponse": { "type": "object", "required": [ - "agents" + "state" ], "properties": { - "agents": { + "display": { + "type": "string", + "nullable": true + }, + "installCommand": { + "type": "string", + "nullable": true + }, + "lastError": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopErrorInfo" + } + ], + "nullable": true + }, + "missingDependencies": { "type": "array", "items": { - "$ref": "#/components/schemas/AgentInfo" + "type": "string" + } + }, + "processes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DesktopProcessInfo" } + }, + "resolution": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopResolution" + } + ], + "nullable": true + }, + "runtimeLogPath": { + "type": "string", + "nullable": true + }, + "startedAt": { + "type": "string", + "nullable": true + }, + "state": { + "$ref": "#/components/schemas/DesktopState" } } }, diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 0654e615..9fd6c92a 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -224,6 +224,16 @@ icon: "rocket" If agents are not installed up front, they are lazily installed when creating a session. + + If you want to use `/v1/desktop/*`, install the desktop runtime packages first: + + ```bash + sandbox-agent install desktop --yes + ``` + + Then use `GET /v1/desktop/status` or `sdk.getDesktopStatus()` to verify the runtime is ready before calling desktop screenshot or input APIs. + + ```typescript import { SandboxAgent } from "sandbox-agent"; diff --git a/docs/sdk-overview.mdx b/docs/sdk-overview.mdx index 5bd2a508..2a724e1d 100644 --- a/docs/sdk-overview.mdx +++ b/docs/sdk-overview.mdx @@ -177,6 +177,44 @@ const writeResult = await sdk.writeFsFile({ path: "./hello.txt" }, "hello"); console.log(health.status, agents.agents.length, entries.length, writeResult.path); ``` +## Desktop API + +The SDK also wraps the desktop host/runtime HTTP API. + +Install desktop dependencies first on Linux hosts: + +```bash +sandbox-agent install desktop --yes +``` + +Then query status, surface remediation if needed, and start the runtime: + +```ts +const status = await sdk.getDesktopStatus(); + +if (status.state === "install_required") { + console.log(status.installCommand); +} + +const started = await sdk.startDesktop({ + width: 1440, + height: 900, + dpi: 96, +}); + +const screenshot = await sdk.takeDesktopScreenshot(); +const displayInfo = await sdk.getDesktopDisplayInfo(); + +await sdk.moveDesktopMouse({ x: 400, y: 300 }); +await sdk.clickDesktop({ x: 400, y: 300, button: "left", clickCount: 1 }); +await sdk.typeDesktopText({ text: "hello world", delayMs: 10 }); +await sdk.pressDesktopKey({ key: "ctrl+l" }); + +await sdk.stopDesktop(); +``` + +Screenshot helpers return `Uint8Array` PNG bytes. The SDK does not attempt to install OS packages remotely; callers should surface `missingDependencies` and `installCommand` from `getDesktopStatus()`. + ## Error handling ```ts diff --git a/frontend/packages/inspector/index.html b/frontend/packages/inspector/index.html index aeec7968..c4dab86d 100644 --- a/frontend/packages/inspector/index.html +++ b/frontend/packages/inspector/index.html @@ -2768,6 +2768,94 @@ gap: 20px; } + .desktop-panel { + display: flex; + flex-direction: column; + gap: 16px; + } + + .desktop-state-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + margin-bottom: 12px; + } + + .desktop-start-controls { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 10px; + } + + .desktop-input-group { + display: flex; + flex-direction: column; + gap: 4px; + } + + .desktop-chip-list { + display: flex; + flex-wrap: wrap; + gap: 8px; + } + + .desktop-command { + margin-top: 6px; + padding: 8px 10px; + border-radius: var(--radius); + border: 1px solid var(--border); + background: var(--surface); + overflow-x: auto; + } + + .desktop-diagnostic-block + .desktop-diagnostic-block { + margin-top: 14px; + } + + .desktop-process-list { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 8px; + } + + .desktop-process-item { + padding: 10px; + border-radius: var(--radius); + border: 1px solid var(--border); + background: var(--surface); + display: flex; + flex-direction: column; + gap: 4px; + } + + .desktop-screenshot-empty { + padding: 18px; + border: 1px dashed var(--border); + border-radius: var(--radius); + color: var(--muted); + background: var(--surface); + text-align: center; + } + + .desktop-screenshot-frame { + border-radius: calc(var(--radius) + 2px); + overflow: hidden; + border: 1px solid var(--border); + background: + linear-gradient(135deg, rgba(15, 23, 42, 0.9), rgba(30, 41, 59, 0.92)), + radial-gradient(circle at top right, rgba(56, 189, 248, 0.12), transparent 40%); + padding: 10px; + } + + .desktop-screenshot-image { + display: block; + width: 100%; + height: auto; + border-radius: var(--radius); + background: rgba(0, 0, 0, 0.24); + } + .processes-section { display: flex; flex-direction: column; @@ -3430,6 +3518,11 @@ grid-template-columns: 1fr; } + .desktop-state-grid, + .desktop-start-controls { + grid-template-columns: 1fr; + } + .session-sidebar { display: none; } diff --git a/frontend/packages/inspector/package.json b/frontend/packages/inspector/package.json index 9671ecb7..28cd0c7c 100644 --- a/frontend/packages/inspector/package.json +++ b/frontend/packages/inspector/package.json @@ -18,6 +18,7 @@ "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.1", "fake-indexeddb": "^6.2.4", + "jsdom": "^26.1.0", "typescript": "^5.7.3", "vite": "^5.4.7", "vitest": "^3.0.0" diff --git a/frontend/packages/inspector/src/components/debug/DebugPanel.tsx b/frontend/packages/inspector/src/components/debug/DebugPanel.tsx index b15b59af..ae74f2a3 100644 --- a/frontend/packages/inspector/src/components/debug/DebugPanel.tsx +++ b/frontend/packages/inspector/src/components/debug/DebugPanel.tsx @@ -1,4 +1,4 @@ -import { ChevronLeft, ChevronRight, Cloud, Play, PlayCircle, Server, Terminal, Wrench } from "lucide-react"; +import { ChevronLeft, ChevronRight, Cloud, Monitor, Play, PlayCircle, Server, Terminal, Wrench } from "lucide-react"; import type { AgentInfo, SandboxAgent, SessionEvent } from "sandbox-agent"; type AgentModeInfo = { id: string; name: string; description: string }; @@ -9,9 +9,10 @@ import ProcessesTab from "./ProcessesTab"; import ProcessRunTab from "./ProcessRunTab"; import SkillsTab from "./SkillsTab"; import RequestLogTab from "./RequestLogTab"; +import DesktopTab from "./DesktopTab"; import type { RequestLog } from "../../types/requestLog"; -export type DebugTab = "log" | "events" | "agents" | "mcp" | "skills" | "processes" | "run-process"; +export type DebugTab = "log" | "events" | "agents" | "desktop" | "mcp" | "skills" | "processes" | "run-process"; const DebugPanel = ({ debugTab, @@ -79,6 +80,10 @@ const DebugPanel = ({ Agents + + + + + {error &&
{error}
} + {screenshotError &&
{screenshotError}
} + +
+
+ + + Desktop Runtime + + + {status?.state ?? "unknown"} + +
+ +
+
+
Display
+
{status?.display ?? "Not assigned"}
+
+
+
Resolution
+
{resolutionLabel}
+
+
+
Started
+
{formatStartedAt(status?.startedAt)}
+
+
+ +
+
+ + setWidth(event.target.value)} + inputMode="numeric" + /> +
+
+ + setHeight(event.target.value)} + inputMode="numeric" + /> +
+
+ + setDpi(event.target.value)} + inputMode="numeric" + /> +
+
+ +
+ + +
+
+ + {status?.missingDependencies && status.missingDependencies.length > 0 && ( +
+
+ Missing Dependencies +
+
+ {status.missingDependencies.map((dependency) => ( + {dependency} + ))} +
+ {status.installCommand && ( + <> +
Install command
+
{status.installCommand}
+ + )} +
+ )} + + {(status?.lastError || status?.runtimeLogPath || (status?.processes?.length ?? 0) > 0) && ( +
+
+ Diagnostics +
+ {status?.lastError && ( +
+
Last error
+
{status.lastError.code}
+
{status.lastError.message}
+
+ )} + {status?.runtimeLogPath && ( +
+
Runtime log
+
{status.runtimeLogPath}
+
+ )} + {status?.processes && status.processes.length > 0 && ( +
+
Processes
+
+ {status.processes.map((process) => ( +
+
+ {process.name} + + {process.running ? "running" : "stopped"} + +
+
+ {process.pid ? `pid ${process.pid}` : "no pid"} +
+ {process.logPath &&
{process.logPath}
} +
+ ))} +
+
+ )} +
+ )} + +
+
+ Latest Screenshot + {status?.state === "active" ? ( + Manual refresh only + ) : null} +
+ + {loading ?
Loading...
: null} + {!loading && !screenshotUrl && ( +
+ {status?.state === "active" + ? "No screenshot loaded yet." + : "Start the desktop runtime to capture a screenshot."} +
+ )} + {screenshotUrl && ( +
+ Desktop screenshot +
+ )} +
+ + ); +}; + +export default DesktopTab; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 45938ad6..a2e7ff61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,7 +17,7 @@ importers: version: 2.7.6 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/boxlite: dependencies: @@ -82,7 +82,7 @@ importers: version: 6.4.1(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) wrangler: specifier: latest version: 4.71.0(@cloudflare/workers-types@4.20260305.1) @@ -110,7 +110,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/daytona: dependencies: @@ -160,7 +160,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/e2b: dependencies: @@ -185,7 +185,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/file-system: dependencies: @@ -417,7 +417,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) frontend/packages/inspector: dependencies: @@ -449,6 +449,9 @@ importers: fake-indexeddb: specifier: ^6.2.4 version: 6.2.5 + jsdom: + specifier: ^26.1.0 + version: 26.1.0 sandbox-agent: specifier: workspace:* version: link:../../../sdks/typescript @@ -460,7 +463,7 @@ importers: version: 5.4.21(@types/node@25.3.5) vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) frontend/packages/website: dependencies: @@ -591,7 +594,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/cli: dependencies: @@ -617,7 +620,7 @@ importers: devDependencies: vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/cli-shared: devDependencies: @@ -665,7 +668,7 @@ importers: devDependencies: vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/gigacode/platforms/darwin-arm64: {} @@ -697,7 +700,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-postgres: dependencies: @@ -722,7 +725,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-rivet: dependencies: @@ -744,7 +747,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-sqlite: dependencies: @@ -769,7 +772,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/react: dependencies: @@ -823,7 +826,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) ws: specifier: ^8.19.0 version: 8.19.0 @@ -839,6 +842,9 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@asamuzakjp/css-color@3.2.0': + resolution: {integrity: sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==} + '@asteasolutions/zod-to-openapi@8.4.0': resolution: {integrity: sha512-Ckp971tmTw4pnv+o7iK85ldBHBKk6gxMaoNyLn3c2Th/fKoTG8G3jdYuOanpdGqwlDB0z01FOjry2d32lfTqrA==} peerDependencies: @@ -1293,6 +1299,34 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} + '@csstools/color-helpers@5.1.0': + resolution: {integrity: sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==} + engines: {node: '>=18'} + + '@csstools/css-calc@2.1.4': + resolution: {integrity: sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-parser-algorithms': ^3.0.5 + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-color-parser@3.1.0': + resolution: {integrity: sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-parser-algorithms': ^3.0.5 + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-parser-algorithms@3.0.5': + resolution: {integrity: sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-tokenizer@3.0.4': + resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==} + engines: {node: '>=18'} + '@daytonaio/api-client@0.149.0': resolution: {integrity: sha512-tlqVFnJll4JUAY3Ictwl7kGI3jo6HP+AcHl8FsZg/lSG7t/SdlZVO9iPPt6kjxmY3WN8BYRI1NYtIFFh8SJolw==} @@ -2954,6 +2988,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -3351,9 +3389,17 @@ packages: resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + cssstyle@4.6.0: + resolution: {integrity: sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==} + engines: {node: '>=18'} + csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + data-urls@5.0.0: + resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} + engines: {node: '>=18'} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -3363,6 +3409,9 @@ packages: supports-color: optional: true + decimal.js@10.6.0: + resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==} + decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} @@ -3837,6 +3886,10 @@ packages: resolution: {integrity: sha512-gJnaDHXKDayjt8ue0n8Gs0A007yKXj4Xzb8+cNjZeYsSzzwKc0Lr+OZgYwVfB0pHfUs17EPoLvrOsEaJ9mj+Tg==} engines: {node: '>=16.9.0'} + html-encoding-sniffer@4.0.0: + resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} + engines: {node: '>=18'} + html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} @@ -3850,10 +3903,22 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + human-signals@5.0.0: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + iconv-lite@0.7.2: resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} engines: {node: '>=0.10.0'} @@ -3929,6 +3994,9 @@ packages: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} + is-potential-custom-element-name@1.0.1: + resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} + is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} @@ -3976,6 +4044,15 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true + jsdom@26.1.0: + resolution: {integrity: sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==} + engines: {node: '>=18'} + peerDependencies: + canvas: ^3.0.0 + peerDependenciesMeta: + canvas: + optional: true + jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} @@ -4333,6 +4410,9 @@ packages: nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + nwsapi@2.2.23: + resolution: {integrity: sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -4662,6 +4742,10 @@ packages: pump@3.0.3: resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + qs@6.14.1: resolution: {integrity: sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==} engines: {node: '>=0.6'} @@ -4840,6 +4924,9 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + rrweb-cssom@0.8.0: + resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==} + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -4857,6 +4944,10 @@ packages: resolution: {integrity: sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==} engines: {node: '>=11.0.0'} + saxes@6.0.0: + resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} + engines: {node: '>=v12.22.7'} + scheduler@0.23.2: resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} @@ -5053,6 +5144,9 @@ packages: engines: {node: '>=16'} hasBin: true + symbol-tree@3.2.4: + resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} + tailwindcss@3.4.19: resolution: {integrity: sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==} engines: {node: '>=14.0.0'} @@ -5120,6 +5214,13 @@ packages: resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} + tldts-core@6.1.86: + resolution: {integrity: sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==} + + tldts@6.1.86: + resolution: {integrity: sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==} + hasBin: true + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -5128,6 +5229,14 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tough-cookie@5.1.2: + resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==} + engines: {node: '>=16'} + + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -5523,9 +5632,30 @@ packages: vscode-languageserver-types@3.17.5: resolution: {integrity: sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==} + w3c-xmlserializer@5.0.0: + resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} + engines: {node: '>=18'} + web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation + + whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + which-pm-runs@1.1.0: resolution: {integrity: sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==} engines: {node: '>=4'} @@ -5606,6 +5736,13 @@ packages: resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} engines: {node: '>= 6.0'} + xml-name-validator@5.0.0: + resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} + engines: {node: '>=18'} + + xmlchars@2.2.0: + resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -5686,6 +5823,14 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@asamuzakjp/css-color@3.2.0': + dependencies: + '@csstools/css-calc': 2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-color-parser': 3.1.0(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + lru-cache: 10.4.3 + '@asteasolutions/zod-to-openapi@8.4.0(zod@4.3.6)': dependencies: openapi3-ts: 4.5.0 @@ -6516,6 +6661,26 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.9 + '@csstools/color-helpers@5.1.0': {} + + '@csstools/css-calc@2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-color-parser@3.1.0(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/color-helpers': 5.1.0 + '@csstools/css-calc': 2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-tokenizer@3.0.4': {} + '@daytonaio/api-client@0.149.0': dependencies: axios: 1.13.5 @@ -8093,6 +8258,8 @@ snapshots: acorn@8.15.0: {} + agent-base@7.1.4: {} + ajv-formats@3.0.1(ajv@8.17.1): optionalDependencies: ajv: 8.17.1 @@ -8563,12 +8730,24 @@ snapshots: dependencies: css-tree: 2.2.1 + cssstyle@4.6.0: + dependencies: + '@asamuzakjp/css-color': 3.2.0 + rrweb-cssom: 0.8.0 + csstype@3.2.3: {} + data-urls@5.0.0: + dependencies: + whatwg-mimetype: 4.0.0 + whatwg-url: 14.2.0 + debug@4.4.3: dependencies: ms: 2.1.3 + decimal.js@10.6.0: {} + decode-named-character-reference@1.3.0: dependencies: character-entities: 2.0.2 @@ -9203,6 +9382,10 @@ snapshots: hono@4.12.2: {} + html-encoding-sniffer@4.0.0: + dependencies: + whatwg-encoding: 3.1.1 + html-escaper@3.0.3: {} html-void-elements@3.0.0: {} @@ -9217,8 +9400,26 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + human-signals@5.0.0: {} + iconv-lite@0.6.3: + dependencies: + safer-buffer: 2.1.2 + iconv-lite@0.7.2: dependencies: safer-buffer: 2.1.2 @@ -9276,6 +9477,8 @@ snapshots: is-plain-obj@4.1.0: {} + is-potential-custom-element-name@1.0.1: {} + is-promise@4.0.0: {} is-stream@3.0.0: {} @@ -9314,6 +9517,33 @@ snapshots: dependencies: argparse: 2.0.1 + jsdom@26.1.0: + dependencies: + cssstyle: 4.6.0 + data-urls: 5.0.0 + decimal.js: 10.6.0 + html-encoding-sniffer: 4.0.0 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + is-potential-custom-element-name: 1.0.1 + nwsapi: 2.2.23 + parse5: 7.3.0 + rrweb-cssom: 0.8.0 + saxes: 6.0.0 + symbol-tree: 3.2.4 + tough-cookie: 5.1.2 + w3c-xmlserializer: 5.0.0 + webidl-conversions: 7.0.0 + whatwg-encoding: 3.1.1 + whatwg-mimetype: 4.0.0 + whatwg-url: 14.2.0 + ws: 8.19.0 + xml-name-validator: 5.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + jsesc@3.1.0: {} json-schema-traverse@1.0.0: {} @@ -9817,6 +10047,8 @@ snapshots: dependencies: boolbase: 1.0.0 + nwsapi@2.2.23: {} + object-assign@4.1.1: {} object-hash@3.0.0: {} @@ -10140,6 +10372,8 @@ snapshots: end-of-stream: 1.4.5 once: 1.4.0 + punycode@2.3.1: {} + qs@6.14.1: dependencies: side-channel: 1.1.0 @@ -10399,6 +10633,8 @@ snapshots: transitivePeerDependencies: - supports-color + rrweb-cssom@0.8.0: {} + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -10411,6 +10647,10 @@ snapshots: sax@1.4.4: {} + saxes@6.0.0: + dependencies: + xmlchars: 2.2.0 + scheduler@0.23.2: dependencies: loose-envify: 1.4.0 @@ -10666,6 +10906,8 @@ snapshots: picocolors: 1.1.1 sax: 1.4.4 + symbol-tree@3.2.4: {} + tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.2): dependencies: '@alloc/quick-lru': 5.2.0 @@ -10771,12 +11013,26 @@ snapshots: tinyspy@4.0.4: {} + tldts-core@6.1.86: {} + + tldts@6.1.86: + dependencies: + tldts-core: 6.1.86 + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 toidentifier@1.0.1: {} + tough-cookie@5.1.2: + dependencies: + tldts: 6.1.86 + + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} trim-lines@3.0.1: {} @@ -11103,7 +11359,7 @@ snapshots: optionalDependencies: vite: 6.4.1(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) - vitest@3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2): + vitest@3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 @@ -11131,6 +11387,7 @@ snapshots: optionalDependencies: '@types/debug': 4.1.12 '@types/node': 22.19.7 + jsdom: 26.1.0 transitivePeerDependencies: - jiti - less @@ -11145,7 +11402,7 @@ snapshots: - tsx - yaml - vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2): + vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.3.5)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 @@ -11173,6 +11430,7 @@ snapshots: optionalDependencies: '@types/debug': 4.1.12 '@types/node': 25.3.5 + jsdom: 26.1.0 transitivePeerDependencies: - jiti - less @@ -11191,8 +11449,25 @@ snapshots: vscode-languageserver-types@3.17.5: {} + w3c-xmlserializer@5.0.0: + dependencies: + xml-name-validator: 5.0.0 + web-namespaces@2.0.1: {} + webidl-conversions@7.0.0: {} + + whatwg-encoding@3.1.1: + dependencies: + iconv-lite: 0.6.3 + + whatwg-mimetype@4.0.0: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + which-pm-runs@1.1.0: {} which@2.0.2: @@ -11265,6 +11540,10 @@ snapshots: dependencies: os-paths: 4.4.0 + xml-name-validator@5.0.0: {} + + xmlchars@2.2.0: {} + xtend@4.0.2: {} xxhash-wasm@1.1.0: {} diff --git a/research/acp/friction.md b/research/acp/friction.md index 023b4f6a..78d3dbd2 100644 --- a/research/acp/friction.md +++ b/research/acp/friction.md @@ -247,3 +247,13 @@ Update this file continuously during the migration. - Owner: Unassigned. - Status: in_progress - Links: `research/acp/simplify-server.md`, `docs/mcp-config.mdx`, `docs/skills-config.mdx` + +- Date: 2026-03-07 +- Area: Desktop host/runtime API boundary +- Issue: Desktop automation needed screenshot/input/file-transfer-like host capabilities, but routing it through ACP would have mixed agent protocol semantics with host-owned runtime control and binary payloads. +- Impact: A desktop feature built as ACP methods would blur the division between agent/session behavior and Sandbox Agent host/runtime APIs, and would complicate binary screenshot transport. +- Proposed direction: Ship desktop as first-party HTTP endpoints under `/v1/desktop/*`, keep health/install/remediation in the server runtime, and expose the feature through the SDK and inspector without ACP extension methods. +- Decision: Accepted and implemented for phase one. +- Owner: Unassigned. +- Status: resolved +- Links: `server/packages/sandbox-agent/src/router.rs`, `server/packages/sandbox-agent/src/desktop_runtime.rs`, `sdks/typescript/src/client.ts`, `frontend/packages/inspector/src/components/debug/DesktopTab.tsx` diff --git a/sdks/typescript/src/client.ts b/sdks/typescript/src/client.ts index 9db97628..195431e0 100644 --- a/sdks/typescript/src/client.ts +++ b/sdks/typescript/src/client.ts @@ -25,6 +25,19 @@ import { type AgentInstallRequest, type AgentInstallResponse, type AgentListResponse, + type DesktopActionResponse, + type DesktopDisplayInfoResponse, + type DesktopKeyboardPressRequest, + type DesktopKeyboardTypeRequest, + type DesktopMouseClickRequest, + type DesktopMouseDragRequest, + type DesktopMouseMoveRequest, + type DesktopMousePositionResponse, + type DesktopMouseScrollRequest, + type DesktopRegionScreenshotQuery, + type DesktopScreenshotQuery, + type DesktopStartRequest, + type DesktopStatusResponse, type FsActionResponse, type FsDeleteQuery, type FsEntriesQuery, @@ -1294,6 +1307,82 @@ export class SandboxAgent { return this.requestHealth(); } + async startDesktop(request: DesktopStartRequest = {}): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/start`, { + body: request, + }); + } + + async stopDesktop(): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/stop`); + } + + async getDesktopStatus(): Promise { + return this.requestJson("GET", `${API_PREFIX}/desktop/status`); + } + + async getDesktopDisplayInfo(): Promise { + return this.requestJson("GET", `${API_PREFIX}/desktop/display/info`); + } + + async takeDesktopScreenshot(query: DesktopScreenshotQuery = {}): Promise { + const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot`, { + query, + accept: "image/png", + }); + const buffer = await response.arrayBuffer(); + return new Uint8Array(buffer); + } + + async takeDesktopRegionScreenshot(query: DesktopRegionScreenshotQuery): Promise { + const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot/region`, { + query, + accept: "image/png", + }); + const buffer = await response.arrayBuffer(); + return new Uint8Array(buffer); + } + + async getDesktopMousePosition(): Promise { + return this.requestJson("GET", `${API_PREFIX}/desktop/mouse/position`); + } + + async moveDesktopMouse(request: DesktopMouseMoveRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/move`, { + body: request, + }); + } + + async clickDesktop(request: DesktopMouseClickRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/click`, { + body: request, + }); + } + + async dragDesktopMouse(request: DesktopMouseDragRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/drag`, { + body: request, + }); + } + + async scrollDesktop(request: DesktopMouseScrollRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/scroll`, { + body: request, + }); + } + + async typeDesktopText(request: DesktopKeyboardTypeRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/type`, { + body: request, + }); + } + + async pressDesktopKey(request: DesktopKeyboardPressRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/press`, { + body: request, + }); + } + async listAgents(options?: AgentQueryOptions): Promise { return this.requestJson("GET", `${API_PREFIX}/agents`, { query: toAgentQuery(options), diff --git a/sdks/typescript/src/generated/openapi.ts b/sdks/typescript/src/generated/openapi.ts index 18374fb3..a5cd07bd 100644 --- a/sdks/typescript/src/generated/openapi.ts +++ b/sdks/typescript/src/generated/openapi.ts @@ -32,6 +32,109 @@ export interface paths { put: operations["put_v1_config_skills"]; delete: operations["delete_v1_config_skills"]; }; + "/v1/desktop/display/info": { + /** + * Get desktop display information. + * @description Performs a health-gated display query against the managed desktop and + * returns the current display identifier and resolution. + */ + get: operations["get_v1_desktop_display_info"]; + }; + "/v1/desktop/keyboard/press": { + /** + * Press a desktop keyboard shortcut. + * @description Performs a health-gated `xdotool key` operation against the managed + * desktop. + */ + post: operations["post_v1_desktop_keyboard_press"]; + }; + "/v1/desktop/keyboard/type": { + /** + * Type desktop keyboard text. + * @description Performs a health-gated `xdotool type` operation against the managed + * desktop. + */ + post: operations["post_v1_desktop_keyboard_type"]; + }; + "/v1/desktop/mouse/click": { + /** + * Click on the desktop. + * @description Performs a health-gated pointer move and click against the managed desktop + * and returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_click"]; + }; + "/v1/desktop/mouse/drag": { + /** + * Drag the desktop mouse. + * @description Performs a health-gated drag gesture against the managed desktop and + * returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_drag"]; + }; + "/v1/desktop/mouse/move": { + /** + * Move the desktop mouse. + * @description Performs a health-gated absolute pointer move on the managed desktop and + * returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_move"]; + }; + "/v1/desktop/mouse/position": { + /** + * Get the current desktop mouse position. + * @description Performs a health-gated mouse position query against the managed desktop. + */ + get: operations["get_v1_desktop_mouse_position"]; + }; + "/v1/desktop/mouse/scroll": { + /** + * Scroll the desktop mouse wheel. + * @description Performs a health-gated scroll gesture at the requested coordinates and + * returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_scroll"]; + }; + "/v1/desktop/screenshot": { + /** + * Capture a full desktop screenshot. + * @description Performs a health-gated full-frame screenshot of the managed desktop and + * returns PNG bytes. + */ + get: operations["get_v1_desktop_screenshot"]; + }; + "/v1/desktop/screenshot/region": { + /** + * Capture a desktop screenshot region. + * @description Performs a health-gated screenshot crop against the managed desktop and + * returns the requested PNG region bytes. + */ + get: operations["get_v1_desktop_screenshot_region"]; + }; + "/v1/desktop/start": { + /** + * Start the private desktop runtime. + * @description Lazily launches the managed Xvfb/openbox stack, validates display health, + * and returns the resulting desktop status snapshot. + */ + post: operations["post_v1_desktop_start"]; + }; + "/v1/desktop/status": { + /** + * Get desktop runtime status. + * @description Returns the current desktop runtime state, dependency status, active + * display metadata, and supervised process information. + */ + get: operations["get_v1_desktop_status"]; + }; + "/v1/desktop/stop": { + /** + * Stop the private desktop runtime. + * @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop + * runtime and returns the resulting status snapshot. + */ + post: operations["post_v1_desktop_stop"]; + }; "/v1/fs/entries": { get: operations["get_v1_fs_entries"]; }; @@ -234,6 +337,119 @@ export interface components { AgentListResponse: { agents: components["schemas"]["AgentInfo"][]; }; + DesktopActionResponse: { + ok: boolean; + }; + DesktopDisplayInfoResponse: { + display: string; + resolution: components["schemas"]["DesktopResolution"]; + }; + DesktopErrorInfo: { + code: string; + message: string; + }; + DesktopKeyboardPressRequest: { + key: string; + }; + DesktopKeyboardTypeRequest: { + /** Format: int32 */ + delayMs?: number | null; + text: string; + }; + /** @enum {string} */ + DesktopMouseButton: "left" | "middle" | "right"; + DesktopMouseClickRequest: { + button?: components["schemas"]["DesktopMouseButton"] | null; + /** Format: int32 */ + clickCount?: number | null; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopMouseDragRequest: { + button?: components["schemas"]["DesktopMouseButton"] | null; + /** Format: int32 */ + endX: number; + /** Format: int32 */ + endY: number; + /** Format: int32 */ + startX: number; + /** Format: int32 */ + startY: number; + }; + DesktopMouseMoveRequest: { + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopMousePositionResponse: { + /** Format: int32 */ + screen?: number | null; + window?: string | null; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopMouseScrollRequest: { + /** Format: int32 */ + deltaX?: number | null; + /** Format: int32 */ + deltaY?: number | null; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopProcessInfo: { + logPath?: string | null; + name: string; + /** Format: int32 */ + pid?: number | null; + running: boolean; + }; + DesktopRegionScreenshotQuery: { + /** Format: int32 */ + height: number; + /** Format: int32 */ + width: number; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopResolution: { + /** Format: int32 */ + dpi?: number | null; + /** Format: int32 */ + height: number; + /** Format: int32 */ + width: number; + }; + DesktopScreenshotQuery: Record; + DesktopStartRequest: { + /** Format: int32 */ + dpi?: number | null; + /** Format: int32 */ + height?: number | null; + /** Format: int32 */ + width?: number | null; + }; + /** @enum {string} */ + DesktopState: "inactive" | "install_required" | "starting" | "active" | "stopping" | "failed"; + DesktopStatusResponse: { + display?: string | null; + installCommand?: string | null; + lastError?: components["schemas"]["DesktopErrorInfo"] | null; + missingDependencies?: string[]; + processes?: components["schemas"]["DesktopProcessInfo"][]; + resolution?: components["schemas"]["DesktopResolution"] | null; + runtimeLogPath?: string | null; + startedAt?: string | null; + state: components["schemas"]["DesktopState"]; + }; /** @enum {string} */ ErrorType: "invalid_request" | "conflict" | "unsupported_agent" | "agent_not_installed" | "install_failed" | "agent_process_exited" | "token_invalid" | "permission_denied" | "not_acceptable" | "unsupported_media_type" | "not_found" | "session_not_found" | "session_already_exists" | "mode_not_supported" | "stream_error" | "timeout"; FsActionResponse: { @@ -811,6 +1027,441 @@ export interface operations { }; }; }; + /** + * Get desktop display information. + * @description Performs a health-gated display query against the managed desktop and + * returns the current display identifier and resolution. + */ + get_v1_desktop_display_info: { + responses: { + /** @description Desktop display information */ + 200: { + content: { + "application/json": components["schemas"]["DesktopDisplayInfoResponse"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or display query failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Press a desktop keyboard shortcut. + * @description Performs a health-gated `xdotool key` operation against the managed + * desktop. + */ + post_v1_desktop_keyboard_press: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopKeyboardPressRequest"]; + }; + }; + responses: { + /** @description Desktop keyboard action result */ + 200: { + content: { + "application/json": components["schemas"]["DesktopActionResponse"]; + }; + }; + /** @description Invalid keyboard press request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Type desktop keyboard text. + * @description Performs a health-gated `xdotool type` operation against the managed + * desktop. + */ + post_v1_desktop_keyboard_type: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopKeyboardTypeRequest"]; + }; + }; + responses: { + /** @description Desktop keyboard action result */ + 200: { + content: { + "application/json": components["schemas"]["DesktopActionResponse"]; + }; + }; + /** @description Invalid keyboard type request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Click on the desktop. + * @description Performs a health-gated pointer move and click against the managed desktop + * and returns the resulting mouse position. + */ + post_v1_desktop_mouse_click: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseClickRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after click */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse click request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Drag the desktop mouse. + * @description Performs a health-gated drag gesture against the managed desktop and + * returns the resulting mouse position. + */ + post_v1_desktop_mouse_drag: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseDragRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after drag */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse drag request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Move the desktop mouse. + * @description Performs a health-gated absolute pointer move on the managed desktop and + * returns the resulting mouse position. + */ + post_v1_desktop_mouse_move: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseMoveRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after move */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse move request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Get the current desktop mouse position. + * @description Performs a health-gated mouse position query against the managed desktop. + */ + get_v1_desktop_mouse_position: { + responses: { + /** @description Desktop mouse position */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input check failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Scroll the desktop mouse wheel. + * @description Performs a health-gated scroll gesture at the requested coordinates and + * returns the resulting mouse position. + */ + post_v1_desktop_mouse_scroll: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseScrollRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after scroll */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse scroll request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Capture a full desktop screenshot. + * @description Performs a health-gated full-frame screenshot of the managed desktop and + * returns PNG bytes. + */ + get_v1_desktop_screenshot: { + responses: { + /** @description Desktop screenshot as PNG bytes */ + 200: { + content: never; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or screenshot capture failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Capture a desktop screenshot region. + * @description Performs a health-gated screenshot crop against the managed desktop and + * returns the requested PNG region bytes. + */ + get_v1_desktop_screenshot_region: { + parameters: { + query: { + /** @description Region x coordinate */ + x: number; + /** @description Region y coordinate */ + y: number; + /** @description Region width */ + width: number; + /** @description Region height */ + height: number; + }; + }; + responses: { + /** @description Desktop screenshot region as PNG bytes */ + 200: { + content: never; + }; + /** @description Invalid screenshot region */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or screenshot capture failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Start the private desktop runtime. + * @description Lazily launches the managed Xvfb/openbox stack, validates display health, + * and returns the resulting desktop status snapshot. + */ + post_v1_desktop_start: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopStartRequest"]; + }; + }; + responses: { + /** @description Desktop runtime status after start */ + 200: { + content: { + "application/json": components["schemas"]["DesktopStatusResponse"]; + }; + }; + /** @description Invalid desktop start request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is already transitioning */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop API unsupported on this platform */ + 501: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime could not be started */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Get desktop runtime status. + * @description Returns the current desktop runtime state, dependency status, active + * display metadata, and supervised process information. + */ + get_v1_desktop_status: { + responses: { + /** @description Desktop runtime status */ + 200: { + content: { + "application/json": components["schemas"]["DesktopStatusResponse"]; + }; + }; + /** @description Authentication required */ + 401: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Stop the private desktop runtime. + * @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop + * runtime and returns the resulting status snapshot. + */ + post_v1_desktop_stop: { + responses: { + /** @description Desktop runtime status after stop */ + 200: { + content: { + "application/json": components["schemas"]["DesktopStatusResponse"]; + }; + }; + /** @description Desktop runtime is already transitioning */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; get_v1_fs_entries: { parameters: { query?: { diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index 6b5c9a45..251ddf1e 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -45,6 +45,24 @@ export type { AgentInstallRequest, AgentInstallResponse, AgentListResponse, + DesktopActionResponse, + DesktopDisplayInfoResponse, + DesktopErrorInfo, + DesktopKeyboardPressRequest, + DesktopKeyboardTypeRequest, + DesktopMouseButton, + DesktopMouseClickRequest, + DesktopMouseDragRequest, + DesktopMouseMoveRequest, + DesktopMousePositionResponse, + DesktopMouseScrollRequest, + DesktopProcessInfo, + DesktopRegionScreenshotQuery, + DesktopResolution, + DesktopScreenshotQuery, + DesktopStartRequest, + DesktopState, + DesktopStatusResponse, FsActionResponse, FsDeleteQuery, FsEntriesQuery, diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts index 54efab48..6167e09e 100644 --- a/sdks/typescript/src/types.ts +++ b/sdks/typescript/src/types.ts @@ -9,6 +9,27 @@ import type { components, operations } from "./generated/openapi.ts"; export type ProblemDetails = components["schemas"]["ProblemDetails"]; export type HealthResponse = JsonResponse; +export type DesktopState = components["schemas"]["DesktopState"]; +export type DesktopResolution = components["schemas"]["DesktopResolution"]; +export type DesktopErrorInfo = components["schemas"]["DesktopErrorInfo"]; +export type DesktopProcessInfo = components["schemas"]["DesktopProcessInfo"]; +export type DesktopStatusResponse = JsonResponse; +export type DesktopStartRequest = JsonRequestBody; +export type DesktopScreenshotQuery = + QueryParams extends never + ? Record + : QueryParams; +export type DesktopRegionScreenshotQuery = QueryParams; +export type DesktopMousePositionResponse = JsonResponse; +export type DesktopMouseButton = components["schemas"]["DesktopMouseButton"]; +export type DesktopMouseMoveRequest = JsonRequestBody; +export type DesktopMouseClickRequest = JsonRequestBody; +export type DesktopMouseDragRequest = JsonRequestBody; +export type DesktopMouseScrollRequest = JsonRequestBody; +export type DesktopKeyboardTypeRequest = JsonRequestBody; +export type DesktopKeyboardPressRequest = JsonRequestBody; +export type DesktopActionResponse = JsonResponse; +export type DesktopDisplayInfoResponse = JsonResponse; export type AgentListResponse = JsonResponse; export type AgentInfo = components["schemas"]["AgentInfo"]; export type AgentQuery = QueryParams; diff --git a/sdks/typescript/tests/helpers/docker.ts b/sdks/typescript/tests/helpers/docker.ts new file mode 100644 index 00000000..28c03adf --- /dev/null +++ b/sdks/typescript/tests/helpers/docker.ts @@ -0,0 +1,298 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, "../../../.."); +const CONTAINER_PORT = 3000; +const DEFAULT_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const DEFAULT_IMAGE_TAG = "sandbox-agent-test:dev"; +const STANDARD_PATHS = new Set([ + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin", + "/bin", +]); + +let cachedImage: string | undefined; +let containerCounter = 0; + +export type DockerSandboxAgentHandle = { + baseUrl: string; + token: string; + dispose: () => Promise; +}; + +export type DockerSandboxAgentOptions = { + env?: Record; + pathMode?: "merge" | "replace"; + timeoutMs?: number; +}; + +type TestLayout = { + rootDir: string; + homeDir: string; + xdgDataHome: string; + xdgStateHome: string; + appDataDir: string; + localAppDataDir: string; + installDir: string; +}; + +export function createDockerTestLayout(): TestLayout { + const tempRoot = join(REPO_ROOT, ".context", "docker-test-"); + mkdirSync(resolve(REPO_ROOT, ".context"), { recursive: true }); + const rootDir = mkdtempSync(tempRoot); + const homeDir = join(rootDir, "home"); + const xdgDataHome = join(rootDir, "xdg-data"); + const xdgStateHome = join(rootDir, "xdg-state"); + const appDataDir = join(rootDir, "appdata", "Roaming"); + const localAppDataDir = join(rootDir, "appdata", "Local"); + const installDir = join(xdgDataHome, "sandbox-agent", "bin"); + + for (const dir of [homeDir, xdgDataHome, xdgStateHome, appDataDir, localAppDataDir, installDir]) { + mkdirSync(dir, { recursive: true }); + } + + return { + rootDir, + homeDir, + xdgDataHome, + xdgStateHome, + appDataDir, + localAppDataDir, + installDir, + }; +} + +export function disposeDockerTestLayout(layout: TestLayout): void { + try { + rmSync(layout.rootDir, { recursive: true, force: true }); + } catch (error) { + if ( + typeof process.getuid === "function" && + typeof process.getgid === "function" + ) { + try { + execFileSync( + "docker", + [ + "run", + "--rm", + "--user", + "0:0", + "--entrypoint", + "sh", + "-v", + `${layout.rootDir}:${layout.rootDir}`, + ensureImage(), + "-c", + `chown -R ${process.getuid()}:${process.getgid()} '${layout.rootDir}'`, + ], + { stdio: "pipe" }, + ); + rmSync(layout.rootDir, { recursive: true, force: true }); + return; + } catch {} + } + throw error; + } +} + +export async function startDockerSandboxAgent( + layout: TestLayout, + options: DockerSandboxAgentOptions = {}, +): Promise { + const image = ensureImage(); + const containerId = uniqueContainerId(); + const env = buildEnv(layout, options.env ?? {}, options.pathMode ?? "merge"); + const mounts = buildMounts(layout.rootDir, env); + + const args = [ + "run", + "-d", + "--rm", + "--name", + containerId, + "-p", + `127.0.0.1::${CONTAINER_PORT}`, + ]; + + if (typeof process.getuid === "function" && typeof process.getgid === "function") { + args.push("--user", `${process.getuid()}:${process.getgid()}`); + } + + if (process.platform === "linux") { + args.push("--add-host", "host.docker.internal:host-gateway"); + } + + for (const mount of mounts) { + args.push("-v", `${mount}:${mount}`); + } + + for (const [key, value] of Object.entries(env)) { + args.push("-e", `${key}=${value}`); + } + + args.push( + image, + "server", + "--host", + "0.0.0.0", + "--port", + String(CONTAINER_PORT), + "--no-token", + ); + + execFileSync("docker", args, { stdio: "pipe" }); + + try { + const mapping = execFileSync("docker", ["port", containerId, `${CONTAINER_PORT}/tcp`], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + const mappingParts = mapping.split(":"); + const hostPort = mappingParts[mappingParts.length - 1]?.trim(); + if (!hostPort) { + throw new Error(`missing mapped host port in ${mapping}`); + } + const baseUrl = `http://127.0.0.1:${hostPort}`; + await waitForHealth(baseUrl, options.timeoutMs ?? 30_000); + + return { + baseUrl, + token: "", + dispose: async () => { + try { + execFileSync("docker", ["rm", "-f", containerId], { stdio: "pipe" }); + } catch {} + }, + }; + } catch (error) { + try { + execFileSync("docker", ["rm", "-f", containerId], { stdio: "pipe" }); + } catch {} + throw error; + } +} + +function ensureImage(): string { + if (cachedImage) { + return cachedImage; + } + + cachedImage = process.env.SANDBOX_AGENT_TEST_IMAGE ?? DEFAULT_IMAGE_TAG; + execFileSync( + "docker", + [ + "build", + "--tag", + cachedImage, + "--file", + resolve(REPO_ROOT, "docker/test-agent/Dockerfile"), + REPO_ROOT, + ], + { + cwd: REPO_ROOT, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + return cachedImage; +} + +function buildEnv( + layout: TestLayout, + extraEnv: Record, + pathMode: "merge" | "replace", +): Record { + const env: Record = { + HOME: layout.homeDir, + USERPROFILE: layout.homeDir, + XDG_DATA_HOME: layout.xdgDataHome, + XDG_STATE_HOME: layout.xdgStateHome, + APPDATA: layout.appDataDir, + LOCALAPPDATA: layout.localAppDataDir, + PATH: DEFAULT_PATH, + }; + + const customPathEntries = new Set(); + for (const entry of (extraEnv.PATH ?? "").split(":")) { + if (!entry || entry === DEFAULT_PATH || !entry.startsWith("/")) continue; + if (entry.startsWith(layout.rootDir)) { + customPathEntries.add(entry); + } + } + if (pathMode === "replace") { + env.PATH = extraEnv.PATH ?? ""; + } else if (customPathEntries.size > 0) { + env.PATH = `${Array.from(customPathEntries).join(":")}:${DEFAULT_PATH}`; + } + + for (const [key, value] of Object.entries(extraEnv)) { + if (key === "PATH") { + continue; + } + env[key] = rewriteLocalhostUrl(key, value); + } + + return env; +} + +function buildMounts(rootDir: string, env: Record): string[] { + const mounts = new Set([rootDir]); + + for (const key of [ + "HOME", + "USERPROFILE", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + "APPDATA", + "LOCALAPPDATA", + "SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR", + ]) { + const value = env[key]; + if (value?.startsWith("/")) { + mounts.add(value); + } + } + + for (const entry of (env.PATH ?? "").split(":")) { + if (entry.startsWith("/") && !STANDARD_PATHS.has(entry)) { + mounts.add(entry); + } + } + + return Array.from(mounts); +} + +async function waitForHealth(baseUrl: string, timeoutMs: number): Promise { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + try { + const response = await fetch(`${baseUrl}/v1/health`); + if (response.ok) { + return; + } + } catch {} + await new Promise((resolve) => setTimeout(resolve, 200)); + } + + throw new Error(`timed out waiting for sandbox-agent health at ${baseUrl}`); +} + +function uniqueContainerId(): string { + containerCounter += 1; + return `sandbox-agent-ts-${process.pid}-${Date.now().toString(36)}-${containerCounter.toString(36)}`; +} + +function rewriteLocalhostUrl(key: string, value: string): string { + if (key.endsWith("_URL") || key.endsWith("_URI")) { + return value + .replace("http://127.0.0.1", "http://host.docker.internal") + .replace("http://localhost", "http://host.docker.internal"); + } + return value; +} diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.ts index 1da387b6..c3de7a3e 100644 --- a/sdks/typescript/tests/integration.test.ts +++ b/sdks/typescript/tests/integration.test.ts @@ -1,50 +1,22 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { existsSync } from "node:fs"; -import { mkdtempSync, rmSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { join } from "node:path"; -import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; import { InMemorySessionPersistDriver, SandboxAgent, type SessionEvent, } from "../src/index.ts"; -import { spawnSandboxAgent, isNodeRuntime, type SandboxAgentSpawnHandle } from "../src/spawn.ts"; +import { isNodeRuntime } from "../src/spawn.ts"; +import { + createDockerTestLayout, + disposeDockerTestLayout, + startDockerSandboxAgent, + type DockerSandboxAgentHandle, +} from "./helpers/docker.ts"; import { prepareMockAgentDataHome } from "./helpers/mock-agent.ts"; import WebSocket from "ws"; -const __dirname = dirname(fileURLToPath(import.meta.url)); - -function findBinary(): string | null { - if (process.env.SANDBOX_AGENT_BIN) { - return process.env.SANDBOX_AGENT_BIN; - } - - const cargoPaths = [ - resolve(__dirname, "../../../target/debug/sandbox-agent"), - resolve(__dirname, "../../../target/release/sandbox-agent"), - ]; - - for (const p of cargoPaths) { - if (existsSync(p)) { - return p; - } - } - - return null; -} - -const BINARY_PATH = findBinary(); -if (!BINARY_PATH) { - throw new Error( - "sandbox-agent binary not found. Build it (cargo build -p sandbox-agent) or set SANDBOX_AGENT_BIN.", - ); -} -if (!process.env.SANDBOX_AGENT_BIN) { - process.env.SANDBOX_AGENT_BIN = BINARY_PATH; -} - function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -81,6 +53,15 @@ async function waitForAsync( throw new Error("timed out waiting for condition"); } +async function withTimeout(promise: Promise, label: string, timeoutMs = 15_000): Promise { + return await Promise.race([ + promise, + sleep(timeoutMs).then(() => { + throw new Error(`${label} timed out after ${timeoutMs}ms`); + }), + ]); +} + function buildTarArchive(entries: Array<{ name: string; content: string }>): Uint8Array { const blocks: Buffer[] = []; @@ -145,34 +126,87 @@ function decodeProcessLogData(data: string, encoding: string): string { function nodeCommand(source: string): { command: string; args: string[] } { return { - command: process.execPath, + command: "node", args: ["-e", source], }; } +function forwardRequest( + defaultFetch: typeof fetch, + baseUrl: string, + outgoing: Request, + parsed: URL, +): Promise { + const forwardedInit: RequestInit & { duplex?: "half" } = { + method: outgoing.method, + headers: new Headers(outgoing.headers), + signal: outgoing.signal, + }; + + if (outgoing.method !== "GET" && outgoing.method !== "HEAD") { + forwardedInit.body = outgoing.body; + forwardedInit.duplex = "half"; + } + + const forwardedUrl = new URL(`${parsed.pathname}${parsed.search}`, baseUrl); + return defaultFetch(forwardedUrl, forwardedInit); +} + +async function launchDesktopFocusWindow(sdk: SandboxAgent, display: string): Promise { + const windowProcess = await sdk.createProcess({ + command: "xterm", + args: [ + "-geometry", + "80x24+40+40", + "-title", + "Sandbox Desktop Test", + "-e", + "sh", + "-lc", + "sleep 60", + ], + env: { DISPLAY: display }, + }); + + await waitForAsync(async () => { + const result = await sdk.runProcess({ + command: "sh", + args: [ + "-lc", + "wid=\"$(xdotool search --onlyvisible --name 'Sandbox Desktop Test' 2>/dev/null | head -n 1 || true)\"; if [ -z \"$wid\" ]; then exit 3; fi; xdotool windowactivate \"$wid\"", + ], + env: { DISPLAY: display }, + timeoutMs: 5_000, + }); + + return result.exitCode === 0 ? true : undefined; + }, 10_000, 200); + + return windowProcess.id; +} + describe("Integration: TypeScript SDK flat session API", () => { - let handle: SandboxAgentSpawnHandle; + let handle: DockerSandboxAgentHandle; let baseUrl: string; let token: string; - let dataHome: string; + let layout: ReturnType; - beforeAll(async () => { - dataHome = mkdtempSync(join(tmpdir(), "sdk-integration-")); - const agentEnv = prepareMockAgentDataHome(dataHome); + beforeEach(async () => { + layout = createDockerTestLayout(); + prepareMockAgentDataHome(layout.xdgDataHome); - handle = await spawnSandboxAgent({ - enabled: true, - log: "silent", + handle = await startDockerSandboxAgent(layout, { timeoutMs: 30000, - env: agentEnv, }); baseUrl = handle.baseUrl; token = handle.token; }); - afterAll(async () => { - await handle.dispose(); - rmSync(dataHome, { recursive: true, force: true }); + afterEach(async () => { + await handle?.dispose?.(); + if (layout) { + disposeDockerTestLayout(layout); + } }); it("detects Node.js runtime", () => { @@ -230,11 +264,12 @@ describe("Integration: TypeScript SDK flat session API", () => { token, }); - const directory = mkdtempSync(join(tmpdir(), "sdk-fs-")); + const directory = join(layout.rootDir, "fs-test"); const nestedDir = join(directory, "nested"); const filePath = join(directory, "notes.txt"); const movedPath = join(directory, "notes-moved.txt"); const uploadDir = join(directory, "uploaded"); + mkdirSync(directory, { recursive: true }); try { const listedAgents = await sdk.listAgents({ config: true, noCache: true }); @@ -294,25 +329,33 @@ describe("Integration: TypeScript SDK flat session API", () => { const parsed = new URL(outgoing.url); seenPaths.push(parsed.pathname); - const forwardedUrl = new URL(`${parsed.pathname}${parsed.search}`, baseUrl); - const forwarded = new Request(forwardedUrl.toString(), outgoing); - return defaultFetch(forwarded); + return forwardRequest(defaultFetch, baseUrl, outgoing, parsed); }; const sdk = await SandboxAgent.connect({ token, fetch: customFetch, }); + let sessionId: string | undefined; - await sdk.getHealth(); - const session = await sdk.createSession({ agent: "mock" }); - const prompt = await session.prompt([{ type: "text", text: "custom fetch integration test" }]); - expect(prompt.stopReason).toBe("end_turn"); - - expect(seenPaths).toContain("/v1/health"); - expect(seenPaths.some((path) => path.startsWith("/v1/acp/"))).toBe(true); + try { + await withTimeout(sdk.getHealth(), "custom fetch getHealth"); + const session = await withTimeout( + sdk.createSession({ agent: "mock" }), + "custom fetch createSession", + ); + sessionId = session.id; + expect(session.agent).toBe("mock"); + await withTimeout(sdk.destroySession(session.id), "custom fetch destroySession"); - await sdk.dispose(); + expect(seenPaths).toContain("/v1/health"); + expect(seenPaths.some((path) => path.startsWith("/v1/acp/"))).toBe(true); + } finally { + if (sessionId) { + await sdk.destroySession(sessionId).catch(() => {}); + } + await withTimeout(sdk.dispose(), "custom fetch dispose"); + } }, 60_000); it("requires baseUrl when fetch is not provided", async () => { @@ -341,9 +384,7 @@ describe("Integration: TypeScript SDK flat session API", () => { } } - const forwardedUrl = new URL(`${parsed.pathname}${parsed.search}`, baseUrl); - const forwarded = new Request(forwardedUrl.toString(), outgoing); - return defaultFetch(forwarded); + return forwardRequest(defaultFetch, baseUrl, outgoing, parsed); }; const sdk = await SandboxAgent.connect({ @@ -631,7 +672,9 @@ describe("Integration: TypeScript SDK flat session API", () => { token, }); - const directory = mkdtempSync(join(tmpdir(), "sdk-config-")); + const directory = join(layout.rootDir, "config-test"); + + mkdirSync(directory, { recursive: true }); const mcpConfig = { type: "local" as const, @@ -882,4 +925,98 @@ describe("Integration: TypeScript SDK flat session API", () => { await sdk.dispose(); } }); + + it("covers desktop status, screenshot, display, mouse, and keyboard helpers", async () => { + const sdk = await SandboxAgent.connect({ + baseUrl, + token, + }); + let focusWindowProcessId: string | undefined; + + try { + const initialStatus = await sdk.getDesktopStatus(); + expect(initialStatus.state).toBe("inactive"); + + const started = await sdk.startDesktop({ + width: 1440, + height: 900, + dpi: 96, + }); + expect(started.state).toBe("active"); + expect(started.display?.startsWith(":")).toBe(true); + expect(started.missingDependencies).toEqual([]); + + const displayInfo = await sdk.getDesktopDisplayInfo(); + expect(displayInfo.display).toBe(started.display); + expect(displayInfo.resolution.width).toBe(1440); + expect(displayInfo.resolution.height).toBe(900); + + const screenshot = await sdk.takeDesktopScreenshot(); + expect(Buffer.from(screenshot.subarray(0, 8)).equals(Buffer.from("\x89PNG\r\n\x1a\n", "binary"))).toBe(true); + + const region = await sdk.takeDesktopRegionScreenshot({ + x: 10, + y: 20, + width: 40, + height: 50, + }); + expect(Buffer.from(region.subarray(0, 8)).equals(Buffer.from("\x89PNG\r\n\x1a\n", "binary"))).toBe(true); + + const moved = await sdk.moveDesktopMouse({ x: 40, y: 50 }); + expect(moved.x).toBe(40); + expect(moved.y).toBe(50); + + const dragged = await sdk.dragDesktopMouse({ + startX: 40, + startY: 50, + endX: 80, + endY: 90, + button: "left", + }); + expect(dragged.x).toBe(80); + expect(dragged.y).toBe(90); + + const clicked = await sdk.clickDesktop({ + x: 80, + y: 90, + button: "left", + clickCount: 1, + }); + expect(clicked.x).toBe(80); + expect(clicked.y).toBe(90); + + const scrolled = await sdk.scrollDesktop({ + x: 80, + y: 90, + deltaY: -2, + }); + expect(scrolled.x).toBe(80); + expect(scrolled.y).toBe(90); + + const position = await sdk.getDesktopMousePosition(); + expect(position.x).toBe(80); + expect(position.y).toBe(90); + + focusWindowProcessId = await launchDesktopFocusWindow(sdk, started.display!); + + const typed = await sdk.typeDesktopText({ + text: "hello desktop", + delayMs: 5, + }); + expect(typed.ok).toBe(true); + + const pressed = await sdk.pressDesktopKey({ key: "ctrl+l" }); + expect(pressed.ok).toBe(true); + + const stopped = await sdk.stopDesktop(); + expect(stopped.state).toBe("inactive"); + } finally { + if (focusWindowProcessId) { + await sdk.killProcess(focusWindowProcessId, { waitMs: 5_000 }).catch(() => {}); + await sdk.deleteProcess(focusWindowProcessId).catch(() => {}); + } + await sdk.stopDesktop().catch(() => {}); + await sdk.dispose(); + } + }); }); diff --git a/sdks/typescript/vitest.config.ts b/sdks/typescript/vitest.config.ts index 8676010b..a3ba3f39 100644 --- a/sdks/typescript/vitest.config.ts +++ b/sdks/typescript/vitest.config.ts @@ -4,5 +4,6 @@ export default defineConfig({ test: { include: ["tests/**/*.test.ts"], testTimeout: 30000, + hookTimeout: 120000, }, }); diff --git a/server/packages/sandbox-agent/src/cli.rs b/server/packages/sandbox-agent/src/cli.rs index b1fc6bb5..9831cfa0 100644 --- a/server/packages/sandbox-agent/src/cli.rs +++ b/server/packages/sandbox-agent/src/cli.rs @@ -11,6 +11,7 @@ mod build_version { include!(concat!(env!("OUT_DIR"), "/version.rs")); } +use crate::desktop_install::{install_desktop, DesktopInstallRequest, DesktopPackageManager}; use crate::router::{ build_router_with_state, shutdown_servers, AppState, AuthConfig, BrandingMode, }; @@ -75,6 +76,8 @@ pub enum Command { Server(ServerArgs), /// Call the HTTP API without writing client code. Api(ApiArgs), + /// Install first-party runtime dependencies. + Install(InstallArgs), /// EXPERIMENTAL: OpenCode compatibility layer (disabled until ACP Phase 7). Opencode(OpencodeArgs), /// Manage the sandbox-agent background daemon. @@ -115,6 +118,12 @@ pub struct ApiArgs { command: ApiCommand, } +#[derive(Args, Debug)] +pub struct InstallArgs { + #[command(subcommand)] + command: InstallCommand, +} + #[derive(Args, Debug)] pub struct OpencodeArgs { #[arg(long, short = 'H', default_value = DEFAULT_HOST)] @@ -153,6 +162,12 @@ pub struct DaemonArgs { command: DaemonCommand, } +#[derive(Subcommand, Debug)] +pub enum InstallCommand { + /// Install desktop runtime dependencies. + Desktop(InstallDesktopArgs), +} + #[derive(Subcommand, Debug)] pub enum DaemonCommand { /// Start the daemon in the background. @@ -304,6 +319,18 @@ pub struct InstallAgentArgs { agent_process_version: Option, } +#[derive(Args, Debug)] +pub struct InstallDesktopArgs { + #[arg(long, default_value_t = false)] + yes: bool, + #[arg(long, default_value_t = false)] + print_only: bool, + #[arg(long, value_enum)] + package_manager: Option, + #[arg(long, default_value_t = false)] + no_fonts: bool, +} + #[derive(Args, Debug)] pub struct CredentialsExtractArgs { #[arg(long, short = 'a', value_enum)] @@ -399,6 +426,7 @@ pub fn run_command(command: &Command, cli: &CliConfig) -> Result<(), CliError> { match command { Command::Server(args) => run_server(cli, args), Command::Api(subcommand) => run_api(&subcommand.command, cli), + Command::Install(subcommand) => run_install(&subcommand.command), Command::Opencode(args) => run_opencode(cli, args), Command::Daemon(subcommand) => run_daemon(&subcommand.command, cli), Command::InstallAgent(args) => install_agent_local(args), @@ -406,6 +434,12 @@ pub fn run_command(command: &Command, cli: &CliConfig) -> Result<(), CliError> { } } +fn run_install(command: &InstallCommand) -> Result<(), CliError> { + match command { + InstallCommand::Desktop(args) => install_desktop_local(args), + } +} + fn run_server(cli: &CliConfig, server: &ServerArgs) -> Result<(), CliError> { let auth = if let Some(token) = cli.token.clone() { AuthConfig::with_token(token) @@ -470,6 +504,17 @@ fn run_api(command: &ApiCommand, cli: &CliConfig) -> Result<(), CliError> { } } +fn install_desktop_local(args: &InstallDesktopArgs) -> Result<(), CliError> { + install_desktop(DesktopInstallRequest { + yes: args.yes, + print_only: args.print_only, + package_manager: args.package_manager, + no_fonts: args.no_fonts, + }) + .map(|_| ()) + .map_err(CliError::Server) +} + fn run_agents(command: &AgentsCommand, cli: &CliConfig) -> Result<(), CliError> { match command { AgentsCommand::List(args) => { diff --git a/server/packages/sandbox-agent/src/desktop_errors.rs b/server/packages/sandbox-agent/src/desktop_errors.rs new file mode 100644 index 00000000..67f99b9e --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_errors.rs @@ -0,0 +1,217 @@ +use sandbox_agent_error::ProblemDetails; +use serde_json::{json, Map, Value}; + +use crate::desktop_types::{DesktopErrorInfo, DesktopProcessInfo}; + +#[derive(Debug, Clone)] +pub struct DesktopProblem { + status: u16, + title: &'static str, + code: &'static str, + message: String, + missing_dependencies: Vec, + install_command: Option, + processes: Vec, +} + +impl DesktopProblem { + pub fn unsupported_platform(message: impl Into) -> Self { + Self::new( + 501, + "Desktop Unsupported", + "desktop_unsupported_platform", + message, + ) + } + + pub fn dependencies_missing( + missing_dependencies: Vec, + install_command: Option, + processes: Vec, + ) -> Self { + let mut message = if missing_dependencies.is_empty() { + "Desktop dependencies are not installed".to_string() + } else { + format!( + "Desktop dependencies are not installed: {}", + missing_dependencies.join(", ") + ) + }; + if let Some(command) = install_command.as_ref() { + message.push_str(&format!( + ". Run `{command}` to install them, or install the required tools manually." + )); + } + Self::new( + 503, + "Desktop Dependencies Missing", + "desktop_dependencies_missing", + message, + ) + .with_missing_dependencies(missing_dependencies) + .with_install_command(install_command) + .with_processes(processes) + } + + pub fn runtime_inactive(message: impl Into) -> Self { + Self::new( + 409, + "Desktop Runtime Inactive", + "desktop_runtime_inactive", + message, + ) + } + + pub fn runtime_starting(message: impl Into) -> Self { + Self::new( + 409, + "Desktop Runtime Starting", + "desktop_runtime_starting", + message, + ) + } + + pub fn runtime_failed( + message: impl Into, + install_command: Option, + processes: Vec, + ) -> Self { + Self::new( + 503, + "Desktop Runtime Failed", + "desktop_runtime_failed", + message, + ) + .with_install_command(install_command) + .with_processes(processes) + } + + pub fn invalid_action(message: impl Into) -> Self { + Self::new( + 400, + "Desktop Invalid Action", + "desktop_invalid_action", + message, + ) + } + + pub fn screenshot_failed( + message: impl Into, + processes: Vec, + ) -> Self { + Self::new( + 502, + "Desktop Screenshot Failed", + "desktop_screenshot_failed", + message, + ) + .with_processes(processes) + } + + pub fn input_failed(message: impl Into, processes: Vec) -> Self { + Self::new(502, "Desktop Input Failed", "desktop_input_failed", message) + .with_processes(processes) + } + + pub fn to_problem_details(&self) -> ProblemDetails { + let mut extensions = Map::new(); + extensions.insert("code".to_string(), Value::String(self.code.to_string())); + if !self.missing_dependencies.is_empty() { + extensions.insert( + "missingDependencies".to_string(), + Value::Array( + self.missing_dependencies + .iter() + .cloned() + .map(Value::String) + .collect(), + ), + ); + } + if let Some(install_command) = self.install_command.as_ref() { + extensions.insert( + "installCommand".to_string(), + Value::String(install_command.clone()), + ); + } + if !self.processes.is_empty() { + extensions.insert("processes".to_string(), json!(self.processes)); + } + + ProblemDetails { + type_: format!("urn:sandbox-agent:error:{}", self.code), + title: self.title.to_string(), + status: self.status, + detail: Some(self.message.clone()), + instance: None, + extensions, + } + } + + pub fn to_error_info(&self) -> DesktopErrorInfo { + DesktopErrorInfo { + code: self.code.to_string(), + message: self.message.clone(), + } + } + + pub fn code(&self) -> &'static str { + self.code + } + + fn new( + status: u16, + title: &'static str, + code: &'static str, + message: impl Into, + ) -> Self { + Self { + status, + title, + code, + message: message.into(), + missing_dependencies: Vec::new(), + install_command: None, + processes: Vec::new(), + } + } + + fn with_missing_dependencies(mut self, missing_dependencies: Vec) -> Self { + self.missing_dependencies = missing_dependencies; + self + } + + fn with_install_command(mut self, install_command: Option) -> Self { + self.install_command = install_command; + self + } + + fn with_processes(mut self, processes: Vec) -> Self { + self.processes = processes; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dependencies_missing_detail_includes_install_command() { + let problem = DesktopProblem::dependencies_missing( + vec!["Xvfb".to_string(), "openbox".to_string()], + Some("sandbox-agent install desktop --yes".to_string()), + Vec::new(), + ); + let details = problem.to_problem_details(); + let detail = details.detail.expect("detail"); + assert!(detail.contains("Desktop dependencies are not installed: Xvfb, openbox")); + assert!(detail.contains("sandbox-agent install desktop --yes")); + assert_eq!( + details.extensions.get("installCommand"), + Some(&Value::String( + "sandbox-agent install desktop --yes".to_string() + )) + ); + } +} diff --git a/server/packages/sandbox-agent/src/desktop_install.rs b/server/packages/sandbox-agent/src/desktop_install.rs new file mode 100644 index 00000000..f6bea210 --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_install.rs @@ -0,0 +1,321 @@ +use std::fmt; +use std::io::{self, Write}; +use std::path::PathBuf; +use std::process::Command as ProcessCommand; + +use clap::ValueEnum; + +const AUTOMATIC_INSTALL_SUPPORTED_DISTROS: &str = + "Automatic desktop dependency installation is supported on Debian/Ubuntu (apt), Fedora/RHEL (dnf), and Alpine (apk)."; +const AUTOMATIC_INSTALL_UNSUPPORTED_ENVS: &str = + "Automatic installation is not supported on macOS, Windows, or Linux distributions without apt, dnf, or apk."; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum DesktopPackageManager { + Apt, + Dnf, + Apk, +} + +#[derive(Debug, Clone)] +pub struct DesktopInstallRequest { + pub yes: bool, + pub print_only: bool, + pub package_manager: Option, + pub no_fonts: bool, +} + +pub(crate) fn desktop_platform_support_message() -> String { + format!("Desktop APIs are only supported on Linux. {AUTOMATIC_INSTALL_SUPPORTED_DISTROS}") +} + +fn linux_install_support_message() -> String { + format!("{AUTOMATIC_INSTALL_SUPPORTED_DISTROS} {AUTOMATIC_INSTALL_UNSUPPORTED_ENVS}") +} + +pub fn install_desktop(request: DesktopInstallRequest) -> Result<(), String> { + if std::env::consts::OS != "linux" { + return Err(format!( + "desktop installation is only supported on Linux. {}", + linux_install_support_message() + )); + } + + let package_manager = match request.package_manager { + Some(value) => value, + None => detect_package_manager().ok_or_else(|| { + format!( + "could not detect a supported package manager. {} Install the desktop dependencies manually on this distribution.", + linux_install_support_message() + ) + })?, + }; + + let packages = desktop_packages(package_manager, request.no_fonts); + let used_sudo = !running_as_root() && find_binary("sudo").is_some(); + if !running_as_root() && !used_sudo { + return Err( + "desktop installation requires root or sudo access; rerun as root or install dependencies manually" + .to_string(), + ); + } + + println!("Desktop package manager: {}", package_manager); + println!("Desktop packages:"); + for package in &packages { + println!(" - {package}"); + } + println!("Install command:"); + println!( + " {}", + render_install_command(package_manager, used_sudo, &packages) + ); + + if request.print_only { + return Ok(()); + } + + if !request.yes && !prompt_yes_no("Proceed with desktop dependency installation? [y/N] ")? { + return Err("installation cancelled".to_string()); + } + + run_install_commands(package_manager, used_sudo, &packages)?; + + println!("Desktop dependencies installed."); + Ok(()) +} + +fn detect_package_manager() -> Option { + if find_binary("apt-get").is_some() { + return Some(DesktopPackageManager::Apt); + } + if find_binary("dnf").is_some() { + return Some(DesktopPackageManager::Dnf); + } + if find_binary("apk").is_some() { + return Some(DesktopPackageManager::Apk); + } + None +} + +fn desktop_packages(package_manager: DesktopPackageManager, no_fonts: bool) -> Vec { + let mut packages = match package_manager { + DesktopPackageManager::Apt => vec![ + "xvfb", + "openbox", + "xdotool", + "imagemagick", + "x11-xserver-utils", + "dbus-x11", + "xauth", + "fonts-dejavu-core", + ], + DesktopPackageManager::Dnf => vec![ + "xorg-x11-server-Xvfb", + "openbox", + "xdotool", + "ImageMagick", + "xrandr", + "dbus-x11", + "xauth", + "dejavu-sans-fonts", + ], + DesktopPackageManager::Apk => vec![ + "xvfb", + "openbox", + "xdotool", + "imagemagick", + "xrandr", + "dbus", + "xauth", + "ttf-dejavu", + ], + } + .into_iter() + .map(str::to_string) + .collect::>(); + + if no_fonts { + packages.retain(|package| { + package != "fonts-dejavu-core" + && package != "dejavu-sans-fonts" + && package != "ttf-dejavu" + }); + } + + packages +} + +fn render_install_command( + package_manager: DesktopPackageManager, + used_sudo: bool, + packages: &[String], +) -> String { + let sudo = if used_sudo { "sudo " } else { "" }; + match package_manager { + DesktopPackageManager::Apt => format!( + "{sudo}apt-get update && {sudo}env DEBIAN_FRONTEND=noninteractive apt-get install -y {}", + packages.join(" ") + ), + DesktopPackageManager::Dnf => { + format!("{sudo}dnf install -y {}", packages.join(" ")) + } + DesktopPackageManager::Apk => { + format!("{sudo}apk add --no-cache {}", packages.join(" ")) + } + } +} + +fn run_install_commands( + package_manager: DesktopPackageManager, + used_sudo: bool, + packages: &[String], +) -> Result<(), String> { + match package_manager { + DesktopPackageManager::Apt => { + run_command(command_with_privilege( + used_sudo, + "apt-get", + vec!["update".to_string()], + ))?; + let mut args = vec![ + "DEBIAN_FRONTEND=noninteractive".to_string(), + "apt-get".to_string(), + "install".to_string(), + "-y".to_string(), + ]; + args.extend(packages.iter().cloned()); + run_command(command_with_privilege(used_sudo, "env", args))?; + } + DesktopPackageManager::Dnf => { + let mut args = vec!["install".to_string(), "-y".to_string()]; + args.extend(packages.iter().cloned()); + run_command(command_with_privilege(used_sudo, "dnf", args))?; + } + DesktopPackageManager::Apk => { + let mut args = vec!["add".to_string(), "--no-cache".to_string()]; + args.extend(packages.iter().cloned()); + run_command(command_with_privilege(used_sudo, "apk", args))?; + } + } + Ok(()) +} + +fn command_with_privilege( + used_sudo: bool, + program: &str, + args: Vec, +) -> (String, Vec) { + if used_sudo { + let mut sudo_args = vec![program.to_string()]; + sudo_args.extend(args); + ("sudo".to_string(), sudo_args) + } else { + (program.to_string(), args) + } +} + +fn run_command((program, args): (String, Vec)) -> Result<(), String> { + let status = ProcessCommand::new(&program) + .args(&args) + .status() + .map_err(|err| format!("failed to run `{program}`: {err}"))?; + if !status.success() { + return Err(format!( + "command `{}` exited with status {}", + format_command(&program, &args), + status + )); + } + Ok(()) +} + +fn prompt_yes_no(prompt: &str) -> Result { + print!("{prompt}"); + io::stdout() + .flush() + .map_err(|err| format!("failed to flush prompt: {err}"))?; + let mut input = String::new(); + io::stdin() + .read_line(&mut input) + .map_err(|err| format!("failed to read confirmation: {err}"))?; + let normalized = input.trim().to_ascii_lowercase(); + Ok(matches!(normalized.as_str(), "y" | "yes")) +} + +fn running_as_root() -> bool { + #[cfg(unix)] + unsafe { + return libc::geteuid() == 0; + } + #[cfg(not(unix))] + { + false + } +} + +fn find_binary(name: &str) -> Option { + let path_env = std::env::var_os("PATH")?; + for path in std::env::split_paths(&path_env) { + let candidate = path.join(name); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + +fn format_command(program: &str, args: &[String]) -> String { + let mut parts = vec![program.to_string()]; + parts.extend(args.iter().cloned()); + parts.join(" ") +} + +impl fmt::Display for DesktopPackageManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DesktopPackageManager::Apt => write!(f, "apt"), + DesktopPackageManager::Dnf => write!(f, "dnf"), + DesktopPackageManager::Apk => write!(f, "apk"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn desktop_platform_support_message_mentions_linux_and_supported_distros() { + let message = desktop_platform_support_message(); + assert!(message.contains("only supported on Linux")); + assert!(message.contains("Debian/Ubuntu (apt)")); + assert!(message.contains("Fedora/RHEL (dnf)")); + assert!(message.contains("Alpine (apk)")); + } + + #[test] + fn linux_install_support_message_mentions_unsupported_environments() { + let message = linux_install_support_message(); + assert!(message.contains("Debian/Ubuntu (apt)")); + assert!(message.contains("Fedora/RHEL (dnf)")); + assert!(message.contains("Alpine (apk)")); + assert!(message.contains("macOS")); + assert!(message.contains("Windows")); + assert!(message.contains("without apt, dnf, or apk")); + } + + #[test] + fn desktop_packages_support_no_fonts() { + let packages = desktop_packages(DesktopPackageManager::Apt, true); + assert!(!packages.iter().any(|value| value == "fonts-dejavu-core")); + assert!(packages.iter().any(|value| value == "xvfb")); + } + + #[test] + fn render_install_command_matches_package_manager() { + let packages = vec!["xvfb".to_string(), "openbox".to_string()]; + let command = render_install_command(DesktopPackageManager::Apk, false, &packages); + assert_eq!(command, "apk add --no-cache xvfb openbox"); + } +} diff --git a/server/packages/sandbox-agent/src/desktop_runtime.rs b/server/packages/sandbox-agent/src/desktop_runtime.rs new file mode 100644 index 00000000..38608ddf --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_runtime.rs @@ -0,0 +1,1501 @@ +use std::collections::HashMap; +use std::fs::{self, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::process::{Output, Stdio}; +use std::sync::Arc; +use std::time::Duration; + +use tokio::process::{Child, Command}; +use tokio::sync::Mutex; + +use crate::desktop_errors::DesktopProblem; +use crate::desktop_install::desktop_platform_support_message; +use crate::desktop_types::{ + DesktopActionResponse, DesktopDisplayInfoResponse, DesktopErrorInfo, + DesktopKeyboardPressRequest, DesktopKeyboardTypeRequest, DesktopMouseButton, + DesktopMouseClickRequest, DesktopMouseDragRequest, DesktopMouseMoveRequest, + DesktopMousePositionResponse, DesktopMouseScrollRequest, DesktopProcessInfo, + DesktopRegionScreenshotQuery, DesktopResolution, DesktopStartRequest, DesktopState, + DesktopStatusResponse, +}; + +const DEFAULT_WIDTH: u32 = 1440; +const DEFAULT_HEIGHT: u32 = 900; +const DEFAULT_DPI: u32 = 96; +const DEFAULT_DISPLAY_NUM: i32 = 99; +const MAX_DISPLAY_PROBE: i32 = 10; +const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(10); +const INPUT_TIMEOUT: Duration = Duration::from_secs(5); +const STARTUP_TIMEOUT: Duration = Duration::from_secs(15); +const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n"; + +#[derive(Debug, Clone)] +pub struct DesktopRuntime { + config: DesktopRuntimeConfig, + inner: Arc>, +} + +#[derive(Debug, Clone)] +pub struct DesktopRuntimeConfig { + state_dir: PathBuf, + display_num: i32, + assume_linux_for_tests: bool, +} + +#[derive(Debug)] +struct DesktopRuntimeStateData { + state: DesktopState, + display_num: i32, + display: Option, + resolution: Option, + started_at: Option, + last_error: Option, + missing_dependencies: Vec, + install_command: Option, + runtime_log_path: PathBuf, + environment: HashMap, + xvfb: Option, + openbox: Option, + dbus_pid: Option, +} + +#[derive(Debug)] +struct ManagedDesktopChild { + name: &'static str, + child: Child, + log_path: PathBuf, +} + +#[derive(Debug, Clone)] +struct DesktopReadyContext { + display: String, + environment: HashMap, + resolution: DesktopResolution, +} + +impl Default for DesktopRuntimeConfig { + fn default() -> Self { + let display_num = std::env::var("SANDBOX_AGENT_DESKTOP_DISPLAY_NUM") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_DISPLAY_NUM); + + let state_dir = std::env::var("SANDBOX_AGENT_DESKTOP_STATE_DIR") + .ok() + .map(PathBuf::from) + .unwrap_or_else(default_state_dir); + + let assume_linux_for_tests = std::env::var("SANDBOX_AGENT_DESKTOP_TEST_ASSUME_LINUX") + .ok() + .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) + .unwrap_or(false); + + Self { + state_dir, + display_num, + assume_linux_for_tests, + } + } +} + +impl DesktopRuntime { + pub fn new() -> Self { + Self::with_config(DesktopRuntimeConfig::default()) + } + + pub fn with_config(config: DesktopRuntimeConfig) -> Self { + let runtime_log_path = config.state_dir.join("desktop-runtime.log"); + Self { + inner: Arc::new(Mutex::new(DesktopRuntimeStateData { + state: DesktopState::Inactive, + display_num: config.display_num, + display: None, + resolution: None, + started_at: None, + last_error: None, + missing_dependencies: Vec::new(), + install_command: None, + runtime_log_path, + environment: HashMap::new(), + xvfb: None, + openbox: None, + dbus_pid: None, + })), + config, + } + } + + pub async fn status(&self) -> DesktopStatusResponse { + let mut state = self.inner.lock().await; + self.refresh_status_locked(&mut state).await; + self.snapshot_locked(&state) + } + + pub async fn start( + &self, + request: DesktopStartRequest, + ) -> Result { + let mut state = self.inner.lock().await; + + if !self.platform_supported() { + let problem = DesktopProblem::unsupported_platform(desktop_platform_support_message()); + self.record_problem_locked(&mut state, &problem); + state.state = DesktopState::Failed; + return Err(problem); + } + + if matches!(state.state, DesktopState::Starting | DesktopState::Stopping) { + return Err(DesktopProblem::runtime_starting( + "Desktop runtime is busy transitioning state", + )); + } + + self.refresh_status_locked(&mut state).await; + if state.state == DesktopState::Active { + return Ok(self.snapshot_locked(&state)); + } + + if !state.missing_dependencies.is_empty() { + return Err(DesktopProblem::dependencies_missing( + state.missing_dependencies.clone(), + state.install_command.clone(), + self.processes_locked(&state), + )); + } + + self.ensure_state_dir_locked(&state).map_err(|err| { + DesktopProblem::runtime_failed(err, None, self.processes_locked(&state)) + })?; + self.write_runtime_log_locked(&state, "starting desktop runtime"); + + let width = request.width.unwrap_or(DEFAULT_WIDTH); + let height = request.height.unwrap_or(DEFAULT_HEIGHT); + let dpi = request.dpi.unwrap_or(DEFAULT_DPI); + validate_start_request(width, height, dpi)?; + + let display_num = self.choose_display_num()?; + let display = format!(":{display_num}"); + let resolution = DesktopResolution { + width, + height, + dpi: Some(dpi), + }; + let environment = self.base_environment(&display)?; + + state.state = DesktopState::Starting; + state.display_num = display_num; + state.display = Some(display.clone()); + state.resolution = Some(resolution.clone()); + state.started_at = None; + state.last_error = None; + state.environment = environment; + state.install_command = None; + + if let Err(problem) = self.start_dbus_locked(&mut state).await { + return Err(self.fail_start_locked(&mut state, problem).await); + } + if let Err(problem) = self.start_xvfb_locked(&mut state, &resolution).await { + return Err(self.fail_start_locked(&mut state, problem).await); + } + if let Err(problem) = self.wait_for_socket(display_num).await { + return Err(self.fail_start_locked(&mut state, problem).await); + } + if let Err(problem) = self.start_openbox_locked(&mut state).await { + return Err(self.fail_start_locked(&mut state, problem).await); + } + + let ready = DesktopReadyContext { + display, + environment: state.environment.clone(), + resolution, + }; + + let display_info = match self.query_display_info_locked(&state, &ready).await { + Ok(display_info) => display_info, + Err(problem) => return Err(self.fail_start_locked(&mut state, problem).await), + }; + state.resolution = Some(display_info.resolution.clone()); + + if let Err(problem) = self.capture_screenshot_locked(&state, None).await { + return Err(self.fail_start_locked(&mut state, problem).await); + } + + state.state = DesktopState::Active; + state.started_at = Some(chrono::Utc::now().to_rfc3339()); + state.last_error = None; + self.write_runtime_log_locked( + &state, + &format!( + "desktop runtime active on {} ({}x{}, dpi {})", + display_info.display, + display_info.resolution.width, + display_info.resolution.height, + display_info.resolution.dpi.unwrap_or(DEFAULT_DPI) + ), + ); + + Ok(self.snapshot_locked(&state)) + } + + pub async fn stop(&self) -> Result { + let mut state = self.inner.lock().await; + if matches!(state.state, DesktopState::Starting | DesktopState::Stopping) { + return Err(DesktopProblem::runtime_starting( + "Desktop runtime is busy transitioning state", + )); + } + + state.state = DesktopState::Stopping; + self.write_runtime_log_locked(&state, "stopping desktop runtime"); + + self.stop_openbox_locked(&mut state).await; + self.stop_xvfb_locked(&mut state).await; + self.stop_dbus_locked(&mut state); + + state.state = DesktopState::Inactive; + state.display = None; + state.resolution = None; + state.started_at = None; + state.last_error = None; + state.missing_dependencies = self.detect_missing_dependencies(); + state.install_command = self.install_command_for(&state.missing_dependencies); + state.environment.clear(); + + Ok(self.snapshot_locked(&state)) + } + + pub async fn shutdown(&self) { + let _ = self.stop().await; + } + + pub async fn screenshot(&self) -> Result, DesktopProblem> { + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + self.capture_screenshot_locked(&state, Some(&ready)).await + } + + pub async fn screenshot_region( + &self, + query: DesktopRegionScreenshotQuery, + ) -> Result, DesktopProblem> { + validate_region(&query)?; + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let crop = format!("{}x{}+{}+{}", query.width, query.height, query.x, query.y); + self.capture_screenshot_with_crop_locked(&state, &ready, &crop) + .await + } + + pub async fn mouse_position(&self) -> Result { + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn move_mouse( + &self, + request: DesktopMouseMoveRequest, + ) -> Result { + validate_coordinates(request.x, request.y)?; + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let args = vec![ + "mousemove".to_string(), + request.x.to_string(), + request.y.to_string(), + ]; + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn click_mouse( + &self, + request: DesktopMouseClickRequest, + ) -> Result { + validate_coordinates(request.x, request.y)?; + let click_count = request.click_count.unwrap_or(1); + if click_count == 0 { + return Err(DesktopProblem::invalid_action( + "clickCount must be greater than 0", + )); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let button = mouse_button_code(request.button.unwrap_or(DesktopMouseButton::Left)); + let mut args = vec![ + "mousemove".to_string(), + request.x.to_string(), + request.y.to_string(), + "click".to_string(), + ]; + if click_count > 1 { + args.push("--repeat".to_string()); + args.push(click_count.to_string()); + } + args.push(button.to_string()); + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn drag_mouse( + &self, + request: DesktopMouseDragRequest, + ) -> Result { + validate_coordinates(request.start_x, request.start_y)?; + validate_coordinates(request.end_x, request.end_y)?; + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let button = mouse_button_code(request.button.unwrap_or(DesktopMouseButton::Left)); + let args = vec![ + "mousemove".to_string(), + request.start_x.to_string(), + request.start_y.to_string(), + "mousedown".to_string(), + button.to_string(), + "mousemove".to_string(), + request.end_x.to_string(), + request.end_y.to_string(), + "mouseup".to_string(), + button.to_string(), + ]; + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn scroll_mouse( + &self, + request: DesktopMouseScrollRequest, + ) -> Result { + validate_coordinates(request.x, request.y)?; + let delta_x = request.delta_x.unwrap_or(0); + let delta_y = request.delta_y.unwrap_or(0); + if delta_x == 0 && delta_y == 0 { + return Err(DesktopProblem::invalid_action( + "deltaX or deltaY must be non-zero", + )); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let mut args = vec![ + "mousemove".to_string(), + request.x.to_string(), + request.y.to_string(), + ]; + + append_scroll_clicks(&mut args, delta_y, 5, 4); + append_scroll_clicks(&mut args, delta_x, 7, 6); + + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn type_text( + &self, + request: DesktopKeyboardTypeRequest, + ) -> Result { + if request.text.is_empty() { + return Err(DesktopProblem::invalid_action("text must not be empty")); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let args = type_text_args(request.text, request.delay_ms.unwrap_or(10)); + self.run_input_command_locked(&state, &ready, args).await?; + Ok(DesktopActionResponse { ok: true }) + } + + pub async fn press_key( + &self, + request: DesktopKeyboardPressRequest, + ) -> Result { + if request.key.trim().is_empty() { + return Err(DesktopProblem::invalid_action("key must not be empty")); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let args = press_key_args(request.key); + self.run_input_command_locked(&state, &ready, args).await?; + Ok(DesktopActionResponse { ok: true }) + } + + pub async fn display_info(&self) -> Result { + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + self.query_display_info_locked(&state, &ready).await + } + + async fn ensure_ready_locked( + &self, + state: &mut DesktopRuntimeStateData, + ) -> Result { + self.refresh_status_locked(state).await; + match state.state { + DesktopState::Active => { + let display = state.display.clone().ok_or_else(|| { + DesktopProblem::runtime_failed( + "Desktop runtime has no active display", + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + let resolution = state.resolution.clone().ok_or_else(|| { + DesktopProblem::runtime_failed( + "Desktop runtime has no active resolution", + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + Ok(DesktopReadyContext { + display, + environment: state.environment.clone(), + resolution, + }) + } + DesktopState::InstallRequired => Err(DesktopProblem::dependencies_missing( + state.missing_dependencies.clone(), + state.install_command.clone(), + self.processes_locked(state), + )), + DesktopState::Inactive => Err(DesktopProblem::runtime_inactive( + "Desktop runtime has not been started", + )), + DesktopState::Starting | DesktopState::Stopping => Err( + DesktopProblem::runtime_starting("Desktop runtime is still transitioning"), + ), + DesktopState::Failed => Err(DesktopProblem::runtime_failed( + state + .last_error + .as_ref() + .map(|error| error.message.clone()) + .unwrap_or_else(|| "Desktop runtime is unhealthy".to_string()), + state.install_command.clone(), + self.processes_locked(state), + )), + } + } + + async fn refresh_status_locked(&self, state: &mut DesktopRuntimeStateData) { + let missing_dependencies = if self.platform_supported() { + self.detect_missing_dependencies() + } else { + Vec::new() + }; + state.missing_dependencies = missing_dependencies.clone(); + state.install_command = self.install_command_for(&missing_dependencies); + + if !self.platform_supported() { + state.state = DesktopState::Failed; + state.last_error = Some( + DesktopProblem::unsupported_platform(desktop_platform_support_message()) + .to_error_info(), + ); + return; + } + + if !missing_dependencies.is_empty() { + state.state = DesktopState::InstallRequired; + state.last_error = Some( + DesktopProblem::dependencies_missing( + missing_dependencies, + state.install_command.clone(), + self.processes_locked(state), + ) + .to_error_info(), + ); + return; + } + + if matches!( + state.state, + DesktopState::Inactive | DesktopState::Starting | DesktopState::Stopping + ) { + if state.state == DesktopState::Inactive { + state.last_error = None; + } + return; + } + + if state.state == DesktopState::Failed + && state.display.is_none() + && state.xvfb.is_none() + && state.openbox.is_none() + && state.dbus_pid.is_none() + { + return; + } + + let Some(display) = state.display.clone() else { + state.state = DesktopState::Failed; + state.last_error = Some( + DesktopProblem::runtime_failed( + "Desktop runtime has no display", + None, + self.processes_locked(state), + ) + .to_error_info(), + ); + return; + }; + + if let Err(problem) = self.ensure_process_running_locked(state, "Xvfb").await { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + if let Err(problem) = self.ensure_process_running_locked(state, "openbox").await { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + + if !socket_path(state.display_num).exists() { + let problem = DesktopProblem::runtime_failed( + format!("X socket for display {display} is missing"), + state.install_command.clone(), + self.processes_locked(state), + ); + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + + let ready = DesktopReadyContext { + display, + environment: state.environment.clone(), + resolution: state.resolution.clone().unwrap_or(DesktopResolution { + width: DEFAULT_WIDTH, + height: DEFAULT_HEIGHT, + dpi: Some(DEFAULT_DPI), + }), + }; + + match self.query_display_info_locked(state, &ready).await { + Ok(display_info) => { + state.resolution = Some(display_info.resolution); + } + Err(problem) => { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + } + + if let Err(problem) = self.capture_screenshot_locked(state, Some(&ready)).await { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + + state.state = DesktopState::Active; + state.last_error = None; + } + + async fn ensure_process_running_locked( + &self, + state: &mut DesktopRuntimeStateData, + name: &str, + ) -> Result<(), DesktopProblem> { + let process = match name { + "Xvfb" => state.xvfb.as_mut(), + "openbox" => state.openbox.as_mut(), + _ => None, + }; + + let Some(process) = process else { + return Err(DesktopProblem::runtime_failed( + format!("{name} is not running"), + state.install_command.clone(), + self.processes_locked(state), + )); + }; + + match process.child.try_wait() { + Ok(None) => Ok(()), + Ok(Some(status)) => Err(DesktopProblem::runtime_failed( + format!("{name} exited with status {status}"), + state.install_command.clone(), + self.processes_locked(state), + )), + Err(err) => Err(DesktopProblem::runtime_failed( + format!("failed to inspect {name}: {err}"), + state.install_command.clone(), + self.processes_locked(state), + )), + } + } + + async fn start_dbus_locked( + &self, + state: &mut DesktopRuntimeStateData, + ) -> Result<(), DesktopProblem> { + if find_binary("dbus-launch").is_none() { + self.write_runtime_log_locked( + state, + "dbus-launch not found; continuing without D-Bus session", + ); + return Ok(()); + } + + let output = run_command_output("dbus-launch", &[], &state.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to launch dbus-launch: {err}"), + None, + self.processes_locked(state), + ) + })?; + + if !output.status.success() { + self.write_runtime_log_locked( + state, + &format!( + "dbus-launch failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + ); + return Ok(()); + } + + for line in String::from_utf8_lossy(&output.stdout).lines() { + if let Some((key, value)) = line.split_once('=') { + let cleaned = value.trim().trim_end_matches(';').to_string(); + if key == "DBUS_SESSION_BUS_ADDRESS" { + state.environment.insert(key.to_string(), cleaned); + } else if key == "DBUS_SESSION_BUS_PID" { + state.dbus_pid = cleaned.parse::().ok(); + } + } + } + + Ok(()) + } + + async fn start_xvfb_locked( + &self, + state: &mut DesktopRuntimeStateData, + resolution: &DesktopResolution, + ) -> Result<(), DesktopProblem> { + let Some(display) = state.display.clone() else { + return Err(DesktopProblem::runtime_failed( + "Desktop display was not configured before starting Xvfb", + None, + self.processes_locked(state), + )); + }; + let args = vec![ + display, + "-screen".to_string(), + "0".to_string(), + format!("{}x{}x24", resolution.width, resolution.height), + "-dpi".to_string(), + resolution.dpi.unwrap_or(DEFAULT_DPI).to_string(), + "-nolisten".to_string(), + "tcp".to_string(), + ]; + let log_path = self.config.state_dir.join("desktop-xvfb.log"); + let child = + self.spawn_logged_process("Xvfb", "Xvfb", &args, &state.environment, &log_path)?; + state.xvfb = Some(child); + Ok(()) + } + + async fn start_openbox_locked( + &self, + state: &mut DesktopRuntimeStateData, + ) -> Result<(), DesktopProblem> { + let log_path = self.config.state_dir.join("desktop-openbox.log"); + let child = + self.spawn_logged_process("openbox", "openbox", &[], &state.environment, &log_path)?; + state.openbox = Some(child); + Ok(()) + } + + async fn stop_xvfb_locked(&self, state: &mut DesktopRuntimeStateData) { + if let Some(mut child) = state.xvfb.take() { + self.write_runtime_log_locked(state, "stopping Xvfb"); + let _ = terminate_child(&mut child.child).await; + } + } + + async fn stop_openbox_locked(&self, state: &mut DesktopRuntimeStateData) { + if let Some(mut child) = state.openbox.take() { + self.write_runtime_log_locked(state, "stopping openbox"); + let _ = terminate_child(&mut child.child).await; + } + } + + fn stop_dbus_locked(&self, state: &mut DesktopRuntimeStateData) { + if let Some(pid) = state.dbus_pid.take() { + #[cfg(unix)] + unsafe { + libc::kill(pid as i32, libc::SIGTERM); + } + } + } + + async fn fail_start_locked( + &self, + state: &mut DesktopRuntimeStateData, + problem: DesktopProblem, + ) -> DesktopProblem { + self.record_problem_locked(state, &problem); + self.write_runtime_log_locked(state, "desktop runtime startup failed; cleaning up"); + self.stop_openbox_locked(state).await; + self.stop_xvfb_locked(state).await; + self.stop_dbus_locked(state); + state.state = DesktopState::Failed; + state.display = None; + state.resolution = None; + state.started_at = None; + state.environment.clear(); + problem + } + + async fn capture_screenshot_locked( + &self, + state: &DesktopRuntimeStateData, + ready: Option<&DesktopReadyContext>, + ) -> Result, DesktopProblem> { + match ready { + Some(ready) => { + self.capture_screenshot_with_crop_locked(state, ready, "") + .await + } + None => { + let ready = DesktopReadyContext { + display: state + .display + .clone() + .unwrap_or_else(|| format!(":{}", state.display_num)), + environment: state.environment.clone(), + resolution: state.resolution.clone().unwrap_or(DesktopResolution { + width: DEFAULT_WIDTH, + height: DEFAULT_HEIGHT, + dpi: Some(DEFAULT_DPI), + }), + }; + self.capture_screenshot_with_crop_locked(state, &ready, "") + .await + } + } + } + + async fn capture_screenshot_with_crop_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + crop: &str, + ) -> Result, DesktopProblem> { + let mut args = vec!["-window".to_string(), "root".to_string()]; + if !crop.is_empty() { + args.push("-crop".to_string()); + args.push(crop.to_string()); + } + args.push("png:-".to_string()); + + let output = run_command_output("import", &args, &ready.environment, SCREENSHOT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::screenshot_failed( + format!("failed to capture desktop screenshot: {err}"), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::screenshot_failed( + format!( + "desktop screenshot command failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + self.processes_locked(state), + )); + } + validate_png(&output.stdout).map_err(|message| { + DesktopProblem::screenshot_failed(message, self.processes_locked(state)) + })?; + Ok(output.stdout) + } + + async fn mouse_position_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + ) -> Result { + let args = vec!["getmouselocation".to_string(), "--shell".to_string()]; + let output = run_command_output("xdotool", &args, &ready.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::input_failed( + format!("failed to query mouse position: {err}"), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::input_failed( + format!( + "mouse position command failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + self.processes_locked(state), + )); + } + parse_mouse_position(&output.stdout) + .map_err(|message| DesktopProblem::input_failed(message, self.processes_locked(state))) + } + + async fn run_input_command_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + args: Vec, + ) -> Result<(), DesktopProblem> { + let output = run_command_output("xdotool", &args, &ready.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::input_failed( + format!("failed to execute desktop input command: {err}"), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::input_failed( + format!( + "desktop input command failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + self.processes_locked(state), + )); + } + Ok(()) + } + + async fn query_display_info_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + ) -> Result { + let args = vec!["--current".to_string()]; + let output = run_command_output("xrandr", &args, &ready.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to query display info: {err}"), + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::runtime_failed( + format!( + "display query failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + state.install_command.clone(), + self.processes_locked(state), + )); + } + let resolution = parse_xrandr_resolution(&output.stdout).map_err(|message| { + DesktopProblem::runtime_failed( + message, + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + Ok(DesktopDisplayInfoResponse { + display: ready.display.clone(), + resolution: DesktopResolution { + dpi: ready.resolution.dpi, + ..resolution + }, + }) + } + + fn detect_missing_dependencies(&self) -> Vec { + let mut missing = Vec::new(); + for (name, binary) in [ + ("Xvfb", "Xvfb"), + ("openbox", "openbox"), + ("xdotool", "xdotool"), + ("import", "import"), + ("xrandr", "xrandr"), + ] { + if find_binary(binary).is_none() { + missing.push(name.to_string()); + } + } + missing + } + + fn install_command_for(&self, missing_dependencies: &[String]) -> Option { + if !self.platform_supported() || missing_dependencies.is_empty() { + None + } else { + Some("sandbox-agent install desktop --yes".to_string()) + } + } + + fn platform_supported(&self) -> bool { + cfg!(target_os = "linux") || self.config.assume_linux_for_tests + } + + fn choose_display_num(&self) -> Result { + for offset in 0..MAX_DISPLAY_PROBE { + let candidate = self.config.display_num + offset; + if !socket_path(candidate).exists() { + return Ok(candidate); + } + } + Err(DesktopProblem::runtime_failed( + "unable to find an available X display starting at :99", + None, + Vec::new(), + )) + } + + fn base_environment(&self, display: &str) -> Result, DesktopProblem> { + let mut environment = HashMap::new(); + environment.insert("DISPLAY".to_string(), display.to_string()); + environment.insert( + "HOME".to_string(), + self.config + .state_dir + .join("home") + .to_string_lossy() + .to_string(), + ); + environment.insert( + "USER".to_string(), + std::env::var("USER").unwrap_or_else(|_| "sandbox-agent".to_string()), + ); + environment.insert( + "PATH".to_string(), + std::env::var("PATH").unwrap_or_default(), + ); + fs::create_dir_all(self.config.state_dir.join("home")).map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to create desktop home: {err}"), + None, + Vec::new(), + ) + })?; + Ok(environment) + } + + fn spawn_logged_process( + &self, + name: &'static str, + command: &str, + args: &[String], + environment: &HashMap, + log_path: &Path, + ) -> Result { + if let Some(parent) = log_path.parent() { + fs::create_dir_all(parent).map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to create desktop log directory: {err}"), + None, + Vec::new(), + ) + })?; + } + + let stdout = OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .map_err(|err| { + DesktopProblem::runtime_failed( + format!( + "failed to open desktop log file {}: {err}", + log_path.display() + ), + None, + Vec::new(), + ) + })?; + let stderr = stdout.try_clone().map_err(|err| { + DesktopProblem::runtime_failed( + format!( + "failed to clone desktop log file {}: {err}", + log_path.display() + ), + None, + Vec::new(), + ) + })?; + + let mut child = Command::new(command); + child.args(args); + child.envs(environment); + child.stdin(Stdio::null()); + child.stdout(Stdio::from(stdout)); + child.stderr(Stdio::from(stderr)); + + let child = child.spawn().map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to spawn {name}: {err}"), + None, + Vec::new(), + ) + })?; + + Ok(ManagedDesktopChild { + name, + child, + log_path: log_path.to_path_buf(), + }) + } + + async fn wait_for_socket(&self, display_num: i32) -> Result<(), DesktopProblem> { + let socket = socket_path(display_num); + let parent = socket + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from("/tmp/.X11-unix")); + let _ = fs::create_dir_all(parent); + + let start = tokio::time::Instant::now(); + while start.elapsed() < STARTUP_TIMEOUT { + if socket.exists() { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + Err(DesktopProblem::runtime_failed( + format!("timed out waiting for X socket {}", socket.display()), + None, + Vec::new(), + )) + } + + fn snapshot_locked(&self, state: &DesktopRuntimeStateData) -> DesktopStatusResponse { + DesktopStatusResponse { + state: state.state, + display: state.display.clone(), + resolution: state.resolution.clone(), + started_at: state.started_at.clone(), + last_error: state.last_error.clone(), + missing_dependencies: state.missing_dependencies.clone(), + install_command: state.install_command.clone(), + processes: self.processes_locked(state), + runtime_log_path: Some(state.runtime_log_path.to_string_lossy().to_string()), + } + } + + fn processes_locked(&self, state: &DesktopRuntimeStateData) -> Vec { + let mut processes = Vec::new(); + if let Some(child) = state.xvfb.as_ref() { + processes.push(DesktopProcessInfo { + name: child.name.to_string(), + pid: child.child.id(), + running: child_is_running(&child.child), + log_path: Some(child.log_path.to_string_lossy().to_string()), + }); + } + if let Some(child) = state.openbox.as_ref() { + processes.push(DesktopProcessInfo { + name: child.name.to_string(), + pid: child.child.id(), + running: child_is_running(&child.child), + log_path: Some(child.log_path.to_string_lossy().to_string()), + }); + } + if let Some(pid) = state.dbus_pid { + processes.push(DesktopProcessInfo { + name: "dbus".to_string(), + pid: Some(pid), + running: process_exists(pid), + log_path: None, + }); + } + processes + } + + fn record_problem_locked(&self, state: &mut DesktopRuntimeStateData, problem: &DesktopProblem) { + state.last_error = Some(problem.to_error_info()); + self.write_runtime_log_locked( + state, + &format!("{}: {}", problem.code(), problem.to_error_info().message), + ); + } + + fn ensure_state_dir_locked(&self, state: &DesktopRuntimeStateData) -> Result<(), String> { + fs::create_dir_all(&self.config.state_dir).map_err(|err| { + format!( + "failed to create desktop state dir {}: {err}", + self.config.state_dir.display() + ) + })?; + if let Some(parent) = state.runtime_log_path.parent() { + fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create runtime log dir {}: {err}", + parent.display() + ) + })?; + } + Ok(()) + } + + fn write_runtime_log_locked(&self, state: &DesktopRuntimeStateData, message: &str) { + if let Some(parent) = state.runtime_log_path.parent() { + let _ = fs::create_dir_all(parent); + } + let line = format!("{} {}\n", chrono::Utc::now().to_rfc3339(), message); + let _ = OpenOptions::new() + .create(true) + .append(true) + .open(&state.runtime_log_path) + .and_then(|mut file| std::io::Write::write_all(&mut file, line.as_bytes())); + } +} + +fn default_state_dir() -> PathBuf { + if let Ok(value) = std::env::var("XDG_STATE_HOME") { + return PathBuf::from(value).join("sandbox-agent").join("desktop"); + } + if let Some(home) = dirs::home_dir() { + return home + .join(".local") + .join("state") + .join("sandbox-agent") + .join("desktop"); + } + std::env::temp_dir().join("sandbox-agent-desktop") +} + +fn socket_path(display_num: i32) -> PathBuf { + PathBuf::from(format!("/tmp/.X11-unix/X{display_num}")) +} + +fn find_binary(name: &str) -> Option { + let path_env = std::env::var_os("PATH")?; + for path in std::env::split_paths(&path_env) { + let candidate = path.join(name); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + +async fn run_command_output( + command: &str, + args: &[String], + environment: &HashMap, + timeout: Duration, +) -> Result { + use tokio::io::AsyncReadExt; + + let mut child = Command::new(command); + child.args(args); + child.envs(environment); + child.stdin(Stdio::null()); + child.stdout(Stdio::piped()); + child.stderr(Stdio::piped()); + + let mut child = child.spawn().map_err(|err| err.to_string())?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "failed to capture child stdout".to_string())?; + let stderr = child + .stderr + .take() + .ok_or_else(|| "failed to capture child stderr".to_string())?; + + let stdout_task = tokio::spawn(async move { + let mut stdout = stdout; + let mut bytes = Vec::new(); + stdout.read_to_end(&mut bytes).await.map(|_| bytes) + }); + let stderr_task = tokio::spawn(async move { + let mut stderr = stderr; + let mut bytes = Vec::new(); + stderr.read_to_end(&mut bytes).await.map(|_| bytes) + }); + + let status = match tokio::time::timeout(timeout, child.wait()).await { + Ok(result) => result.map_err(|err| err.to_string())?, + Err(_) => { + terminate_child(&mut child).await?; + let _ = stdout_task.await; + let _ = stderr_task.await; + return Err(format!("command timed out after {}s", timeout.as_secs())); + } + }; + + let stdout = stdout_task + .await + .map_err(|err| err.to_string())? + .map_err(|err| err.to_string())?; + let stderr = stderr_task + .await + .map_err(|err| err.to_string())? + .map_err(|err| err.to_string())?; + + Ok(Output { + status, + stdout, + stderr, + }) +} + +async fn terminate_child(child: &mut Child) -> Result<(), String> { + if let Ok(Some(_)) = child.try_wait() { + return Ok(()); + } + child.start_kill().map_err(|err| err.to_string())?; + let _ = tokio::time::timeout(Duration::from_secs(5), child.wait()).await; + Ok(()) +} + +fn child_is_running(child: &Child) -> bool { + child.id().is_some() +} + +fn process_exists(pid: u32) -> bool { + #[cfg(unix)] + unsafe { + return libc::kill(pid as i32, 0) == 0 + || std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH); + } + #[cfg(not(unix))] + { + let _ = pid; + false + } +} + +fn validate_png(bytes: &[u8]) -> Result<(), String> { + if bytes.len() < PNG_SIGNATURE.len() || &bytes[..PNG_SIGNATURE.len()] != PNG_SIGNATURE { + return Err("desktop screenshot did not return PNG bytes".to_string()); + } + Ok(()) +} + +fn parse_xrandr_resolution(bytes: &[u8]) -> Result { + let text = String::from_utf8_lossy(bytes); + for line in text.lines() { + if let Some(index) = line.find(" current ") { + let tail = &line[index + " current ".len()..]; + let mut parts = tail.split(','); + if let Some(current) = parts.next() { + let dims: Vec<&str> = current.split_whitespace().collect(); + if dims.len() >= 3 { + let width = dims[0] + .parse::() + .map_err(|_| "failed to parse xrandr width".to_string())?; + let height = dims[2] + .parse::() + .map_err(|_| "failed to parse xrandr height".to_string())?; + return Ok(DesktopResolution { + width, + height, + dpi: None, + }); + } + } + } + } + Err("unable to parse xrandr current resolution".to_string()) +} + +fn parse_mouse_position(bytes: &[u8]) -> Result { + let text = String::from_utf8_lossy(bytes); + let mut x = None; + let mut y = None; + let mut screen = None; + let mut window = None; + for line in text.lines() { + if let Some((key, value)) = line.split_once('=') { + match key { + "X" => x = value.parse::().ok(), + "Y" => y = value.parse::().ok(), + "SCREEN" => screen = value.parse::().ok(), + "WINDOW" => window = Some(value.to_string()), + _ => {} + } + } + } + match (x, y) { + (Some(x), Some(y)) => Ok(DesktopMousePositionResponse { + x, + y, + screen, + window, + }), + _ => Err("unable to parse xdotool mouse position".to_string()), + } +} + +fn type_text_args(text: String, delay_ms: u32) -> Vec { + vec![ + "type".to_string(), + "--delay".to_string(), + delay_ms.to_string(), + "--".to_string(), + text, + ] +} + +fn press_key_args(key: String) -> Vec { + vec!["key".to_string(), "--".to_string(), key] +} + +fn validate_start_request(width: u32, height: u32, dpi: u32) -> Result<(), DesktopProblem> { + if width == 0 || height == 0 { + return Err(DesktopProblem::invalid_action( + "Desktop width and height must be greater than 0", + )); + } + if dpi == 0 { + return Err(DesktopProblem::invalid_action( + "Desktop dpi must be greater than 0", + )); + } + Ok(()) +} + +fn validate_region(query: &DesktopRegionScreenshotQuery) -> Result<(), DesktopProblem> { + validate_coordinates(query.x, query.y)?; + if query.width == 0 || query.height == 0 { + return Err(DesktopProblem::invalid_action( + "Screenshot region width and height must be greater than 0", + )); + } + Ok(()) +} + +fn validate_coordinates(x: i32, y: i32) -> Result<(), DesktopProblem> { + if x < 0 || y < 0 { + return Err(DesktopProblem::invalid_action( + "Desktop coordinates must be non-negative", + )); + } + Ok(()) +} + +fn mouse_button_code(button: DesktopMouseButton) -> u8 { + match button { + DesktopMouseButton::Left => 1, + DesktopMouseButton::Middle => 2, + DesktopMouseButton::Right => 3, + } +} + +fn append_scroll_clicks( + args: &mut Vec, + delta: i32, + positive_button: u8, + negative_button: u8, +) { + if delta == 0 { + return; + } + let button = if delta > 0 { + positive_button + } else { + negative_button + }; + let repeat = delta.unsigned_abs(); + args.push("click".to_string()); + if repeat > 1 { + args.push("--repeat".to_string()); + args.push(repeat.to_string()); + } + args.push(button.to_string()); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_xrandr_resolution_reads_current_geometry() { + let bytes = b"Screen 0: minimum 1 x 1, current 1440 x 900, maximum 32767 x 32767\n"; + let parsed = parse_xrandr_resolution(bytes).expect("parse resolution"); + assert_eq!(parsed.width, 1440); + assert_eq!(parsed.height, 900); + } + + #[test] + fn parse_mouse_position_reads_shell_output() { + let bytes = b"X=123\nY=456\nSCREEN=0\nWINDOW=0\n"; + let parsed = parse_mouse_position(bytes).expect("parse mouse position"); + assert_eq!(parsed.x, 123); + assert_eq!(parsed.y, 456); + assert_eq!(parsed.screen, Some(0)); + assert_eq!(parsed.window.as_deref(), Some("0")); + } + + #[test] + fn png_validation_rejects_non_png_bytes() { + let error = validate_png(b"not png").expect_err("validation should fail"); + assert!(error.contains("PNG")); + } + + #[test] + fn type_text_args_insert_double_dash_before_user_text() { + let args = type_text_args("--help".to_string(), 5); + assert_eq!(args, vec!["type", "--delay", "5", "--", "--help"]); + } + + #[test] + fn press_key_args_insert_double_dash_before_user_key() { + let args = press_key_args("--help".to_string()); + assert_eq!(args, vec!["key", "--", "--help"]); + } + + #[test] + fn append_scroll_clicks_uses_positive_direction_buttons() { + let mut args = Vec::new(); + append_scroll_clicks(&mut args, 2, 5, 4); + append_scroll_clicks(&mut args, -3, 7, 6); + assert_eq!( + args, + vec!["click", "--repeat", "2", "5", "click", "--repeat", "3", "6"] + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn run_command_output_kills_child_on_timeout() { + let pid_file = std::env::temp_dir().join(format!( + "sandbox-agent-desktop-runtime-timeout-{}.pid", + std::process::id() + )); + let _ = std::fs::remove_file(&pid_file); + let command = format!("echo $$ > {}; exec sleep 30", pid_file.display()); + let args = vec!["-c".to_string(), command]; + + let error = run_command_output("sh", &args, &HashMap::new(), Duration::from_millis(200)) + .await + .expect_err("command should time out"); + assert!(error.contains("timed out")); + + let pid = std::fs::read_to_string(&pid_file) + .expect("pid file should exist") + .trim() + .parse::() + .expect("pid should parse"); + + for _ in 0..20 { + if !process_exists(pid) { + let _ = std::fs::remove_file(&pid_file); + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + let _ = std::fs::remove_file(&pid_file); + panic!("timed out child process {pid} still exists after timeout cleanup"); + } +} diff --git a/server/packages/sandbox-agent/src/desktop_types.rs b/server/packages/sandbox-agent/src/desktop_types.rs new file mode 100644 index 00000000..c6a7b7e0 --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_types.rs @@ -0,0 +1,173 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DesktopState { + Inactive, + InstallRequired, + Starting, + Active, + Stopping, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopResolution { + pub width: u32, + pub height: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dpi: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopErrorInfo { + pub code: String, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopProcessInfo { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pid: Option, + pub running: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub log_path: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopStatusResponse { + pub state: DesktopState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, + #[serde(default)] + pub missing_dependencies: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub install_command: Option, + #[serde(default)] + pub processes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_log_path: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct DesktopStartRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub width: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub height: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dpi: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct DesktopScreenshotQuery {} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopRegionScreenshotQuery { + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMousePositionResponse { + pub x: i32, + pub y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub screen: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum DesktopMouseButton { + Left, + Middle, + Right, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseMoveRequest { + pub x: i32, + pub y: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseClickRequest { + pub x: i32, + pub y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub click_count: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseDragRequest { + pub start_x: i32, + pub start_y: i32, + pub end_x: i32, + pub end_y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseScrollRequest { + pub x: i32, + pub y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delta_x: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delta_y: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopKeyboardTypeRequest { + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delay_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopKeyboardPressRequest { + pub key: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopActionResponse { + pub ok: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopDisplayInfoResponse { + pub display: String, + pub resolution: DesktopResolution, +} diff --git a/server/packages/sandbox-agent/src/lib.rs b/server/packages/sandbox-agent/src/lib.rs index e84b10b2..4ec26238 100644 --- a/server/packages/sandbox-agent/src/lib.rs +++ b/server/packages/sandbox-agent/src/lib.rs @@ -3,6 +3,10 @@ mod acp_proxy_runtime; pub mod cli; pub mod daemon; +mod desktop_errors; +mod desktop_install; +mod desktop_runtime; +pub mod desktop_types; mod process_runtime; pub mod router; pub mod server_logs; diff --git a/server/packages/sandbox-agent/src/router.rs b/server/packages/sandbox-agent/src/router.rs index 110c3258..9c1a6909 100644 --- a/server/packages/sandbox-agent/src/router.rs +++ b/server/packages/sandbox-agent/src/router.rs @@ -37,6 +37,9 @@ use tracing::Span; use utoipa::{Modify, OpenApi, ToSchema}; use crate::acp_proxy_runtime::{AcpProxyRuntime, ProxyPostOutcome}; +use crate::desktop_errors::DesktopProblem; +use crate::desktop_runtime::DesktopRuntime; +use crate::desktop_types::*; use crate::process_runtime::{ decode_input_bytes, ProcessLogFilter, ProcessLogFilterStream, ProcessRuntime, ProcessRuntimeConfig, ProcessSnapshot, ProcessStartSpec, ProcessStatus, ProcessStream, RunSpec, @@ -87,6 +90,7 @@ pub struct AppState { acp_proxy: Arc, opencode_server_manager: Arc, process_runtime: Arc, + desktop_runtime: Arc, pub(crate) branding: BrandingMode, version_cache: Mutex>, } @@ -111,12 +115,14 @@ impl AppState { }, )); let process_runtime = Arc::new(ProcessRuntime::new()); + let desktop_runtime = Arc::new(DesktopRuntime::new()); Self { auth, agent_manager, acp_proxy, opencode_server_manager, process_runtime, + desktop_runtime, branding, version_cache: Mutex::new(HashMap::new()), } @@ -138,6 +144,10 @@ impl AppState { self.process_runtime.clone() } + pub(crate) fn desktop_runtime(&self) -> Arc { + self.desktop_runtime.clone() + } + pub(crate) fn purge_version_cache(&self, agent: AgentId) { self.version_cache.lock().unwrap().remove(&agent); } @@ -172,6 +182,31 @@ pub fn build_router(state: AppState) -> Router { pub fn build_router_with_state(shared: Arc) -> (Router, Arc) { let mut v1_router = Router::new() .route("/health", get(get_v1_health)) + .route("/desktop/status", get(get_v1_desktop_status)) + .route("/desktop/start", post(post_v1_desktop_start)) + .route("/desktop/stop", post(post_v1_desktop_stop)) + .route("/desktop/screenshot", get(get_v1_desktop_screenshot)) + .route( + "/desktop/screenshot/region", + get(get_v1_desktop_screenshot_region), + ) + .route( + "/desktop/mouse/position", + get(get_v1_desktop_mouse_position), + ) + .route("/desktop/mouse/move", post(post_v1_desktop_mouse_move)) + .route("/desktop/mouse/click", post(post_v1_desktop_mouse_click)) + .route("/desktop/mouse/drag", post(post_v1_desktop_mouse_drag)) + .route("/desktop/mouse/scroll", post(post_v1_desktop_mouse_scroll)) + .route( + "/desktop/keyboard/type", + post(post_v1_desktop_keyboard_type), + ) + .route( + "/desktop/keyboard/press", + post(post_v1_desktop_keyboard_press), + ) + .route("/desktop/display/info", get(get_v1_desktop_display_info)) .route("/agents", get(get_v1_agents)) .route("/agents/:agent", get(get_v1_agent)) .route("/agents/:agent/install", post(post_v1_agent_install)) @@ -316,12 +351,26 @@ async fn opencode_unavailable() -> Response { pub async fn shutdown_servers(state: &Arc) { state.acp_proxy().shutdown_all().await; state.opencode_server_manager().shutdown().await; + state.desktop_runtime().shutdown().await; } #[derive(OpenApi)] #[openapi( paths( get_v1_health, + get_v1_desktop_status, + post_v1_desktop_start, + post_v1_desktop_stop, + get_v1_desktop_screenshot, + get_v1_desktop_screenshot_region, + get_v1_desktop_mouse_position, + post_v1_desktop_mouse_move, + post_v1_desktop_mouse_click, + post_v1_desktop_mouse_drag, + post_v1_desktop_mouse_scroll, + post_v1_desktop_keyboard_type, + post_v1_desktop_keyboard_press, + get_v1_desktop_display_info, get_v1_agents, get_v1_agent, post_v1_agent_install, @@ -360,6 +409,24 @@ pub async fn shutdown_servers(state: &Arc) { components( schemas( HealthResponse, + DesktopState, + DesktopResolution, + DesktopErrorInfo, + DesktopProcessInfo, + DesktopStatusResponse, + DesktopStartRequest, + DesktopScreenshotQuery, + DesktopRegionScreenshotQuery, + DesktopMousePositionResponse, + DesktopMouseButton, + DesktopMouseMoveRequest, + DesktopMouseClickRequest, + DesktopMouseDragRequest, + DesktopMouseScrollRequest, + DesktopKeyboardTypeRequest, + DesktopKeyboardPressRequest, + DesktopActionResponse, + DesktopDisplayInfoResponse, ServerStatus, ServerStatusInfo, AgentCapabilities, @@ -438,6 +505,12 @@ impl From for ApiError { } } +impl From for ApiError { + fn from(value: DesktopProblem) -> Self { + Self::Problem(value.to_problem_details()) + } +} + impl IntoResponse for ApiError { fn into_response(self) -> Response { let problem = match &self { @@ -476,6 +549,305 @@ async fn get_v1_health() -> Json { }) } +/// Get desktop runtime status. +/// +/// Returns the current desktop runtime state, dependency status, active +/// display metadata, and supervised process information. +#[utoipa::path( + get, + path = "/v1/desktop/status", + tag = "v1", + responses( + (status = 200, description = "Desktop runtime status", body = DesktopStatusResponse), + (status = 401, description = "Authentication required", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_status( + State(state): State>, +) -> Result, ApiError> { + Ok(Json(state.desktop_runtime().status().await)) +} + +/// Start the private desktop runtime. +/// +/// Lazily launches the managed Xvfb/openbox stack, validates display health, +/// and returns the resulting desktop status snapshot. +#[utoipa::path( + post, + path = "/v1/desktop/start", + tag = "v1", + request_body = DesktopStartRequest, + responses( + (status = 200, description = "Desktop runtime status after start", body = DesktopStatusResponse), + (status = 400, description = "Invalid desktop start request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is already transitioning", body = ProblemDetails), + (status = 501, description = "Desktop API unsupported on this platform", body = ProblemDetails), + (status = 503, description = "Desktop runtime could not be started", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_start( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let status = state.desktop_runtime().start(body).await?; + Ok(Json(status)) +} + +/// Stop the private desktop runtime. +/// +/// Terminates the managed openbox/Xvfb/dbus processes owned by the desktop +/// runtime and returns the resulting status snapshot. +#[utoipa::path( + post, + path = "/v1/desktop/stop", + tag = "v1", + responses( + (status = 200, description = "Desktop runtime status after stop", body = DesktopStatusResponse), + (status = 409, description = "Desktop runtime is already transitioning", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_stop( + State(state): State>, +) -> Result, ApiError> { + let status = state.desktop_runtime().stop().await?; + Ok(Json(status)) +} + +/// Capture a full desktop screenshot. +/// +/// Performs a health-gated full-frame screenshot of the managed desktop and +/// returns PNG bytes. +#[utoipa::path( + get, + path = "/v1/desktop/screenshot", + tag = "v1", + responses( + (status = 200, description = "Desktop screenshot as PNG bytes"), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or screenshot capture failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_screenshot( + State(state): State>, +) -> Result { + let bytes = state.desktop_runtime().screenshot().await?; + Ok(([(header::CONTENT_TYPE, "image/png")], Bytes::from(bytes)).into_response()) +} + +/// Capture a desktop screenshot region. +/// +/// Performs a health-gated screenshot crop against the managed desktop and +/// returns the requested PNG region bytes. +#[utoipa::path( + get, + path = "/v1/desktop/screenshot/region", + tag = "v1", + params( + ("x" = i32, Query, description = "Region x coordinate"), + ("y" = i32, Query, description = "Region y coordinate"), + ("width" = u32, Query, description = "Region width"), + ("height" = u32, Query, description = "Region height") + ), + responses( + (status = 200, description = "Desktop screenshot region as PNG bytes"), + (status = 400, description = "Invalid screenshot region", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or screenshot capture failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_screenshot_region( + State(state): State>, + Query(query): Query, +) -> Result { + let bytes = state.desktop_runtime().screenshot_region(query).await?; + Ok(([(header::CONTENT_TYPE, "image/png")], Bytes::from(bytes)).into_response()) +} + +/// Get the current desktop mouse position. +/// +/// Performs a health-gated mouse position query against the managed desktop. +#[utoipa::path( + get, + path = "/v1/desktop/mouse/position", + tag = "v1", + responses( + (status = 200, description = "Desktop mouse position", body = DesktopMousePositionResponse), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input check failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_mouse_position( + State(state): State>, +) -> Result, ApiError> { + let position = state.desktop_runtime().mouse_position().await?; + Ok(Json(position)) +} + +/// Move the desktop mouse. +/// +/// Performs a health-gated absolute pointer move on the managed desktop and +/// returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/move", + tag = "v1", + request_body = DesktopMouseMoveRequest, + responses( + (status = 200, description = "Desktop mouse position after move", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse move request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_move( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().move_mouse(body).await?; + Ok(Json(position)) +} + +/// Click on the desktop. +/// +/// Performs a health-gated pointer move and click against the managed desktop +/// and returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/click", + tag = "v1", + request_body = DesktopMouseClickRequest, + responses( + (status = 200, description = "Desktop mouse position after click", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse click request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_click( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().click_mouse(body).await?; + Ok(Json(position)) +} + +/// Drag the desktop mouse. +/// +/// Performs a health-gated drag gesture against the managed desktop and +/// returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/drag", + tag = "v1", + request_body = DesktopMouseDragRequest, + responses( + (status = 200, description = "Desktop mouse position after drag", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse drag request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_drag( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().drag_mouse(body).await?; + Ok(Json(position)) +} + +/// Scroll the desktop mouse wheel. +/// +/// Performs a health-gated scroll gesture at the requested coordinates and +/// returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/scroll", + tag = "v1", + request_body = DesktopMouseScrollRequest, + responses( + (status = 200, description = "Desktop mouse position after scroll", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse scroll request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_scroll( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().scroll_mouse(body).await?; + Ok(Json(position)) +} + +/// Type desktop keyboard text. +/// +/// Performs a health-gated `xdotool type` operation against the managed +/// desktop. +#[utoipa::path( + post, + path = "/v1/desktop/keyboard/type", + tag = "v1", + request_body = DesktopKeyboardTypeRequest, + responses( + (status = 200, description = "Desktop keyboard action result", body = DesktopActionResponse), + (status = 400, description = "Invalid keyboard type request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_keyboard_type( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let response = state.desktop_runtime().type_text(body).await?; + Ok(Json(response)) +} + +/// Press a desktop keyboard shortcut. +/// +/// Performs a health-gated `xdotool key` operation against the managed +/// desktop. +#[utoipa::path( + post, + path = "/v1/desktop/keyboard/press", + tag = "v1", + request_body = DesktopKeyboardPressRequest, + responses( + (status = 200, description = "Desktop keyboard action result", body = DesktopActionResponse), + (status = 400, description = "Invalid keyboard press request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 502, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_keyboard_press( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let response = state.desktop_runtime().press_key(body).await?; + Ok(Json(response)) +} + +/// Get desktop display information. +/// +/// Performs a health-gated display query against the managed desktop and +/// returns the current display identifier and resolution. +#[utoipa::path( + get, + path = "/v1/desktop/display/info", + tag = "v1", + responses( + (status = 200, description = "Desktop display information", body = DesktopDisplayInfoResponse), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or display query failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_display_info( + State(state): State>, +) -> Result, ApiError> { + let info = state.desktop_runtime().display_info().await?; + Ok(Json(info)) +} + #[utoipa::path( get, path = "/v1/agents", diff --git a/server/packages/sandbox-agent/tests/support/docker.rs b/server/packages/sandbox-agent/tests/support/docker.rs new file mode 100644 index 00000000..9305d95c --- /dev/null +++ b/server/packages/sandbox-agent/tests/support/docker.rs @@ -0,0 +1,593 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::OnceLock; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use sandbox_agent::router::AuthConfig; +use serial_test::serial; +use tempfile::TempDir; + +const CONTAINER_PORT: u16 = 3000; +const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const DEFAULT_IMAGE_TAG: &str = "sandbox-agent-test:dev"; +const STANDARD_PATHS: &[&str] = &[ + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin", + "/bin", +]; + +static IMAGE_TAG: OnceLock = OnceLock::new(); +static DOCKER_BIN: OnceLock = OnceLock::new(); +static CONTAINER_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone)] +pub struct DockerApp { + base_url: String, +} + +impl DockerApp { + pub fn http_url(&self, path: &str) -> String { + format!("{}{}", self.base_url, path) + } + + pub fn ws_url(&self, path: &str) -> String { + let suffix = self + .base_url + .strip_prefix("http://") + .unwrap_or(&self.base_url); + format!("ws://{suffix}{path}") + } +} + +pub struct TestApp { + pub app: DockerApp, + install_dir: PathBuf, + _root: TempDir, + container_id: String, +} + +#[derive(Default)] +pub struct TestAppOptions { + pub env: BTreeMap, + pub extra_paths: Vec, + pub replace_path: bool, +} + +impl TestApp { + pub fn new(auth: AuthConfig) -> Self { + Self::with_setup(auth, |_| {}) + } + + pub fn with_setup(auth: AuthConfig, setup: F) -> Self + where + F: FnOnce(&Path), + { + Self::with_options(auth, TestAppOptions::default(), setup) + } + + pub fn with_options(auth: AuthConfig, options: TestAppOptions, setup: F) -> Self + where + F: FnOnce(&Path), + { + let root = tempfile::tempdir().expect("create docker test root"); + let layout = TestLayout::new(root.path()); + layout.create(); + setup(&layout.install_dir); + + let container_id = unique_container_id(); + let image = ensure_test_image(); + let env = build_env(&layout, &auth, &options); + let mounts = build_mounts(root.path(), &env); + let base_url = run_container(&container_id, &image, &mounts, &env, &auth); + + Self { + app: DockerApp { base_url }, + install_dir: layout.install_dir, + _root: root, + container_id, + } + } + + pub fn install_path(&self) -> &Path { + &self.install_dir + } + + pub fn root_path(&self) -> &Path { + self._root.path() + } +} + +impl Drop for TestApp { + fn drop(&mut self) { + let _ = Command::new(docker_bin()) + .args(["rm", "-f", &self.container_id]) + .output(); + } +} + +pub struct LiveServer { + base_url: String, +} + +impl LiveServer { + pub async fn spawn(app: DockerApp) -> Self { + Self { + base_url: app.base_url, + } + } + + pub fn http_url(&self, path: &str) -> String { + format!("{}{}", self.base_url, path) + } + + pub fn ws_url(&self, path: &str) -> String { + let suffix = self + .base_url + .strip_prefix("http://") + .unwrap_or(&self.base_url); + format!("ws://{suffix}{path}") + } + + pub async fn shutdown(self) {} +} + +struct TestLayout { + home: PathBuf, + xdg_data_home: PathBuf, + xdg_state_home: PathBuf, + appdata: PathBuf, + local_appdata: PathBuf, + install_dir: PathBuf, +} + +impl TestLayout { + fn new(root: &Path) -> Self { + let home = root.join("home"); + let xdg_data_home = root.join("xdg-data"); + let xdg_state_home = root.join("xdg-state"); + let appdata = root.join("appdata").join("Roaming"); + let local_appdata = root.join("appdata").join("Local"); + let install_dir = xdg_data_home.join("sandbox-agent").join("bin"); + Self { + home, + xdg_data_home, + xdg_state_home, + appdata, + local_appdata, + install_dir, + } + } + + fn create(&self) { + for dir in [ + &self.home, + &self.xdg_data_home, + &self.xdg_state_home, + &self.appdata, + &self.local_appdata, + &self.install_dir, + ] { + fs::create_dir_all(dir).expect("create docker test dir"); + } + } +} + +fn ensure_test_image() -> String { + IMAGE_TAG + .get_or_init(|| { + let repo_root = repo_root(); + let image_tag = std::env::var("SANDBOX_AGENT_TEST_IMAGE") + .unwrap_or_else(|_| DEFAULT_IMAGE_TAG.to_string()); + let output = Command::new(docker_bin()) + .args(["build", "--tag", &image_tag, "--file"]) + .arg( + repo_root + .join("docker") + .join("test-agent") + .join("Dockerfile"), + ) + .arg(&repo_root) + .output() + .expect("build sandbox-agent test image"); + if !output.status.success() { + panic!( + "failed to build sandbox-agent test image: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + image_tag + }) + .clone() +} + +fn build_env( + layout: &TestLayout, + auth: &AuthConfig, + options: &TestAppOptions, +) -> BTreeMap { + let mut env = BTreeMap::new(); + env.insert( + "HOME".to_string(), + layout.home.to_string_lossy().to_string(), + ); + env.insert( + "USERPROFILE".to_string(), + layout.home.to_string_lossy().to_string(), + ); + env.insert( + "XDG_DATA_HOME".to_string(), + layout.xdg_data_home.to_string_lossy().to_string(), + ); + env.insert( + "XDG_STATE_HOME".to_string(), + layout.xdg_state_home.to_string_lossy().to_string(), + ); + env.insert( + "APPDATA".to_string(), + layout.appdata.to_string_lossy().to_string(), + ); + env.insert( + "LOCALAPPDATA".to_string(), + layout.local_appdata.to_string_lossy().to_string(), + ); + + for (key, value) in std::env::vars() { + if key == "PATH" { + continue; + } + if key == "XDG_STATE_HOME" || key == "HOME" || key == "USERPROFILE" { + continue; + } + if key.starts_with("SANDBOX_AGENT_") || key.starts_with("OPENCODE_COMPAT_") { + env.insert(key.clone(), rewrite_localhost_url(&key, &value)); + } + } + + if let Some(token) = auth.token.as_ref() { + env.insert("SANDBOX_AGENT_TEST_AUTH_TOKEN".to_string(), token.clone()); + } + + if options.replace_path { + env.insert( + "PATH".to_string(), + options.env.get("PATH").cloned().unwrap_or_default(), + ); + } else { + let mut custom_path_entries = + custom_path_entries(layout.install_dir.parent().expect("install base")); + custom_path_entries.extend(explicit_path_entries()); + custom_path_entries.extend( + options + .extra_paths + .iter() + .filter(|path| path.is_absolute() && path.exists()) + .cloned(), + ); + custom_path_entries.sort(); + custom_path_entries.dedup(); + + if custom_path_entries.is_empty() { + env.insert("PATH".to_string(), DEFAULT_PATH.to_string()); + } else { + let joined = custom_path_entries + .iter() + .map(|path| path.to_string_lossy().to_string()) + .collect::>() + .join(":"); + env.insert("PATH".to_string(), format!("{joined}:{DEFAULT_PATH}")); + } + } + + for (key, value) in &options.env { + if key == "PATH" { + continue; + } + env.insert(key.clone(), rewrite_localhost_url(key, value)); + } + + env +} + +fn build_mounts(root: &Path, env: &BTreeMap) -> Vec { + let mut mounts = BTreeSet::new(); + mounts.insert(root.to_path_buf()); + + for key in [ + "HOME", + "USERPROFILE", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + "APPDATA", + "LOCALAPPDATA", + "SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR", + ] { + if let Some(value) = env.get(key) { + let path = PathBuf::from(value); + if path.is_absolute() { + mounts.insert(path); + } + } + } + + if let Some(path_value) = env.get("PATH") { + for entry in path_value.split(':') { + if entry.is_empty() || STANDARD_PATHS.contains(&entry) { + continue; + } + let path = PathBuf::from(entry); + if path.is_absolute() && path.exists() { + mounts.insert(path); + } + } + } + + mounts.into_iter().collect() +} + +fn run_container( + container_id: &str, + image: &str, + mounts: &[PathBuf], + env: &BTreeMap, + auth: &AuthConfig, +) -> String { + let mut args = vec![ + "run".to_string(), + "-d".to_string(), + "--rm".to_string(), + "--name".to_string(), + container_id.to_string(), + "-p".to_string(), + format!("127.0.0.1::{CONTAINER_PORT}"), + ]; + + #[cfg(unix)] + { + args.push("--user".to_string()); + args.push(format!("{}:{}", unsafe { libc::geteuid() }, unsafe { + libc::getegid() + })); + } + + if cfg!(target_os = "linux") { + args.push("--add-host".to_string()); + args.push("host.docker.internal:host-gateway".to_string()); + } + + for mount in mounts { + args.push("-v".to_string()); + args.push(format!("{}:{}", mount.display(), mount.display())); + } + + for (key, value) in env { + args.push("-e".to_string()); + args.push(format!("{key}={value}")); + } + + args.push(image.to_string()); + args.push("server".to_string()); + args.push("--host".to_string()); + args.push("0.0.0.0".to_string()); + args.push("--port".to_string()); + args.push(CONTAINER_PORT.to_string()); + match auth.token.as_ref() { + Some(token) => { + args.push("--token".to_string()); + args.push(token.clone()); + } + None => args.push("--no-token".to_string()), + } + + let output = Command::new(docker_bin()) + .args(&args) + .output() + .expect("start docker test container"); + if !output.status.success() { + panic!( + "failed to start docker test container: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let port_output = Command::new(docker_bin()) + .args(["port", container_id, &format!("{CONTAINER_PORT}/tcp")]) + .output() + .expect("resolve mapped docker port"); + if !port_output.status.success() { + panic!( + "failed to resolve docker test port: {}", + String::from_utf8_lossy(&port_output.stderr) + ); + } + + let mapping = String::from_utf8(port_output.stdout) + .expect("docker port utf8") + .trim() + .to_string(); + let host_port = mapping.rsplit(':').next().expect("mapped host port").trim(); + let base_url = format!("http://127.0.0.1:{host_port}"); + wait_for_health(&base_url, auth.token.as_deref()); + base_url +} + +fn wait_for_health(base_url: &str, token: Option<&str>) { + let started = SystemTime::now(); + loop { + if probe_health(base_url, token) { + return; + } + + if started + .elapsed() + .unwrap_or_else(|_| Duration::from_secs(0)) + .gt(&Duration::from_secs(30)) + { + panic!("timed out waiting for sandbox-agent docker test server"); + } + thread::sleep(Duration::from_millis(200)); + } +} + +fn probe_health(base_url: &str, token: Option<&str>) -> bool { + let address = base_url.strip_prefix("http://").unwrap_or(base_url); + let mut stream = match TcpStream::connect(address) { + Ok(stream) => stream, + Err(_) => return false, + }; + let _ = stream.set_read_timeout(Some(Duration::from_secs(2))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(2))); + + let mut request = + format!("GET /v1/health HTTP/1.1\r\nHost: {address}\r\nConnection: close\r\n"); + if let Some(token) = token { + request.push_str(&format!("Authorization: Bearer {token}\r\n")); + } + request.push_str("\r\n"); + + if stream.write_all(request.as_bytes()).is_err() { + return false; + } + + let mut response = String::new(); + if stream.read_to_string(&mut response).is_err() { + return false; + } + + response.starts_with("HTTP/1.1 200") || response.starts_with("HTTP/1.0 200") +} + +fn custom_path_entries(root: &Path) -> Vec { + let mut entries = Vec::new(); + if let Some(value) = std::env::var_os("PATH") { + for entry in std::env::split_paths(&value) { + if !entry.exists() { + continue; + } + if entry.starts_with(root) || entry.starts_with(std::env::temp_dir()) { + entries.push(entry); + } + } + } + entries.sort(); + entries.dedup(); + entries +} + +fn explicit_path_entries() -> Vec { + let mut entries = Vec::new(); + if let Some(value) = std::env::var_os("SANDBOX_AGENT_TEST_EXTRA_PATHS") { + for entry in std::env::split_paths(&value) { + if entry.is_absolute() && entry.exists() { + entries.push(entry); + } + } + } + entries +} + +fn rewrite_localhost_url(key: &str, value: &str) -> String { + if key.ends_with("_URL") || key.ends_with("_URI") { + return value + .replace("http://127.0.0.1", "http://host.docker.internal") + .replace("http://localhost", "http://host.docker.internal"); + } + value.to_string() +} + +fn unique_container_id() -> String { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|value| value.as_millis()) + .unwrap_or(0); + let counter = CONTAINER_COUNTER.fetch_add(1, Ordering::Relaxed); + format!( + "sandbox-agent-test-{}-{millis}-{counter}", + std::process::id() + ) +} + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../..") + .canonicalize() + .expect("repo root") +} + +fn docker_bin() -> &'static Path { + DOCKER_BIN + .get_or_init(|| { + if let Some(value) = std::env::var_os("SANDBOX_AGENT_TEST_DOCKER_BIN") { + let path = PathBuf::from(value); + if path.exists() { + return path; + } + } + + for candidate in [ + "/usr/local/bin/docker", + "/opt/homebrew/bin/docker", + "/usr/bin/docker", + ] { + let path = PathBuf::from(candidate); + if path.exists() { + return path; + } + } + + PathBuf::from("docker") + }) + .as_path() +} + +#[cfg(test)] +mod tests { + use super::*; + + struct EnvVarGuard { + key: &'static str, + old: Option, + } + + impl EnvVarGuard { + fn set(key: &'static str, value: &Path) -> Self { + let old = std::env::var_os(key); + std::env::set_var(key, value); + Self { key, old } + } + } + + impl Drop for EnvVarGuard { + fn drop(&mut self) { + match self.old.as_ref() { + Some(value) => std::env::set_var(self.key, value), + None => std::env::remove_var(self.key), + } + } + } + + #[test] + #[serial] + fn build_env_keeps_test_local_xdg_state_home() { + let root = tempfile::tempdir().expect("create docker support tempdir"); + let host_state = tempfile::tempdir().expect("create host xdg state tempdir"); + let _guard = EnvVarGuard::set("XDG_STATE_HOME", host_state.path()); + + let layout = TestLayout::new(root.path()); + layout.create(); + + let env = build_env(&layout, &AuthConfig::disabled(), &TestAppOptions::default()); + assert_eq!( + env.get("XDG_STATE_HOME"), + Some(&layout.xdg_state_home.to_string_lossy().to_string()) + ); + } +} diff --git a/server/packages/sandbox-agent/tests/v1_agent_process_matrix.rs b/server/packages/sandbox-agent/tests/v1_agent_process_matrix.rs index 029ca25e..fc88c4c4 100644 --- a/server/packages/sandbox-agent/tests/v1_agent_process_matrix.rs +++ b/server/packages/sandbox-agent/tests/v1_agent_process_matrix.rs @@ -1,37 +1,14 @@ use std::fs; use std::path::Path; -use axum::body::Body; -use axum::http::{Method, Request, StatusCode}; use futures::StreamExt; -use http_body_util::BodyExt; -use sandbox_agent::router::{build_router, AppState, AuthConfig}; -use sandbox_agent_agent_management::agents::AgentManager; +use reqwest::{Method, StatusCode}; +use sandbox_agent::router::AuthConfig; use serde_json::{json, Value}; -use tempfile::TempDir; -use tower::util::ServiceExt; -struct TestApp { - app: axum::Router, - _install_dir: TempDir, -} - -impl TestApp { - fn with_setup(setup: F) -> Self - where - F: FnOnce(&Path), - { - let install_dir = tempfile::tempdir().expect("create temp install dir"); - setup(install_dir.path()); - let manager = AgentManager::new(install_dir.path()).expect("create agent manager"); - let state = AppState::new(AuthConfig::disabled(), manager); - let app = build_router(state); - Self { - app, - _install_dir: install_dir, - } - } -} +#[path = "support/docker.rs"] +mod docker_support; +use docker_support::TestApp; fn write_executable(path: &Path, script: &str) { fs::write(path, script).expect("write executable"); @@ -101,28 +78,29 @@ fn setup_stub_agent_process_only(install_dir: &Path, agent: &str) { } async fn send_request( - app: &axum::Router, + app: &docker_support::DockerApp, method: Method, uri: &str, body: Option, ) -> (StatusCode, Vec) { - let mut builder = Request::builder().method(method).uri(uri); - let request_body = if let Some(body) = body { - builder = builder.header("content-type", "application/json"); - Body::from(body.to_string()) + let client = reqwest::Client::new(); + let response = if let Some(body) = body { + client + .request(method, app.http_url(uri)) + .header("content-type", "application/json") + .body(body.to_string()) + .send() + .await + .expect("request handled") } else { - Body::empty() + client + .request(method, app.http_url(uri)) + .send() + .await + .expect("request handled") }; - - let request = builder.body(request_body).expect("build request"); - let response = app.clone().oneshot(request).await.expect("request handled"); let status = response.status(); - let bytes = response - .into_body() - .collect() - .await - .expect("collect body") - .to_bytes(); + let bytes = response.bytes().await.expect("collect body"); (status, bytes.to_vec()) } @@ -145,7 +123,7 @@ async fn agent_process_matrix_smoke_and_jsonrpc_conformance() { .chain(agent_process_only_agents.iter()) .copied() .collect(); - let test_app = TestApp::with_setup(|install_dir| { + let test_app = TestApp::with_setup(AuthConfig::disabled(), |install_dir| { for agent in native_agents { setup_stub_artifacts(install_dir, agent); } @@ -201,21 +179,15 @@ async fn agent_process_matrix_smoke_and_jsonrpc_conformance() { assert_eq!(new_json["id"], 2, "{agent}: session/new id"); assert_eq!(new_json["result"]["echoedMethod"], "session/new"); - let request = Request::builder() - .method(Method::GET) - .uri(format!("/v1/acp/{agent}-server")) - .body(Body::empty()) - .expect("build sse request"); - - let response = test_app - .app - .clone() - .oneshot(request) + let response = reqwest::Client::new() + .get(test_app.app.http_url(&format!("/v1/acp/{agent}-server"))) + .header("accept", "text/event-stream") + .send() .await .expect("sse response"); assert_eq!(response.status(), StatusCode::OK); - let mut stream = response.into_body().into_data_stream(); + let mut stream = response.bytes_stream(); let chunk = tokio::time::timeout(std::time::Duration::from_secs(5), async move { while let Some(item) = stream.next().await { let bytes = item.expect("sse chunk"); diff --git a/server/packages/sandbox-agent/tests/v1_api.rs b/server/packages/sandbox-agent/tests/v1_api.rs index fa572e6b..02558a71 100644 --- a/server/packages/sandbox-agent/tests/v1_api.rs +++ b/server/packages/sandbox-agent/tests/v1_api.rs @@ -1,128 +1,19 @@ use std::fs; use std::io::{Read, Write}; -use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::net::{TcpListener, TcpStream}; use std::path::Path; use std::time::Duration; -use axum::body::Body; -use axum::http::{header, HeaderMap, Method, Request, StatusCode}; -use axum::Router; use futures::StreamExt; -use http_body_util::BodyExt; -use sandbox_agent::router::{build_router, AppState, AuthConfig}; -use sandbox_agent_agent_management::agents::AgentManager; +use reqwest::header::{self, HeaderMap, HeaderName, HeaderValue}; +use reqwest::{Method, StatusCode}; +use sandbox_agent::router::AuthConfig; use serde_json::{json, Value}; use serial_test::serial; -use tempfile::TempDir; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; -use tower::util::ServiceExt; -struct TestApp { - app: Router, - install_dir: TempDir, -} - -impl TestApp { - fn new(auth: AuthConfig) -> Self { - Self::with_setup(auth, |_| {}) - } - - fn with_setup(auth: AuthConfig, setup: F) -> Self - where - F: FnOnce(&Path), - { - let install_dir = tempfile::tempdir().expect("create temp install dir"); - setup(install_dir.path()); - let manager = AgentManager::new(install_dir.path()).expect("create agent manager"); - let state = AppState::new(auth, manager); - let app = build_router(state); - Self { app, install_dir } - } - - fn install_path(&self) -> &Path { - self.install_dir.path() - } -} - -struct EnvVarGuard { - key: &'static str, - previous: Option, -} - -struct LiveServer { - address: SocketAddr, - shutdown_tx: Option>, - task: JoinHandle<()>, -} - -impl LiveServer { - async fn spawn(app: Router) -> Self { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind live server"); - let address = listener.local_addr().expect("live server address"); - let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - - let task = tokio::spawn(async move { - let server = - axum::serve(listener, app.into_make_service()).with_graceful_shutdown(async { - let _ = shutdown_rx.await; - }); - - let _ = server.await; - }); - - Self { - address, - shutdown_tx: Some(shutdown_tx), - task, - } - } - - fn http_url(&self, path: &str) -> String { - format!("http://{}{}", self.address, path) - } - - fn ws_url(&self, path: &str) -> String { - format!("ws://{}{}", self.address, path) - } - - async fn shutdown(mut self) { - if let Some(shutdown_tx) = self.shutdown_tx.take() { - let _ = shutdown_tx.send(()); - } - - let _ = tokio::time::timeout(Duration::from_secs(3), async { - let _ = self.task.await; - }) - .await; - } -} - -impl EnvVarGuard { - fn set(key: &'static str, value: &str) -> Self { - let previous = std::env::var_os(key); - std::env::set_var(key, value); - Self { key, previous } - } - - fn set_os(key: &'static str, value: &std::ffi::OsStr) -> Self { - let previous = std::env::var_os(key); - std::env::set_var(key, value); - Self { key, previous } - } -} - -impl Drop for EnvVarGuard { - fn drop(&mut self) { - if let Some(previous) = self.previous.as_ref() { - std::env::set_var(self.key, previous); - } else { - std::env::remove_var(self.key); - } - } -} +#[path = "support/docker.rs"] +mod docker_support; +use docker_support::{LiveServer, TestApp}; fn write_executable(path: &Path, script: &str) { fs::write(path, script).expect("write executable"); @@ -168,17 +59,18 @@ exit 0 } fn serve_registry_once(document: Value) -> String { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind registry server"); - let address = listener.local_addr().expect("registry address"); + let listener = TcpListener::bind("0.0.0.0:0").expect("bind registry server"); + let port = listener.local_addr().expect("registry address").port(); let body = document.to_string(); - std::thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - respond_json(&mut stream, &body); + std::thread::spawn(move || loop { + match listener.accept() { + Ok((mut stream, _)) => respond_json(&mut stream, &body), + Err(_) => break, } }); - format!("http://{address}/registry.json") + format!("http://127.0.0.1:{port}/registry.json") } fn respond_json(stream: &mut TcpStream, body: &str) { @@ -196,74 +88,96 @@ fn respond_json(stream: &mut TcpStream, body: &str) { } async fn send_request( - app: &Router, + app: &docker_support::DockerApp, method: Method, uri: &str, body: Option, headers: &[(&str, &str)], ) -> (StatusCode, HeaderMap, Vec) { - let mut builder = Request::builder().method(method).uri(uri); + let client = reqwest::Client::new(); + let mut builder = client.request(method, app.http_url(uri)); for (name, value) in headers { - builder = builder.header(*name, *value); + let header_name = HeaderName::from_bytes(name.as_bytes()).expect("header name"); + let header_value = HeaderValue::from_str(value).expect("header value"); + builder = builder.header(header_name, header_value); } - let request_body = if let Some(body) = body { - builder = builder.header(header::CONTENT_TYPE, "application/json"); - Body::from(body.to_string()) + let response = if let Some(body) = body { + builder + .header(header::CONTENT_TYPE, "application/json") + .body(body.to_string()) + .send() + .await + .expect("request handled") } else { - Body::empty() + builder.send().await.expect("request handled") }; - - let request = builder.body(request_body).expect("build request"); - let response = app.clone().oneshot(request).await.expect("request handled"); let status = response.status(); let headers = response.headers().clone(); - let bytes = response - .into_body() - .collect() - .await - .expect("collect body") - .to_bytes(); + let bytes = response.bytes().await.expect("collect body"); (status, headers, bytes.to_vec()) } async fn send_request_raw( - app: &Router, + app: &docker_support::DockerApp, method: Method, uri: &str, body: Option>, headers: &[(&str, &str)], content_type: Option<&str>, ) -> (StatusCode, HeaderMap, Vec) { - let mut builder = Request::builder().method(method).uri(uri); + let client = reqwest::Client::new(); + let mut builder = client.request(method, app.http_url(uri)); for (name, value) in headers { - builder = builder.header(*name, *value); + let header_name = HeaderName::from_bytes(name.as_bytes()).expect("header name"); + let header_value = HeaderValue::from_str(value).expect("header value"); + builder = builder.header(header_name, header_value); } - let request_body = if let Some(body) = body { + let response = if let Some(body) = body { if let Some(content_type) = content_type { builder = builder.header(header::CONTENT_TYPE, content_type); } - Body::from(body) + builder.body(body).send().await.expect("request handled") } else { - Body::empty() + builder.send().await.expect("request handled") }; - - let request = builder.body(request_body).expect("build request"); - let response = app.clone().oneshot(request).await.expect("request handled"); let status = response.status(); let headers = response.headers().clone(); - let bytes = response - .into_body() - .collect() - .await - .expect("collect body") - .to_bytes(); + let bytes = response.bytes().await.expect("collect body"); (status, headers, bytes.to_vec()) } +async fn launch_desktop_focus_window(app: &docker_support::DockerApp, display: &str) { + let command = r#"nohup xterm -geometry 80x24+40+40 -title 'Sandbox Desktop Test' -e sh -lc 'sleep 60' >/tmp/sandbox-agent-xterm.log 2>&1 < /dev/null & for _ in $(seq 1 50); do wid="$(xdotool search --onlyvisible --name 'Sandbox Desktop Test' 2>/dev/null | head -n 1 || true)"; if [ -n "$wid" ]; then xdotool windowactivate "$wid"; exit 0; fi; sleep 0.1; done; exit 1"#; + let (status, _, body) = send_request( + app, + Method::POST, + "/v1/processes/run", + Some(json!({ + "command": "sh", + "args": ["-lc", command], + "env": { + "DISPLAY": display, + }, + "timeoutMs": 10_000 + })), + &[], + ) + .await; + + assert_eq!( + status, + StatusCode::OK, + "unexpected desktop focus window launch response: {}", + String::from_utf8_lossy(&body) + ); + let parsed = parse_json(&body); + assert_eq!(parsed["exitCode"], 0); +} + fn parse_json(bytes: &[u8]) -> Value { if bytes.is_empty() { Value::Null @@ -284,7 +198,7 @@ fn initialize_payload() -> Value { }) } -async fn bootstrap_server(app: &Router, server_id: &str, agent: &str) { +async fn bootstrap_server(app: &docker_support::DockerApp, server_id: &str, agent: &str) { let initialize = initialize_payload(); let (status, _, _body) = send_request( app, @@ -297,17 +211,17 @@ async fn bootstrap_server(app: &Router, server_id: &str, agent: &str) { assert_eq!(status, StatusCode::OK); } -async fn read_first_sse_data(app: &Router, server_id: &str) -> String { - let request = Request::builder() - .method(Method::GET) - .uri(format!("/v1/acp/{server_id}")) - .body(Body::empty()) - .expect("build request"); - - let response = app.clone().oneshot(request).await.expect("sse response"); +async fn read_first_sse_data(app: &docker_support::DockerApp, server_id: &str) -> String { + let client = reqwest::Client::new(); + let response = client + .get(app.http_url(&format!("/v1/acp/{server_id}"))) + .header("accept", "text/event-stream") + .send() + .await + .expect("sse response"); assert_eq!(response.status(), StatusCode::OK); - let mut stream = response.into_body().into_data_stream(); + let mut stream = response.bytes_stream(); tokio::time::timeout(Duration::from_secs(5), async move { while let Some(chunk) = stream.next().await { let bytes = chunk.expect("stream chunk"); @@ -323,21 +237,21 @@ async fn read_first_sse_data(app: &Router, server_id: &str) -> String { } async fn read_first_sse_data_with_last_id( - app: &Router, + app: &docker_support::DockerApp, server_id: &str, last_event_id: u64, ) -> String { - let request = Request::builder() - .method(Method::GET) - .uri(format!("/v1/acp/{server_id}")) + let client = reqwest::Client::new(); + let response = client + .get(app.http_url(&format!("/v1/acp/{server_id}"))) + .header("accept", "text/event-stream") .header("last-event-id", last_event_id.to_string()) - .body(Body::empty()) - .expect("build request"); - - let response = app.clone().oneshot(request).await.expect("sse response"); + .send() + .await + .expect("sse response"); assert_eq!(response.status(), StatusCode::OK); - let mut stream = response.into_body().into_data_stream(); + let mut stream = response.bytes_stream(); tokio::time::timeout(Duration::from_secs(5), async move { while let Some(chunk) = stream.next().await { let bytes = chunk.expect("stream chunk"); @@ -375,5 +289,7 @@ mod acp_transport; mod config_endpoints; #[path = "v1_api/control_plane.rs"] mod control_plane; +#[path = "v1_api/desktop.rs"] +mod desktop; #[path = "v1_api/processes.rs"] mod processes; diff --git a/server/packages/sandbox-agent/tests/v1_api/config_endpoints.rs b/server/packages/sandbox-agent/tests/v1_api/config_endpoints.rs index 3aec8ca5..e212c86b 100644 --- a/server/packages/sandbox-agent/tests/v1_api/config_endpoints.rs +++ b/server/packages/sandbox-agent/tests/v1_api/config_endpoints.rs @@ -22,8 +22,9 @@ async fn mcp_config_requires_directory_and_name() { #[tokio::test] async fn mcp_config_crud_round_trip() { let test_app = TestApp::new(AuthConfig::disabled()); - let project = tempfile::tempdir().expect("tempdir"); - let directory = project.path().to_string_lossy().to_string(); + let project = test_app.root_path().join("mcp-config-project"); + fs::create_dir_all(&project).expect("create project dir"); + let directory = project.to_string_lossy().to_string(); let entry = json!({ "type": "local", @@ -99,8 +100,9 @@ async fn skills_config_requires_directory_and_name() { #[tokio::test] async fn skills_config_crud_round_trip() { let test_app = TestApp::new(AuthConfig::disabled()); - let project = tempfile::tempdir().expect("tempdir"); - let directory = project.path().to_string_lossy().to_string(); + let project = test_app.root_path().join("skills-config-project"); + fs::create_dir_all(&project).expect("create project dir"); + let directory = project.to_string_lossy().to_string(); let entry = json!({ "sources": [ diff --git a/server/packages/sandbox-agent/tests/v1_api/control_plane.rs b/server/packages/sandbox-agent/tests/v1_api/control_plane.rs index dc352cad..fdd41313 100644 --- a/server/packages/sandbox-agent/tests/v1_api/control_plane.rs +++ b/server/packages/sandbox-agent/tests/v1_api/control_plane.rs @@ -1,4 +1,5 @@ use super::*; +use std::collections::BTreeMap; #[tokio::test] async fn v1_health_removed_legacy_and_opencode_unmounted() { @@ -137,10 +138,19 @@ async fn v1_filesystem_endpoints_round_trip() { #[tokio::test] #[serial] async fn require_preinstall_blocks_missing_agent() { - let test_app = { - let _preinstall = EnvVarGuard::set("SANDBOX_AGENT_REQUIRE_PREINSTALL", "true"); - TestApp::new(AuthConfig::disabled()) - }; + let mut env = BTreeMap::new(); + env.insert( + "SANDBOX_AGENT_REQUIRE_PREINSTALL".to_string(), + "true".to_string(), + ); + let test_app = TestApp::with_options( + AuthConfig::disabled(), + docker_support::TestAppOptions { + env, + ..Default::default() + }, + |_| {}, + ); let (status, _, body) = send_request( &test_app.app, @@ -176,20 +186,26 @@ async fn lazy_install_runs_on_first_bootstrap() { ] })); - let _registry = EnvVarGuard::set("SANDBOX_AGENT_ACP_REGISTRY_URL", ®istry_url); - let test_app = TestApp::with_setup(AuthConfig::disabled(), |install_path| { - fs::create_dir_all(install_path.join("agent_processes")) - .expect("create agent processes dir"); - write_executable(&install_path.join("codex"), "#!/usr/bin/env sh\nexit 0\n"); - fs::create_dir_all(install_path.join("bin")).expect("create bin dir"); - write_fake_npm(&install_path.join("bin").join("npm")); - }); - - let original_path = std::env::var_os("PATH").unwrap_or_default(); - let mut paths = vec![test_app.install_path().join("bin")]; - paths.extend(std::env::split_paths(&original_path)); - let merged_path = std::env::join_paths(paths).expect("join PATH"); - let _path_guard = EnvVarGuard::set_os("PATH", merged_path.as_os_str()); + let helper_bin_root = tempfile::tempdir().expect("helper bin tempdir"); + let helper_bin = helper_bin_root.path().join("bin"); + fs::create_dir_all(&helper_bin).expect("create helper bin dir"); + write_fake_npm(&helper_bin.join("npm")); + + let mut env = BTreeMap::new(); + env.insert("SANDBOX_AGENT_ACP_REGISTRY_URL".to_string(), registry_url); + let test_app = TestApp::with_options( + AuthConfig::disabled(), + docker_support::TestAppOptions { + env, + extra_paths: vec![helper_bin.clone()], + ..Default::default() + }, + |install_path| { + fs::create_dir_all(install_path.join("agent_processes")) + .expect("create agent processes dir"); + write_executable(&install_path.join("codex"), "#!/usr/bin/env sh\nexit 0\n"); + }, + ); let (status, _, _) = send_request( &test_app.app, diff --git a/server/packages/sandbox-agent/tests/v1_api/desktop.rs b/server/packages/sandbox-agent/tests/v1_api/desktop.rs new file mode 100644 index 00000000..cfc6c9e5 --- /dev/null +++ b/server/packages/sandbox-agent/tests/v1_api/desktop.rs @@ -0,0 +1,226 @@ +use super::*; +use serial_test::serial; +use std::collections::BTreeMap; + +#[tokio::test] +#[serial] +async fn v1_desktop_status_reports_install_required_when_dependencies_are_missing() { + let temp = tempfile::tempdir().expect("create empty path tempdir"); + let mut env = BTreeMap::new(); + env.insert( + "PATH".to_string(), + temp.path().to_string_lossy().to_string(), + ); + + let test_app = TestApp::with_options( + AuthConfig::disabled(), + docker_support::TestAppOptions { + env, + replace_path: true, + ..Default::default() + }, + |_| {}, + ); + + let (status, _, body) = + send_request(&test_app.app, Method::GET, "/v1/desktop/status", None, &[]).await; + + assert_eq!(status, StatusCode::OK); + let parsed = parse_json(&body); + assert_eq!(parsed["state"], "install_required"); + assert!(parsed["missingDependencies"] + .as_array() + .expect("missingDependencies array") + .iter() + .any(|value| value == "Xvfb")); + assert_eq!( + parsed["installCommand"], + "sandbox-agent install desktop --yes" + ); +} + +#[tokio::test] +#[serial] +async fn v1_desktop_lifecycle_and_actions_work_with_real_runtime() { + let test_app = TestApp::new(AuthConfig::disabled()); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/start", + Some(json!({ + "width": 1440, + "height": 900, + "dpi": 96 + })), + &[], + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "unexpected start response: {}", + String::from_utf8_lossy(&body) + ); + let parsed = parse_json(&body); + assert_eq!(parsed["state"], "active"); + let display = parsed["display"] + .as_str() + .expect("desktop display") + .to_string(); + assert!(display.starts_with(':')); + assert_eq!(parsed["resolution"]["width"], 1440); + assert_eq!(parsed["resolution"]["height"], 900); + + let (status, headers, body) = send_request_raw( + &test_app.app, + Method::GET, + "/v1/desktop/screenshot", + None, + &[], + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("image/png") + ); + assert!(body.starts_with(b"\x89PNG\r\n\x1a\n")); + + let (status, _, body) = send_request_raw( + &test_app.app, + Method::GET, + "/v1/desktop/screenshot/region?x=10&y=20&width=30&height=40", + None, + &[], + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(body.starts_with(b"\x89PNG\r\n\x1a\n")); + + let (status, _, body) = send_request( + &test_app.app, + Method::GET, + "/v1/desktop/display/info", + None, + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let display_info = parse_json(&body); + assert_eq!(display_info["display"], display); + assert_eq!(display_info["resolution"]["width"], 1440); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/move", + Some(json!({ "x": 400, "y": 300 })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let mouse = parse_json(&body); + assert_eq!(mouse["x"], 400); + assert_eq!(mouse["y"], 300); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/drag", + Some(json!({ + "startX": 100, + "startY": 110, + "endX": 220, + "endY": 230, + "button": "left" + })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let dragged = parse_json(&body); + assert_eq!(dragged["x"], 220); + assert_eq!(dragged["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/click", + Some(json!({ + "x": 220, + "y": 230, + "button": "left", + "clickCount": 1 + })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let clicked = parse_json(&body); + assert_eq!(clicked["x"], 220); + assert_eq!(clicked["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/scroll", + Some(json!({ + "x": 220, + "y": 230, + "deltaY": -3 + })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let scrolled = parse_json(&body); + assert_eq!(scrolled["x"], 220); + assert_eq!(scrolled["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::GET, + "/v1/desktop/mouse/position", + None, + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let position = parse_json(&body); + assert_eq!(position["x"], 220); + assert_eq!(position["y"], 230); + + launch_desktop_focus_window(&test_app.app, &display).await; + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/keyboard/type", + Some(json!({ "text": "hello world", "delayMs": 5 })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(parse_json(&body)["ok"], true); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/keyboard/press", + Some(json!({ "key": "ctrl+l" })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(parse_json(&body)["ok"], true); + + let (status, _, body) = + send_request(&test_app.app, Method::POST, "/v1/desktop/stop", None, &[]).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(parse_json(&body)["state"], "inactive"); +} diff --git a/server/packages/sandbox-agent/tests/v1_api/processes.rs b/server/packages/sandbox-agent/tests/v1_api/processes.rs index 3c02029b..8a036438 100644 --- a/server/packages/sandbox-agent/tests/v1_api/processes.rs +++ b/server/packages/sandbox-agent/tests/v1_api/processes.rs @@ -413,22 +413,17 @@ async fn v1_process_logs_follow_sse_streams_entries() { .expect("process id") .to_string(); - let request = Request::builder() - .method(Method::GET) - .uri(format!( + let response = reqwest::Client::new() + .get(test_app.app.http_url(&format!( "/v1/processes/{process_id}/logs?stream=stdout&follow=true" - )) - .body(Body::empty()) - .expect("build request"); - let response = test_app - .app - .clone() - .oneshot(request) + ))) + .header("accept", "text/event-stream") + .send() .await .expect("sse response"); assert_eq!(response.status(), StatusCode::OK); - let mut stream = response.into_body().into_data_stream(); + let mut stream = response.bytes_stream(); let chunk = tokio::time::timeout(Duration::from_secs(5), async move { while let Some(chunk) = stream.next().await { let bytes = chunk.expect("stream chunk");