diff --git a/packages/core/src/agent-os.ts b/packages/core/src/agent-os.ts index 0d4d1cb603..d5d35417ee 100644 --- a/packages/core/src/agent-os.ts +++ b/packages/core/src/agent-os.ts @@ -595,6 +595,14 @@ export interface AgentOsOptions { * Defaults to the hardened builtin set used by the native sidecar bridge. */ allowedNodeBuiltins?: string[]; + /** + * Opt in to a high-resolution monotonic guest clock (microsecond class) + * for guest Node processes. Default `false` keeps the security-oriented + * 1ms timer resolution — untrusted guest code should not get a precise + * timer (timing side channels). Enable only for trusted benchmarking or + * profiling workloads. + */ + highResolutionTime?: boolean; /** Root filesystem configuration. Defaults to an overlay with the bundled base snapshot as its deepest lower. */ rootFilesystem?: RootFilesystemConfig; /** Filesystems to mount at boot time. */ @@ -2916,6 +2924,7 @@ export class AgentOs { // emulation), matching the prior behavior where Agent OS only // constrained the builtin allow-list. ...(options?.allowedNodeBuiltins !== undefined || + options?.highResolutionTime !== undefined || snapshotUserlandCode !== undefined ? { jsRuntime: { @@ -2924,6 +2933,9 @@ export class AgentOs { ...(options?.allowedNodeBuiltins !== undefined ? { allowedBuiltins: options.allowedNodeBuiltins } : {}), + ...(options?.highResolutionTime !== undefined + ? { highResolutionTime: options.highResolutionTime } + : {}), ...(snapshotUserlandCode !== undefined ? { snapshotUserlandCode } : {}), diff --git a/packages/core/src/options-schema.ts b/packages/core/src/options-schema.ts index 3600da3b38..0bdd4a3f64 100644 --- a/packages/core/src/options-schema.ts +++ b/packages/core/src/options-schema.ts @@ -270,6 +270,7 @@ export const agentOsOptionFieldSchemas = { defaultSoftware: z.boolean().optional(), loopbackExemptPorts: z.array(z.number().int().min(0).max(65535)).optional(), allowedNodeBuiltins: stringArray.optional(), + highResolutionTime: z.boolean().optional(), rootFilesystem: rootFilesystemConfigSchema.optional(), mounts: z.array(mountConfigSchema).optional(), moduleAccessCwd: z.string().optional(), diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md index 332b8b1b53..5e3068027f 100644 --- a/scripts/benchmarks/README.md +++ b/scripts/benchmarks/README.md @@ -40,6 +40,14 @@ are: Set `BENCH_FAMILIES=net,fs` with `BENCH_ONLY=fuzz-perf` to run only those latency-matrix families and skip the fuzz/leak/footprint stages. +The `fuzz-perf` latency matrix reports native, host Node, guest VM JS, and +optional vm-wasm p50s. The vm-wasm lane runs the same Rust `native-baseline` +CLI as a `wasm32-wasip1` command inside the benchmark VM for supported +filesystem/control ops. Its in-VM clock is 1ms-quantized, so fast samples may +show millisecond buckets; the runner reports those samples as-is and relies on +the benchmark iteration count for amplification rather than fabricating +sub-millisecond precision. + - `coldstart-sleep` - `echo-cold-warm` - `memory-sleep` diff --git a/scripts/benchmarks/lib/layers.ts b/scripts/benchmarks/lib/layers.ts index 183a0b76a2..a7e04eb26d 100644 --- a/scripts/benchmarks/lib/layers.ts +++ b/scripts/benchmarks/lib/layers.ts @@ -1,22 +1,51 @@ import { execFileSync, spawnSync } from "node:child_process"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + copyFileSync, + existsSync, + mkdtempSync, + mkdirSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { AgentOs } from "@rivet-dev/agentos-core"; +import { AgentOs, createHostDirBackend } from "@rivet-dev/agentos-core"; import type { NativeOp } from "./native.js"; import { runNativeLayer } from "./native.js"; import { nowMs, round, stats, type Stats } from "./perf-utils.js"; +const DEFAULT_NATIVE_BASELINE_WASM = + "/home/nathan/.herdr/workspaces/agent-os/secure-exec-perf-rules/target/wasm32-wasip1/release/native-baseline.wasm"; +const WASM_COMMAND_NAME = "native-baseline"; +const WASM_COMMAND_PATH = `/__secure_exec/commands/0/${WASM_COMMAND_NAME}`; +const WASM_BASE_DIR = "/mnt/native-baseline-wasm"; +const WASM_SUPPORTED_OPS = new Set([ + "fs_stat", + "fs_write", + "fs_read", + "fs_open_close", + "fs_mkdir_rmdir", + "fs_rename", + "fs_readdir", + "fs_fsync", + "cpu_loop", + "alloc_free", +]); +let wasmCommandDir: string | undefined; +let wasmWritableDir: string | undefined; + export interface LayerSamples { native: number[]; node: number[]; guest: number[]; + wasm?: number[]; } export interface LayerStats { native: Stats; node: Stats; guest: Stats; + wasm?: Stats; } export interface BenchmarkOp { @@ -46,9 +75,37 @@ export interface OpResult { tax: { emulation: number; total: number; + wasm?: number; }; } +export function supportsWasmLayer(op: NativeOp): boolean { + return WASM_SUPPORTED_OPS.has(op); +} + +export function wasmLayerMounts(): + | { path: string; plugin: ReturnType }[] + | undefined { + const wasm = resolveNativeBaselineWasm(); + if (!wasm) return undefined; + return [ + { + path: "/__secure_exec/commands/0", + plugin: createHostDirBackend({ + hostPath: ensureWasmCommandDir(wasm), + readOnly: true, + }), + }, + { + path: WASM_BASE_DIR, + plugin: createHostDirBackend({ + hostPath: ensureWasmWritableDir(), + readOnly: false, + }), + }, + ]; +} + export function timedProgram(operationSource: string, setupSource?: string): string { return ` const iters = Number(process.env.BENCH_ITERATIONS || 20); @@ -163,6 +220,67 @@ export async function runGuestSpawn( return samples; } +function resolveNativeBaselineWasm(): string | undefined { + const wasm = process.env.NATIVE_BASELINE_WASM ?? DEFAULT_NATIVE_BASELINE_WASM; + if (!wasm || !existsSync(wasm)) return undefined; + return wasm; +} + +function ensureWasmCommandDir(wasmPath: string): string { + if (wasmCommandDir) return wasmCommandDir; + const dir = mkdtempSync(join(tmpdir(), "agentos-native-baseline-wasm-cmd-")); + mkdirSync(dir, { recursive: true }); + copyFileSync(wasmPath, join(dir, WASM_COMMAND_NAME)); + wasmCommandDir = dir; + return wasmCommandDir; +} + +function ensureWasmWritableDir(): string { + if (wasmWritableDir) return wasmWritableDir; + wasmWritableDir = mkdtempSync(join(tmpdir(), "agentos-native-baseline-wasm-data-")); + return wasmWritableDir; +} + +export async function runWasmLayer( + vm: AgentOs, + nativeOp: NativeOp, + iters: number, + warmup: number, +): Promise { + if (!supportsWasmLayer(nativeOp)) return undefined; + if (!resolveNativeBaselineWasm()) return undefined; + const hostBaseDir = join(ensureWasmWritableDir(), nativeOp); + rmSync(hostBaseDir, { recursive: true, force: true }); + mkdirSync(hostBaseDir, { recursive: true }); + const guestBaseDir = `${WASM_BASE_DIR}/${nativeOp}`; + const result = await vm.execArgv(WASM_COMMAND_PATH, [ + "--op", + nativeOp, + "--iters", + String(iters), + "--warmup", + String(warmup), + "--base-dir", + guestBaseDir, + ]); + if (result.exitCode !== 0) { + throw new Error(`wasm native-baseline ${nativeOp} exited ${result.exitCode}\n${result.stderr}`); + } + const parsed = JSON.parse(result.stdout) as { + unit?: string; + samples?: number[]; + unsupported?: boolean; + op?: string; + }; + if (parsed.unsupported) { + throw new Error(`wasm native-baseline unexpectedly returned unsupported for ${nativeOp}`); + } + if (parsed.unit !== "ns" || !Array.isArray(parsed.samples)) { + throw new Error(`wasm native-baseline emitted unexpected output: ${result.stdout}`); + } + return parsed.samples.map((ns) => ns / 1e6); +} + export async function runOp( op: BenchmarkOp, vm: AgentOs, @@ -182,10 +300,12 @@ export async function runOp( warmup, `${op.family}-${op.name}`, ); + const wasm = await runWasmLayer(vm, op.nativeOp, iters, warmup); const layers = { native: stats(native), node: stats(node), guest: stats(guest), + ...(wasm ? { wasm: stats(wasm) } : {}), }; return { family: op.family, @@ -197,6 +317,7 @@ export async function runOp( tax: { emulation: round(layers.guest.p50 / layers.node.p50), total: round(layers.guest.p50 / layers.native.p50), + ...(layers.wasm ? { wasm: round(layers.wasm.p50 / layers.native.p50) } : {}), }, }; } diff --git a/scripts/benchmarks/run-all.ts b/scripts/benchmarks/run-all.ts index 2d8ac4f6a5..6ae61e198c 100644 --- a/scripts/benchmarks/run-all.ts +++ b/scripts/benchmarks/run-all.ts @@ -1,6 +1,11 @@ import { AgentOs } from "@rivet-dev/agentos-core"; import { allOps } from "./families/index.js"; -import { runOp, type OpResult } from "./lib/layers.js"; +import { + runOp, + supportsWasmLayer, + wasmLayerMounts, + type OpResult, +} from "./lib/layers.js"; import { findingsFromLatency, refutedFromLatency, writeJson } from "./lib/report.js"; import { getHardware, printTable } from "./lib/perf-utils.js"; import { runFuzz } from "./fuzz/run.js"; @@ -17,8 +22,17 @@ const FAMILY_FILTER = process.env.BENCH_FAMILIES .filter(Boolean); export async function runLatencyMatrix(): Promise { + const wasmMounts = wasmLayerMounts(); + if (!wasmMounts) { + console.error("vm-wasm lane disabled: native-baseline wasm artifact was not found"); + } const vm = await AgentOs.create({ permissions: { fs: "allow", network: "allow", childProcess: "allow", process: "allow" }, + ...(wasmMounts ? { mounts: wasmMounts } : {}), + // Benchmark VM: opt in to the µs-resolution guest clock so sub-ms guest + // samples are real instead of 1ms-floor artifacts. Never enable this for + // untrusted workloads (timing side channels) — off by default everywhere. + highResolutionTime: true, }); try { const results: OpResult[] = []; @@ -27,6 +41,9 @@ export async function runLatencyMatrix(): Promise { : allOps; for (const op of ops) { console.error(`latency ${op.family}/${op.name}`); + if (supportsWasmLayer(op.nativeOp)) { + console.error(" wasm lane: guest JS measured first, wasm native-baseline after"); + } results.push(await runOp(op, vm, ITERATIONS, WARMUP)); } return results; @@ -81,6 +98,18 @@ async function main(): Promise { const diff = compareBaselineFile(`${RESULTS_DIR}/findings.json`, baselinePath); writeJson(`${RESULTS_DIR}/regression-diff.json`, diff); + printTable( + ["family", "op", "native p50", "node p50", "guest p50", "wasm p50"], + latency.map((result) => [ + result.family, + result.op, + `${result.layers.native.p50}ms`, + `${result.layers.node.p50}ms`, + `${result.layers.guest.p50}ms`, + result.layers.wasm ? `${result.layers.wasm.p50}ms` : "-", + ]), + ); + printTable( ["family", "op", "guest/node", "guest/native", "file:line"], findingsJson.findings.map((finding) => [ diff --git a/scripts/benchmarks/run-benchmarks.sh b/scripts/benchmarks/run-benchmarks.sh index 40d9fc434c..5f75481da6 100755 --- a/scripts/benchmarks/run-benchmarks.sh +++ b/scripts/benchmarks/run-benchmarks.sh @@ -57,6 +57,26 @@ build_native_baseline() { echo "Using native baseline: $NATIVE_BASELINE_BIN" >&2 } +build_native_baseline_wasm() { + local secure_exec_root + secure_exec_root="$(resolve_secure_exec_root)" + if command -v rustup >/dev/null 2>&1 && ! rustup target list --installed | grep -qx "wasm32-wasip1"; then + echo "=== Skipping vm-wasm lane: Rust target wasm32-wasip1 is not installed ===" >&2 + echo "Install it with: rustup target add wasm32-wasip1" >&2 + export NATIVE_BASELINE_WASM="" + return + fi + echo "" >&2 + echo "=== Building native-baseline wasm32-wasip1 ===" >&2 + cargo build --release --target wasm32-wasip1 -p native-baseline --manifest-path "$secure_exec_root/Cargo.toml" >&2 + if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then + export NATIVE_BASELINE_WASM="${NATIVE_BASELINE_WASM:-$CARGO_TARGET_DIR/wasm32-wasip1/release/native-baseline.wasm}" + else + export NATIVE_BASELINE_WASM="${NATIVE_BASELINE_WASM:-$secure_exec_root/target/wasm32-wasip1/release/native-baseline.wasm}" + fi + echo "Using wasm native baseline: $NATIVE_BASELINE_WASM" >&2 +} + run() { local name="$1" shift @@ -332,6 +352,7 @@ fi # leak, footprint, findings, and regression diff. if should_run "fuzz-perf"; then build_native_baseline + build_native_baseline_wasm export NODE_OPTIONS="${NODE_OPTIONS:---expose-gc}" export BENCH_ITERATIONS="${BENCH_FUZZ_PERF_ITERATIONS:-${BENCH_ITERATIONS:-20}}" export BENCH_WARMUP="${BENCH_FUZZ_PERF_WARMUP:-${BENCH_WARMUP:-5}}"