diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f148b6f2..ad01d30e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,7 +4,7 @@ steps: - mise#a5845c5082d3a4fe36dd77ae74973dfc86fc91a2: version: "2026.5.12" install_args: shellcheck - command: shellcheck -x .buildkite/hooks/pre-command scripts/base-image-tag.sh scripts/cleanroom-root-helper.sh scripts/benchmark-tti.sh scripts/build-go.sh scripts/install-go.sh scripts/install.sh scripts/install-global.sh scripts/package-darwin-vz-helper.sh scripts/release.sh scripts/e2e-observability.sh scripts/ci-with-host-lock.sh scripts/ci-go-test-engine.sh scripts/ci-auth-oidc-smoke.sh scripts/ci-example-smoke.sh scripts/ci-examples-firecracker.sh scripts/ci-examples-darwin-vz.sh scripts/ci-cleanroom-e2e.sh scripts/ci-darwin-vz-e2e.sh scripts/ci-darwin-vz-filehandle-e2e.sh scripts/build-macos-release-pkg.sh scripts/notarize-macos-package.sh scripts/ci-macos-release-pkg.sh scripts/ci-buildkite-release.sh + command: shellcheck -x .buildkite/hooks/pre-command scripts/base-image-tag.sh scripts/cleanroom-root-helper.sh scripts/benchmark-tti.sh benchmarks/darwin-vz/macos-minimal/build-runner.sh benchmarks/darwin-vz/macos-minimal/build-viewer.sh benchmarks/darwin-vz/macos-minimal/build-create-bundle.sh benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh benchmarks/darwin-vz/macos-minimal/build-guest-agent-pkg.sh benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh scripts/build-go.sh scripts/install-go.sh scripts/install.sh scripts/install-global.sh scripts/package-darwin-vz-helper.sh scripts/release.sh scripts/e2e-observability.sh scripts/ci-with-host-lock.sh scripts/ci-go-test-engine.sh scripts/ci-auth-oidc-smoke.sh scripts/ci-example-smoke.sh scripts/ci-examples-firecracker.sh scripts/ci-examples-darwin-vz.sh scripts/ci-cleanroom-e2e.sh scripts/ci-darwin-vz-e2e.sh scripts/ci-darwin-vz-filehandle-e2e.sh scripts/build-macos-release-pkg.sh scripts/notarize-macos-package.sh scripts/ci-macos-release-pkg.sh scripts/ci-buildkite-release.sh cache: paths: - "~/.local/share/mise" diff --git a/benchmarks/darwin-vz/macos-minimal/README.md b/benchmarks/darwin-vz/macos-minimal/README.md new file mode 100644 index 00000000..e017f5c5 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/README.md @@ -0,0 +1,307 @@ +# Minimal macOS darwin-vz benchmark + +This directory contains a standalone Virtualization.framework probe for macOS +guest experiments. It is not a Cleanroom backend and does not replace the +Linux `benchmarks/darwin-vz/minimal` benchmark. + +The runner boots a prepared Apple Silicon macOS VM bundle, connects to a guest +agent over a VZ virtio socket, sends a small exec request, streams guest +stdout/stderr to the host, and writes timing metadata as JSON. + +## Bundle layout + +The runner accepts either a bundle metadata file or a directory containing +`bundle.json`. Relative paths are resolved from the metadata file's directory. + +```json +{ + "schema_version": 1, + "os": "macos", + "arch": "arm64", + "macos_version": "15.5", + "macos_build": "24F74", + "vcpus": 4, + "memory_mib": 8192, + "disk": "disk.img", + "auxiliary_storage": "auxiliary.storage", + "hardware_model": "hardware-model.bin", + "machine_identifier": "machine-identifier.bin", + "agent": { + "transport": "virtio_socket", + "port": 10700, + "version": "0.1.0" + }, + "display": { + "width_px": 1024, + "height_px": 768, + "pixels_per_inch": 72 + } +} +``` + +The hardware model and machine identifier files are the opaque +Virtualization.framework data representations for `VZMacHardwareModel` and +`VZMacMachineIdentifier`. A later image-prep slice should generate and clone +these safely; this probe only validates and consumes an existing bundle. + +## Build + +```bash +benchmarks/darwin-vz/macos-minimal/build-runner.sh +``` + +The script writes `dist/darwin-vz-macos-minimal` and signs it with +`cmd/cleanroom-darwin-vz/entitlements.plist` by default. + +For setup or manual image finalization, build the viewer: + +```bash +benchmarks/darwin-vz/macos-minimal/build-viewer.sh +``` + +The viewer writes `dist/darwin-vz-macos-viewer`. It boots the same bundle in a +`VZVirtualMachineView` window and can expose one host directory read-only using +the macOS guest automount tag. Pass `--validate-only` to check the bundle and +share configuration without starting the VM. For headless diagnostics, pass +`--screenshot /tmp/vm.png`; the viewer writes the PNG from inside its own +process after the VM starts, which works even when host `screencapture` cannot +capture the VZ window. + +## Create a local bundle from an IPSW + +```bash +benchmarks/darwin-vz/macos-minimal/build-create-bundle.sh + +dist/darwin-vz-macos-create-bundle \ + --ipsw /path/to/UniversalMac.ipsw \ + --out /path/to/cleanroom-macos-bundle \ + --disk-size-gib 120 +``` + +The create-bundle tool installs macOS from a local Apple Silicon IPSW into the +same bundle layout consumed by the runner: + +- `bundle.json` +- `disk.img` +- `auxiliary.storage` +- `hardware-model.bin` +- `machine-identifier.bin` + +It chooses CPU and memory defaults that satisfy the restore image requirements +and writes the macOS version/build discovered from the IPSW. The default +`agent.version` is `uninstalled`; use `--vcpus`, `--memory-mib`, `--agent-port`, +`--agent-version`, and `--display` to override the metadata written to +`bundle.json`. + +This only prepares the VM bundle. It does not install the Cleanroom macOS guest +agent inside the guest. Before the runner can execute commands, boot the bundle +once, finish macOS setup, install the guest agent as a LaunchDaemon, then shut +the guest down cleanly. + +## Prepare an agent bundle + +Build the minimal macOS guest agent: + +```bash +benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh +``` + +Build the package that can install and bootstrap the LaunchDaemon inside a +running guest: + +```bash +benchmarks/darwin-vz/macos-minimal/build-guest-agent-pkg.sh +``` + +Then clone a base bundle and install the agent into the clone's APFS Data +volume: + +```bash +benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh \ + --base /path/to/cleanroom-macos-base \ + --out /path/to/cleanroom-macos-agent \ + --force +``` + +The prepare script leaves the base bundle untouched. It installs +`/usr/local/bin/cleanroom-macos-guest-agent`, writes +`/Library/LaunchDaemons/com.buildkite.cleanroom.macos-guest-agent.plist`, and +updates `bundle.json` with the installed agent version. + +When the script is run without root privileges, it may be unable to set +root-owned metadata on the installed files. By default it fails in that case +because launchd may reject the daemon. Use `--allow-unverified-ownership` only +for inspecting the offline image flow; a bundle prepared that way is not +command-runnable until the live smoke proves the agent starts. + +For a setup boot or manual guest finalization, copy +`dist/cleanroom-macos-guest-agent.pkg` into the guest and run: + +```bash +sudo installer -pkg /path/to/cleanroom-macos-guest-agent.pkg -target / +``` + +One local way to make the package available to the guest is to boot the bundle +with the viewer and share `dist/`: + +```bash +dist/darwin-vz-macos-viewer \ + --bundle /path/to/cleanroom-macos-agent \ + --shared-directory dist +``` + +Inside the guest, the package is available at: + +```bash +/Volumes/My Shared Files/cleanroom-macos-guest-agent.pkg +``` + +If the guest stops at loginwindow, the image still needs user/session +finalization before an agent installed as a LaunchAgent can run. The root-owned +LaunchDaemon path below avoids requiring a logged-in user, but it must be +installed by a privileged in-guest step or by a privileged offline image-prep +step. + +The package is script-only so it does not archive host-side AppleDouble +metadata. Its postinstall runs inside the guest as root, writes the agent and +LaunchDaemon plist as `root:wheel`, then attempts to load and start +`com.buildkite.cleanroom.macos-guest-agent`. + +For a non-root host-side probe, prepare a user-cron bundle: + +```bash +benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh \ + --base /path/to/cleanroom-macos-base \ + --out /path/to/cleanroom-macos-usercron \ + --install-mode user-cron \ + --force +``` + +This experimental mode creates a local `cleanroom` admin user in the guest +image, installs the agent under `/Users/cleanroom/bin`, and writes a user +crontab that starts the agent on port 10700. The default UID/GID matches the +current host user so files created through a rootless APFS mount have the +expected guest ownership. It is meant for the standalone harness while the +privileged image-finalization path is still being proved; it runs commands as +the `cleanroom` user, not as root. + +To produce a LaunchDaemon-backed headless bundle without host sudo or GUI +automation, run the in-guest finalizer: + +```bash +benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh \ + --base /path/to/cleanroom-macos-base \ + --out /path/to/cleanroom-macos-finalized \ + --metrics-dir /tmp/cleanroom-macos-finalize \ + --force +``` + +The finalizer creates a temporary `user-cron` bootstrap bundle without +configuring autologin, boots it once, uses the bootstrap agent to run `sudo` +inside the guest, installs +`/usr/local/bin/cleanroom-macos-guest-agent` and +`/Library/LaunchDaemons/com.buildkite.cleanroom.macos-guest-agent.plist` as +`root:wheel`, writes `/private/var/db/cleanroom-macos-guest-agent.finalized`, +then boots the bundle again to prove exec is served by the LaunchDaemon as +`root`. Between those boots it removes the temporary user record, home +directory, and crontab from the cloned Data volume while the VM is stopped. The +bootstrap cron entry still checks the finalized marker before starting the user +agent, so a failed finalization leaves an inert bootstrap path after the +LaunchDaemon marker is written. + +To produce a GUI-capable local harness image, pass `--profile gui`: + +```bash +benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh \ + --base /path/to/cleanroom-macos-base \ + --out /path/to/cleanroom-macos-gui \ + --profile gui \ + --metrics-dir /tmp/cleanroom-macos-gui-finalize \ + --force +``` + +The GUI profile still installs the root LaunchDaemon on `agent.port`, but it +keeps the `cleanroom` user, leaves autologin configured, and rewrites that +user's LaunchAgent to serve exec on `user_agent.port`. The finalizer then boots +the image again, verifies the root daemon, connects to the user agent, launches +TextEdit with `open -a TextEdit`, and attempts a guest-side `screencapture`. +The screenshot can be unavailable when the VM is running through the headless +runner without an attached VZ view, so the hard smoke assertion is app launch +through the user session. This is a local harness profile for proving GUI +session mechanics, not production GUI automation support. + +## Validate metadata + +```bash +dist/darwin-vz-macos-minimal \ + --bundle /path/to/bundle.json \ + --validate-only +``` + +Validation checks the manifest shape, required files, the current host's +Virtualization.framework support, and whether the hardware model can be loaded +on this host. It does not start the VM. + +## Run a command + +```bash +dist/darwin-vz-macos-minimal \ + --bundle /path/to/bundle.json \ + --metrics /tmp/macos-cleanroom-smoke.json \ + -- /usr/bin/sw_vers +``` + +The command defaults to `/usr/bin/sw_vers`. Guest stdout and stderr are +streamed to the matching host streams. The runner exits with the guest command +exit code and writes timing metadata to the path provided by `--metrics`. + +For GUI-profile bundles, use `--agent user` to connect to the user LaunchAgent +instead of the root LaunchDaemon: + +```bash +dist/darwin-vz-macos-minimal \ + --bundle /path/to/cleanroom-macos-gui \ + --agent user \ + -- /usr/bin/open -a TextEdit +``` + +## Run through the production helper + +Build the helper-backed runner: + +```bash +benchmarks/darwin-vz/macos-minimal/build-helper-runner.sh +``` + +The helper runner starts `cleanroom-darwin-vz`, sends the experimental +`StartMacOSVM` helper operation, connects to the helper-managed proxy socket, +and runs the command through the macOS guest agent. This exercises the +production helper boundary while still staying outside the public Cleanroom +adapter path. + +```bash +dist/darwin-vz-macos-helper-runner \ + --helper dist/cleanroom-darwin-vz.app \ + --bundle /path/to/cleanroom-macos-gui \ + --agent user \ + --metrics /tmp/macos-cleanroom-helper-gui.json \ + -- /bin/sh -lc '/usr/bin/open -a TextEdit && /bin/sleep 2 && /usr/bin/pgrep -x TextEdit' +``` + +For GUI-profile bundles, the `--agent user` flag is required because GUI apps +need the logged-in Aqua session owned by the bundle's autologin user. The root +LaunchDaemon can run headless commands, but it cannot launch user-session GUI +apps. + +The guest agent protocol is deliberately tiny for the probe: + +- host sends one newline-delimited JSON request: + `{"command":["/usr/bin/sw_vers"]}` +- host sends `{"type":"eof"}` immediately after the request because this + runner does not forward interactive stdin yet +- guest sends newline-delimited JSON frames: + `stdout`, `stderr`, and a final `exit` frame +- `stdout` and `stderr` frame `data` values are base64-encoded bytes + +Slice 2 of `docs/plans/macos-cleanrooms.md` owns turning this into a packaged +macOS guest agent. diff --git a/benchmarks/darwin-vz/macos-minimal/build-create-bundle.sh b/benchmarks/darwin-vz/macos-minimal/build-create-bundle.sh new file mode 100755 index 00000000..79714be1 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/build-create-bundle.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +OUTPUT_PATH="${1:-${REPO_ROOT}/dist/darwin-vz-macos-create-bundle}" +ENTITLEMENTS_PATH="${CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS:-${REPO_ROOT}/cmd/cleanroom-darwin-vz/entitlements.plist}" + +mkdir -p "$(dirname "${OUTPUT_PATH}")" +xcrun swiftc \ + -O \ + -framework Virtualization \ + "${SCRIPT_DIR}/create-bundle.swift" \ + -o "${OUTPUT_PATH}" + +codesign --force --sign "${CLEANROOM_DARWIN_VZ_MACOS_MINIMAL_SIGN_IDENTITY:--}" \ + --entitlements "${ENTITLEMENTS_PATH}" \ + "${OUTPUT_PATH}" diff --git a/benchmarks/darwin-vz/macos-minimal/build-guest-agent-pkg.sh b/benchmarks/darwin-vz/macos-minimal/build-guest-agent-pkg.sh new file mode 100755 index 00000000..d0270282 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/build-guest-agent-pkg.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +OUT="${1:-"${ROOT_DIR}/dist/cleanroom-macos-guest-agent.pkg"}" +AGENT_BIN="${CLEANROOM_MACOS_GUEST_AGENT_BIN:-"${ROOT_DIR}/dist/cleanroom-macos-guest-agent"}" +AGENT_VERSION="${CLEANROOM_MACOS_GUEST_AGENT_VERSION:-0.1.0}" +AGENT_PORT="${CLEANROOM_MACOS_GUEST_AGENT_PORT:-10700}" +LABEL="com.buildkite.cleanroom.macos-guest-agent" + +case "${AGENT_PORT}" in + ''|*[!0-9]*) + echo "build-guest-agent-pkg: CLEANROOM_MACOS_GUEST_AGENT_PORT must be numeric" >&2 + exit 1 + ;; +esac + +if [[ ! -x "${AGENT_BIN}" ]]; then + "${ROOT_DIR}/benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh" "${AGENT_BIN}" >/dev/null +fi + +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/cleanroom-macos-agent-pkg.XXXXXX")" +cleanup() { + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +SCRIPTS="${TMP_DIR}/scripts" +PLIST="${SCRIPTS}/${LABEL}.plist" + +mkdir -p \ + "${SCRIPTS}" \ + "$(dirname "${OUT}")" + +install -m 0755 "${AGENT_BIN}" "${SCRIPTS}/cleanroom-macos-guest-agent" +sed "s/10700<\\/string>/${AGENT_PORT}<\\/string>/" \ + "${ROOT_DIR}/cmd/cleanroom-macos-guest-agent/${LABEL}.plist" > "${PLIST}" +chmod 0644 "${PLIST}" +xattr -cr "${SCRIPTS}" 2>/dev/null || true + +cat > "${SCRIPTS}/postinstall" <<'EOF' +#!/bin/sh +set -eu + +label="com.buildkite.cleanroom.macos-guest-agent" +script_dir=$(cd "$(dirname "$0")" && pwd) +target_volume=${3:-/} + +case "${target_volume}" in + ""|"/") + target_root="" + bootstrap=1 + ;; + *) + target_root=${target_volume%/} + bootstrap=0 + ;; +esac + +agent_path="${target_root}/usr/local/bin/cleanroom-macos-guest-agent" +plist="${target_root}/Library/LaunchDaemons/${label}.plist" + +/usr/bin/install -d -o root -g wheel -m 0755 "${target_root}/usr/local/bin" +/usr/bin/install -d -o root -g wheel -m 0755 "${target_root}/Library/LaunchDaemons" +/usr/bin/install -o root -g wheel -m 0755 \ + "${script_dir}/cleanroom-macos-guest-agent" \ + "${agent_path}" +/usr/bin/install -o root -g wheel -m 0644 \ + "${script_dir}/${label}.plist" \ + "${plist}" + +/usr/bin/xattr -c "${agent_path}" "${plist}" >/dev/null 2>&1 || true + +if [ "${bootstrap}" -eq 1 ] && [ -x /bin/launchctl ] && [ -f "${plist}" ]; then + /bin/launchctl bootout "system/${label}" >/dev/null 2>&1 || true + /bin/launchctl bootstrap system "${plist}" >/dev/null 2>&1 || true + /bin/launchctl kickstart -k "system/${label}" >/dev/null 2>&1 || true +fi + +exit 0 +EOF +chmod 0755 "${SCRIPTS}/postinstall" +xattr -cr "${SCRIPTS}" 2>/dev/null || true + +COPYFILE_DISABLE=1 pkgbuild \ + --nopayload \ + --scripts "${SCRIPTS}" \ + --identifier "${LABEL}" \ + --version "${AGENT_VERSION}" \ + --install-location / \ + "${OUT}" >/dev/null + +echo "${OUT}" diff --git a/benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh b/benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh new file mode 100755 index 00000000..e1b395cf --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +OUT="${1:-"${ROOT_DIR}/dist/cleanroom-macos-guest-agent"}" + +mkdir -p "$(dirname "${OUT}")" + +( + cd "${ROOT_DIR}" + GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -o "${OUT}" ./cmd/cleanroom-macos-guest-agent +) + +if command -v codesign >/dev/null 2>&1; then + codesign --force --sign - "${OUT}" >/dev/null +fi + +echo "${OUT}" diff --git a/benchmarks/darwin-vz/macos-minimal/build-helper-runner.sh b/benchmarks/darwin-vz/macos-minimal/build-helper-runner.sh new file mode 100755 index 00000000..8160573d --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/build-helper-runner.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +OUTPUT_PATH="${1:-${REPO_ROOT}/dist/darwin-vz-macos-helper-runner}" + +mkdir -p "$(dirname "${OUTPUT_PATH}")" +go build -o "${OUTPUT_PATH}" "${REPO_ROOT}/benchmarks/darwin-vz/macos-minimal" diff --git a/benchmarks/darwin-vz/macos-minimal/build-runner.sh b/benchmarks/darwin-vz/macos-minimal/build-runner.sh new file mode 100755 index 00000000..9d27cdda --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/build-runner.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +OUTPUT_PATH="${1:-${REPO_ROOT}/dist/darwin-vz-macos-minimal}" +ENTITLEMENTS_PATH="${CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS:-${REPO_ROOT}/cmd/cleanroom-darwin-vz/entitlements.plist}" + +mkdir -p "$(dirname "${OUTPUT_PATH}")" +xcrun swiftc \ + -O \ + -framework Virtualization \ + "${SCRIPT_DIR}/runner.swift" \ + -o "${OUTPUT_PATH}" + +codesign --force --sign "${CLEANROOM_DARWIN_VZ_MACOS_MINIMAL_SIGN_IDENTITY:--}" \ + --entitlements "${ENTITLEMENTS_PATH}" \ + "${OUTPUT_PATH}" diff --git a/benchmarks/darwin-vz/macos-minimal/build-viewer.sh b/benchmarks/darwin-vz/macos-minimal/build-viewer.sh new file mode 100755 index 00000000..58e220b9 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/build-viewer.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +OUTPUT_PATH="${1:-${REPO_ROOT}/dist/darwin-vz-macos-viewer}" +ENTITLEMENTS_PATH="${CLEANROOM_DARWIN_VZ_HELPER_ENTITLEMENTS:-${REPO_ROOT}/cmd/cleanroom-darwin-vz/entitlements.plist}" + +mkdir -p "$(dirname "${OUTPUT_PATH}")" + +swiftc \ + -O \ + -framework AppKit \ + -framework Virtualization \ + "${REPO_ROOT}/benchmarks/darwin-vz/macos-minimal/viewer.swift" \ + -o "${OUTPUT_PATH}" + +codesign --force --sign - --entitlements "${ENTITLEMENTS_PATH}" "${OUTPUT_PATH}" >/dev/null + +echo "${OUTPUT_PATH}" diff --git a/benchmarks/darwin-vz/macos-minimal/create-bundle.swift b/benchmarks/darwin-vz/macos-minimal/create-bundle.swift new file mode 100644 index 00000000..e086791d --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/create-bundle.swift @@ -0,0 +1,533 @@ +import Foundation +import Virtualization + +private struct Options { + var ipswPath = "" + var outputPath = "" + var diskSizeGiB: UInt64 = 120 + var vcpus: Int? + var memoryMiB: UInt64? + var agentPort: UInt32 = 10700 + var agentVersion = "uninstalled" + var displayWidthPx = 1024 + var displayHeightPx = 768 + var displayPixelsPerInch = 72 + var force = false +} + +private struct BundleManifest: Encodable { + let schemaVersion: Int + let os: String + let arch: String + let macOSVersion: String + let macOSBuild: String + let vcpus: Int + let memoryMiB: UInt64 + let disk: String + let auxiliaryStorage: String + let hardwareModel: String + let machineIdentifier: String + let agent: AgentManifest + let display: DisplayManifest + + enum CodingKeys: String, CodingKey { + case schemaVersion = "schema_version" + case os + case arch + case macOSVersion = "macos_version" + case macOSBuild = "macos_build" + case vcpus + case memoryMiB = "memory_mib" + case disk + case auxiliaryStorage = "auxiliary_storage" + case hardwareModel = "hardware_model" + case machineIdentifier = "machine_identifier" + case agent + case display + } +} + +private struct AgentManifest: Encodable { + let transport: String + let port: UInt32 + let version: String +} + +private struct DisplayManifest: Encodable { + let widthPx: Int + let heightPx: Int + let pixelsPerInch: Int + + enum CodingKeys: String, CodingKey { + case widthPx = "width_px" + case heightPx = "height_px" + case pixelsPerInch = "pixels_per_inch" + } +} + +private struct BundlePaths { + let directoryURL: URL + let manifestURL: URL + let diskURL: URL + let auxiliaryStorageURL: URL + let hardwareModelURL: URL + let machineIdentifierURL: URL + + init(directoryURL: URL) { + self.directoryURL = directoryURL + self.manifestURL = directoryURL.appendingPathComponent("bundle.json") + self.diskURL = directoryURL.appendingPathComponent("disk.img") + self.auxiliaryStorageURL = directoryURL.appendingPathComponent("auxiliary.storage") + self.hardwareModelURL = directoryURL.appendingPathComponent("hardware-model.bin") + self.machineIdentifierURL = directoryURL.appendingPathComponent("machine-identifier.bin") + } +} + +private enum CreateBundleError: LocalizedError { + case usage(String) + case invalid(String) + case io(String) + case vm(String) + + var errorDescription: String? { + switch self { + case .usage(let message), .invalid(let message), .io(let message), .vm(let message): + return message + } + } +} + +private final class ProgressReporter { + private let progress: Progress + private let queue = DispatchQueue(label: "cleanroom.benchmark.darwin-vz-macos-bundle.progress") + private var source: DispatchSourceTimer? + private var lastPercent = -1 + + init(progress: Progress) { + self.progress = progress + } + + func start() { + let source = DispatchSource.makeTimerSource(queue: queue) + source.schedule(deadline: .now(), repeating: .seconds(5)) + source.setEventHandler { [weak self] in + self?.printProgress() + } + self.source = source + source.resume() + } + + func stop() { + source?.cancel() + source = nil + } + + private func printProgress() { + let percent = Int((progress.fractionCompleted * 100.0).rounded(.down)) + guard percent != lastPercent else { + return + } + lastPercent = percent + fputs("macOS install progress: \(max(0, min(percent, 100)))%\n", stderr) + } +} + +private func usage() -> String { + """ + Usage: + create-bundle --ipsw --out [options] + + Options: + --disk-size-gib Raw disk size. Default: 120. + --vcpus vCPU count. Default: max(4, restore image minimum). + --memory-mib Guest memory. Default: max(8192, restore image minimum). + --agent-port Guest agent virtio socket port written to bundle.json. Default: 10700. + --agent-version Guest agent version written to bundle.json. Default: uninstalled. + --display Display geometry. Default: 1024x768@72. + --force Replace an existing output directory. + -h, --help Show this help. + + The tool installs macOS from a local Apple Silicon IPSW and writes the + bundle layout consumed by darwin-vz-macos-minimal. It does not install the + Cleanroom macOS guest agent inside the guest. + """ +} + +private func parseOptions(_ args: [String]) throws -> Options { + var opts = Options() + var i = 0 + while i < args.count { + let arg = args[i] + + func value() throws -> String { + guard i + 1 < args.count else { + throw CreateBundleError.usage("missing value for \(arg)\n\n\(usage())") + } + i += 1 + return args[i] + } + + switch arg { + case "--ipsw": + opts.ipswPath = try value() + case "--out": + opts.outputPath = try value() + case "--disk-size-gib": + guard let value = UInt64(try value()), value >= 20 else { + throw CreateBundleError.invalid("--disk-size-gib must be at least 20") + } + opts.diskSizeGiB = value + case "--vcpus": + guard let value = Int(try value()), value > 0 else { + throw CreateBundleError.invalid("--vcpus must be greater than zero") + } + opts.vcpus = value + case "--memory-mib": + guard let value = UInt64(try value()), value >= 1024 else { + throw CreateBundleError.invalid("--memory-mib must be at least 1024") + } + opts.memoryMiB = value + case "--agent-port": + guard let value = UInt32(try value()), value > 0 else { + throw CreateBundleError.invalid("--agent-port must be greater than zero") + } + opts.agentPort = value + case "--agent-version": + let value = try value() + guard !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw CreateBundleError.invalid("--agent-version must not be empty") + } + opts.agentVersion = value + case "--display": + let display = try parseDisplay(try value()) + opts.displayWidthPx = display.width + opts.displayHeightPx = display.height + opts.displayPixelsPerInch = display.ppi + case "--force": + opts.force = true + case "-h", "--help": + throw CreateBundleError.usage(usage()) + default: + throw CreateBundleError.usage("unknown argument: \(arg)\n\n\(usage())") + } + i += 1 + } + + if opts.ipswPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw CreateBundleError.usage("missing --ipsw\n\n\(usage())") + } + if opts.outputPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw CreateBundleError.usage("missing --out\n\n\(usage())") + } + return opts +} + +private func parseDisplay(_ input: String) throws -> (width: Int, height: Int, ppi: Int) { + let displayParts = input.split(separator: "@", omittingEmptySubsequences: false) + guard displayParts.count == 1 || displayParts.count == 2 else { + throw CreateBundleError.invalid("--display must use WxH or WxH@ppi") + } + + let sizeParts = displayParts[0].lowercased().split(separator: "x", omittingEmptySubsequences: false) + guard sizeParts.count == 2, let width = Int(sizeParts[0]), let height = Int(sizeParts[1]), width > 0, height > 0 else { + throw CreateBundleError.invalid("--display must use positive WxH dimensions") + } + + let ppi: Int + if displayParts.count == 2 { + guard let value = Int(displayParts[1]), value > 0 else { + throw CreateBundleError.invalid("--display ppi must be greater than zero") + } + ppi = value + } else { + ppi = 72 + } + return (width, height, ppi) +} + +private func absoluteURL(_ path: String) -> URL { + URL(fileURLWithPath: NSString(string: path).expandingTildeInPath) +} + +private func requireFile(_ url: URL, label: String) throws { + var isDir: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDir), !isDir.boolValue else { + throw CreateBundleError.invalid("\(label) does not exist or is not a file: \(url.path)") + } +} + +private func prepareOutputDirectory(_ outputURL: URL, force: Bool) throws -> BundlePaths { + let fm = FileManager.default + if fm.fileExists(atPath: outputURL.path), !force { + throw CreateBundleError.invalid("output directory already exists: \(outputURL.path)") + } + + let parent = outputURL.deletingLastPathComponent() + try fm.createDirectory(at: parent, withIntermediateDirectories: true) + let tempURL = parent.appendingPathComponent(".\(outputURL.lastPathComponent).tmp-\(UUID().uuidString)") + try fm.createDirectory(at: tempURL, withIntermediateDirectories: false) + return BundlePaths(directoryURL: tempURL) +} + +private func commitOutputDirectory(tempURL: URL, finalURL: URL, force: Bool) throws { + let fm = FileManager.default + guard fm.fileExists(atPath: finalURL.path) else { + try fm.moveItem(at: tempURL, to: finalURL) + return + } + guard force else { + throw CreateBundleError.invalid("output directory appeared during install: \(finalURL.path)") + } + + let replacementURL = finalURL.deletingLastPathComponent() + .appendingPathComponent(".\(finalURL.lastPathComponent).replaced-\(UUID().uuidString)") + try fm.moveItem(at: finalURL, to: replacementURL) + do { + try fm.moveItem(at: tempURL, to: finalURL) + try? fm.removeItem(at: replacementURL) + } catch { + try? fm.moveItem(at: replacementURL, to: finalURL) + throw error + } +} + +private func createRawDisk(at url: URL, sizeGiB: UInt64) throws { + guard sizeGiB <= UInt64.max / 1024 / 1024 / 1024 else { + throw CreateBundleError.invalid("disk size is too large") + } + guard FileManager.default.createFile(atPath: url.path, contents: nil) else { + throw CreateBundleError.io("failed to create disk image: \(url.path)") + } + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.truncate(atOffset: sizeGiB * 1024 * 1024 * 1024) +} + +private func loadRestoreImage(from ipswURL: URL) throws -> VZMacOSRestoreImage { + let sem = DispatchSemaphore(value: 0) + var loadedImage: VZMacOSRestoreImage? + var loadError: Error? + + VZMacOSRestoreImage.load(from: ipswURL) { result in + switch result { + case .success(let image): + loadedImage = image + case .failure(let error): + loadError = error + } + sem.signal() + } + sem.wait() + + if let loadError { + throw CreateBundleError.vm("failed to load IPSW: \(loadError)") + } + guard let loadedImage else { + throw CreateBundleError.vm("failed to load IPSW") + } + if #available(macOS 13.0, *), !loadedImage.isSupported { + throw CreateBundleError.vm("IPSW is not supported on this host") + } + return loadedImage +} + +private func formatVersion(_ version: OperatingSystemVersion) -> String { + if version.patchVersion == 0 { + return "\(version.majorVersion).\(version.minorVersion)" + } + return "\(version.majorVersion).\(version.minorVersion).\(version.patchVersion)" +} + +private func buildConfiguration( + paths: BundlePaths, + requirements: VZMacOSConfigurationRequirements, + machineIdentifier: VZMacMachineIdentifier, + vcpus: Int, + memoryMiB: UInt64, + displayWidthPx: Int, + displayHeightPx: Int, + displayPixelsPerInch: Int +) throws -> VZVirtualMachineConfiguration { + let config = VZVirtualMachineConfiguration() + config.bootLoader = VZMacOSBootLoader() + config.cpuCount = vcpus + config.memorySize = memoryMiB * 1024 * 1024 + + let platform = VZMacPlatformConfiguration() + platform.hardwareModel = requirements.hardwareModel + platform.machineIdentifier = machineIdentifier + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: paths.auxiliaryStorageURL) + config.platform = platform + + let graphics = VZMacGraphicsDeviceConfiguration() + graphics.displays = [ + VZMacGraphicsDisplayConfiguration( + widthInPixels: displayWidthPx, + heightInPixels: displayHeightPx, + pixelsPerInch: displayPixelsPerInch + ) + ] + config.graphicsDevices = [graphics] + config.keyboards = [VZUSBKeyboardConfiguration()] + config.pointingDevices = [VZUSBScreenCoordinatePointingDeviceConfiguration()] + + let disk = try VZDiskImageStorageDeviceAttachment( + url: paths.diskURL, + readOnly: false, + cachingMode: .automatic, + synchronizationMode: .full + ) + config.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)] + config.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + config.socketDevices = [VZVirtioSocketDeviceConfiguration()] + + try config.validate() + return config +} + +private func installMacOS(vm: VZVirtualMachine, queue: DispatchQueue, ipswURL: URL) throws { + let sem = DispatchSemaphore(value: 0) + var installError: Error? + var reporter: ProgressReporter? + + queue.async { + let installer = VZMacOSInstaller(virtualMachine: vm, restoringFromImageAt: ipswURL) + reporter = ProgressReporter(progress: installer.progress) + reporter?.start() + installer.install { result in + reporter?.stop() + if case .failure(let error) = result { + installError = error + } + sem.signal() + } + } + + sem.wait() + if let installError { + throw CreateBundleError.vm("macOS install failed: \(installError)") + } +} + +private func writeManifest( + paths: BundlePaths, + image: VZMacOSRestoreImage, + vcpus: Int, + memoryMiB: UInt64, + agentPort: UInt32, + agentVersion: String, + displayWidthPx: Int, + displayHeightPx: Int, + displayPixelsPerInch: Int +) throws { + let manifest = BundleManifest( + schemaVersion: 1, + os: "macos", + arch: "arm64", + macOSVersion: formatVersion(image.operatingSystemVersion), + macOSBuild: image.buildVersion, + vcpus: vcpus, + memoryMiB: memoryMiB, + disk: paths.diskURL.lastPathComponent, + auxiliaryStorage: paths.auxiliaryStorageURL.lastPathComponent, + hardwareModel: paths.hardwareModelURL.lastPathComponent, + machineIdentifier: paths.machineIdentifierURL.lastPathComponent, + agent: AgentManifest(transport: "virtio_socket", port: agentPort, version: agentVersion), + display: DisplayManifest(widthPx: displayWidthPx, heightPx: displayHeightPx, pixelsPerInch: displayPixelsPerInch) + ) + + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + let encoded = try encoder.encode(manifest) + try (encoded + Data([0x0A])).write(to: paths.manifestURL) +} + +private func run() throws { + let opts = try parseOptions(Array(CommandLine.arguments.dropFirst())) + guard VZVirtualMachine.isSupported else { + throw CreateBundleError.vm("Virtualization.framework is not supported on this host") + } + + let ipswURL = absoluteURL(opts.ipswPath) + try requireFile(ipswURL, label: "IPSW") + let outputURL = absoluteURL(opts.outputPath) + let paths = try prepareOutputDirectory(outputURL, force: opts.force) + var committed = false + defer { + if !committed { + try? FileManager.default.removeItem(at: paths.directoryURL) + } + } + + let image = try loadRestoreImage(from: ipswURL) + guard let requirements = image.mostFeaturefulSupportedConfiguration else { + throw CreateBundleError.vm("IPSW has no configuration supported by this host") + } + guard requirements.hardwareModel.isSupported else { + throw CreateBundleError.vm("IPSW hardware model is not supported by this host") + } + + let minimumVCPUs = Int(requirements.minimumSupportedCPUCount) + let minimumMemoryMiB = (requirements.minimumSupportedMemorySize + 1024 * 1024 - 1) / 1024 / 1024 + let vcpus = opts.vcpus ?? max(4, minimumVCPUs) + let memoryMiB = opts.memoryMiB ?? max(8192, minimumMemoryMiB) + guard vcpus >= minimumVCPUs else { + throw CreateBundleError.invalid("--vcpus \(vcpus) is below restore image minimum \(minimumVCPUs)") + } + guard memoryMiB <= UInt64.max / 1024 / 1024 else { + throw CreateBundleError.invalid("--memory-mib is too large") + } + guard memoryMiB >= minimumMemoryMiB else { + throw CreateBundleError.invalid("--memory-mib \(memoryMiB) is below restore image minimum \(minimumMemoryMiB)") + } + + fputs("creating macOS bundle in \(outputURL.path)\n", stderr) + fputs("restore image: macOS \(formatVersion(image.operatingSystemVersion)) build \(image.buildVersion)\n", stderr) + fputs("guest resources: \(vcpus) vCPUs, \(memoryMiB) MiB memory, \(opts.diskSizeGiB) GiB disk\n", stderr) + + _ = try VZMacAuxiliaryStorage(creatingStorageAt: paths.auxiliaryStorageURL, hardwareModel: requirements.hardwareModel) + try createRawDisk(at: paths.diskURL, sizeGiB: opts.diskSizeGiB) + let machineIdentifier = VZMacMachineIdentifier() + try requirements.hardwareModel.dataRepresentation.write(to: paths.hardwareModelURL) + try machineIdentifier.dataRepresentation.write(to: paths.machineIdentifierURL) + + let queue = DispatchQueue(label: "cleanroom.benchmark.darwin-vz-macos-bundle.install") + let config = try buildConfiguration( + paths: paths, + requirements: requirements, + machineIdentifier: machineIdentifier, + vcpus: vcpus, + memoryMiB: memoryMiB, + displayWidthPx: opts.displayWidthPx, + displayHeightPx: opts.displayHeightPx, + displayPixelsPerInch: opts.displayPixelsPerInch + ) + let vm = VZVirtualMachine(configuration: config, queue: queue) + try installMacOS(vm: vm, queue: queue, ipswURL: ipswURL) + try writeManifest( + paths: paths, + image: image, + vcpus: vcpus, + memoryMiB: memoryMiB, + agentPort: opts.agentPort, + agentVersion: opts.agentVersion, + displayWidthPx: opts.displayWidthPx, + displayHeightPx: opts.displayHeightPx, + displayPixelsPerInch: opts.displayPixelsPerInch + ) + try commitOutputDirectory(tempURL: paths.directoryURL, finalURL: outputURL, force: opts.force) + committed = true + + fputs("wrote bundle: \(outputURL.appendingPathComponent("bundle.json").path)\n", stderr) + fputs("note: install the Cleanroom macOS guest agent before running darwin-vz-macos-minimal against this bundle\n", stderr) +} + +do { + try run() +} catch CreateBundleError.usage(let message) { + fputs(message + "\n", stderr) + Foundation.exit(message == usage() ? 0 : 2) +} catch { + fputs("create-bundle: \(error.localizedDescription)\n", stderr) + Foundation.exit(1) +} diff --git a/benchmarks/darwin-vz/macos-minimal/example-bundle.json b/benchmarks/darwin-vz/macos-minimal/example-bundle.json new file mode 100644 index 00000000..399b24c6 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/example-bundle.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "os": "macos", + "arch": "arm64", + "macos_version": "15.5", + "macos_build": "24F74", + "vcpus": 4, + "memory_mib": 8192, + "disk": "disk.img", + "auxiliary_storage": "auxiliary.storage", + "hardware_model": "hardware-model.bin", + "machine_identifier": "machine-identifier.bin", + "agent": { + "transport": "virtio_socket", + "port": 10700, + "version": "0.1.0" + }, + "display": { + "width_px": 1024, + "height_px": 768, + "pixels_per_inch": 72 + } +} diff --git a/benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh b/benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh new file mode 100755 index 00000000..3cc93e6c --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh @@ -0,0 +1,694 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +BASE="" +OUT="" +FORCE=0 +AGENT_BIN="${ROOT_DIR}/dist/cleanroom-macos-guest-agent" +AGENT_VERSION="0.1.0" +AGENT_USER="cleanroom" +AGENT_PASSWORD="cleanroom" +AGENT_UID="$(id -u)" +AGENT_GID="$(id -g)" +PROFILE="headless" +USER_AGENT_PORT="" +RUNNER="${ROOT_DIR}/dist/darwin-vz-macos-minimal" +TIMEOUT=240 +METRICS_DIR="" +KEEP_BOOTSTRAP=0 +LABEL="com.buildkite.cleanroom.macos-guest-agent" +FINALIZED_MARKER="/private/var/db/cleanroom-macos-guest-agent.finalized" + +usage() { + cat <<'EOF' +Usage: + finalize-agent-bundle --base --out [options] + +Options: + --agent-bin Agent binary to install. Default: dist/cleanroom-macos-guest-agent. + --agent-version Version to write to bundle.json. Default: 0.1.0. + --agent-user Temporary bootstrap user. Default: cleanroom. + --agent-uid UID for the temporary bootstrap user. Default: current host UID. + --agent-gid GID for the temporary bootstrap user. Default: current host GID. + --profile Image profile to finalize: headless or gui. Default: headless. + --user-agent-port User LaunchAgent vsock port for --profile gui. Default: agent port + 1. + --runner macOS runner binary. Default: dist/darwin-vz-macos-minimal. + --timeout Timeout for each VM boot/exec. Default: 240. + --metrics-dir Keep finalize.json, smoke.json, and optional gui-smoke.json metrics in this directory. + --keep-bootstrap Keep the temporary bootstrap bundle after a failure. + --force Replace an existing output directory. + -h, --help Show this help. + +The script clones a base macOS VM bundle, creates a temporary rootless +user-cron bootstrap bundle, boots it once to install the root-owned +LaunchDaemon from inside the guest, then boots it again to prove exec is served +by the LaunchDaemon as root. The gui profile keeps a real autologin user and +adds a user LaunchAgent on a separate vsock port. +EOF +} + +die() { + echo "finalize-agent-bundle: $*" >&2 + exit 1 +} + +redact_finalize_metrics() { + /usr/bin/python3 - "$1" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + metrics = json.load(f) +metrics["command"] = ["/bin/sh", "-lc", ""] +with open(path, "w", encoding="utf-8") as f: + json.dump(metrics, f, indent=2, sort_keys=True) + f.write("\n") +PY +} + +remove_bootstrap_user_offline() ( + set -euo pipefail + + local bundle="$1" + local user="$2" + local profile="$3" + local attached_disk="" + local data_volume="" + + # shellcheck disable=SC2329 # invoked by the EXIT trap in this subshell + cleanup_offline_mount() { + if [[ -n "${data_volume}" ]]; then + diskutil unmount "${data_volume}" >/dev/null 2>&1 || true + fi + if [[ -n "${attached_disk}" ]]; then + hdiutil detach "${attached_disk}" >/dev/null 2>&1 || true + fi + } + trap cleanup_offline_mount EXIT + + local attach_output + attach_output="$(hdiutil attach -nomount "${bundle}/disk.img")" + attached_disk="$(printf '%s\n' "${attach_output}" | awk '/GUID_partition_scheme/ {print $1; exit}')" + [[ -n "${attached_disk}" ]] || die "could not identify attached disk for offline bootstrap cleanup" + + local main_container + main_container="$(diskutil list "${attached_disk}" | awk ' + /Apple_APFS Container disk/ && $0 !~ /ISC|Recovery/ { + for (i = 1; i <= NF; i++) { + if ($i ~ /^disk[0-9]+$/ && $(i - 1) == "Container") { + print $i + exit + } + } + } + ')" + [[ -n "${main_container}" ]] || die "could not identify APFS container for offline bootstrap cleanup" + + data_volume="$(diskutil apfs list "${main_container}" | awk ' + /APFS Volume Disk \(Role\):/ && /\(Data\)/ { + for (i = 1; i <= NF; i++) { + if ($i == "(Role):" && (i + 1) <= NF) { + print $(i + 1) + exit + } + } + } + ')" + [[ -n "${data_volume}" ]] || die "could not identify Data volume for offline bootstrap cleanup" + + diskutil mount "${data_volume}" >/dev/null + local mount_point + mount_point="$(diskutil info "${data_volume}" | awk -F: '/Mount Point/ {sub(/^[[:space:]]+/, "", $2); print $2; exit}')" + [[ -n "${mount_point}" && "${mount_point}" != "Not mounted" ]] || die "could not identify Data volume mount point for offline bootstrap cleanup" + + chmod u+rwx \ + "${mount_point}/private/var/db/dslocal/nodes/Default" \ + "${mount_point}/private/var/db/dslocal/nodes/Default/users" \ + "${mount_point}/private/var/db/dslocal/nodes/Default/groups" 2>/dev/null || true + + /usr/bin/python3 - "${mount_point}" "${user}" "${profile}" <<'PY' +import plistlib +import shutil +import sys +from pathlib import Path + +mount = Path(sys.argv[1]) +user = sys.argv[2] +profile = sys.argv[3] +remove_user = profile == "headless" +node = mount / "private/var/db/dslocal/nodes/Default" +user_path = node / "users" / f"{user}.plist" +guid = None + +if remove_user and user_path.exists(): + with user_path.open("rb") as f: + record = plistlib.load(f) + values = record.get("generateduid") or [] + if values: + guid = values[0] + user_path.unlink() + +groups_dir = node / "groups" +if remove_user and groups_dir.is_dir(): + for group_path in groups_dir.glob("*.plist"): + with group_path.open("rb") as f: + group = plistlib.load(f) + changed = False + users = [value for value in group.get("users", []) if value != user] + if users != group.get("users", []): + group["users"] = users + changed = True + if guid: + members = [value for value in group.get("groupmembers", []) if value != guid] + if members != group.get("groupmembers", []): + group["groupmembers"] = members + changed = True + if changed: + with group_path.open("wb") as f: + plistlib.dump(group, f, fmt=plistlib.FMT_BINARY) + +loginwindow = mount / "Library/Preferences/com.apple.loginwindow.plist" +if remove_user and loginwindow.exists(): + with loginwindow.open("rb") as f: + values = plistlib.load(f) + for key in ["autoLoginUser", "lastUserName"]: + if values.get(key) == user: + values.pop(key, None) + if values.get("RecentUsers"): + values["RecentUsers"] = [value for value in values["RecentUsers"] if value != user] + with loginwindow.open("wb") as f: + plistlib.dump(values, f, fmt=plistlib.FMT_BINARY) + +paths = [f"private/var/at/tabs/{user}"] +if remove_user: + paths.extend([ + f"Users/{user}", + "private/etc/kcpassword", + ]) + +for rel in paths: + path = mount / rel + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() +PY + + diskutil unmount "${data_volume}" >/dev/null + data_volume="" + hdiutil detach "${attached_disk}" >/dev/null + attached_disk="" + + if [[ "${profile}" == "headless" ]]; then + echo "removed offline bootstrap user: ${user}" + else + echo "removed offline bootstrap cron: ${user}" + fi +) + +write_bundle_profile_metadata() { + /usr/bin/python3 - "${TMP_BOOTSTRAP}/bundle.json" "${PROFILE}" "${AGENT_VERSION}" "${USER_AGENT_PORT}" "${AGENT_USER}" <<'PY' +import json +import sys + +path = sys.argv[1] +profile = sys.argv[2] +version = sys.argv[3] +user_agent_port = int(sys.argv[4]) +user = sys.argv[5] + +with open(path, "r", encoding="utf-8") as f: + manifest = json.load(f) + +manifest["image_profile"] = profile +if profile == "gui": + manifest["user_agent"] = { + "transport": "virtio_socket", + "port": user_agent_port, + "version": version, + "user": user, + } +else: + manifest.pop("user_agent", None) + +with open(path, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2, sort_keys=True) + f.write("\n") +PY +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --base) + [[ $# -ge 2 ]] || die "missing value for --base" + BASE="$2" + shift 2 + ;; + --out) + [[ $# -ge 2 ]] || die "missing value for --out" + OUT="$2" + shift 2 + ;; + --agent-bin) + [[ $# -ge 2 ]] || die "missing value for --agent-bin" + AGENT_BIN="$2" + shift 2 + ;; + --agent-version) + [[ $# -ge 2 ]] || die "missing value for --agent-version" + AGENT_VERSION="$2" + shift 2 + ;; + --agent-user) + [[ $# -ge 2 ]] || die "missing value for --agent-user" + AGENT_USER="$2" + shift 2 + ;; + --agent-uid) + [[ $# -ge 2 ]] || die "missing value for --agent-uid" + AGENT_UID="$2" + shift 2 + ;; + --agent-gid) + [[ $# -ge 2 ]] || die "missing value for --agent-gid" + AGENT_GID="$2" + shift 2 + ;; + --profile) + [[ $# -ge 2 ]] || die "missing value for --profile" + PROFILE="$2" + shift 2 + ;; + --user-agent-port) + [[ $# -ge 2 ]] || die "missing value for --user-agent-port" + USER_AGENT_PORT="$2" + shift 2 + ;; + --runner) + [[ $# -ge 2 ]] || die "missing value for --runner" + RUNNER="$2" + shift 2 + ;; + --timeout) + [[ $# -ge 2 ]] || die "missing value for --timeout" + TIMEOUT="$2" + shift 2 + ;; + --metrics-dir) + [[ $# -ge 2 ]] || die "missing value for --metrics-dir" + METRICS_DIR="$2" + shift 2 + ;; + --keep-bootstrap) + KEEP_BOOTSTRAP=1 + shift + ;; + --force) + FORCE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "${BASE}" ]] || die "missing --base" +[[ -n "${OUT}" ]] || die "missing --out" +[[ -d "${BASE}" ]] || die "base bundle does not exist: ${BASE}" +[[ -f "${BASE}/bundle.json" ]] || die "base bundle missing bundle.json: ${BASE}" +[[ -n "${AGENT_VERSION}" ]] || die "--agent-version must not be empty" +[[ "${AGENT_UID}" =~ ^[0-9]+$ ]] || die "--agent-uid must be numeric" +[[ "${AGENT_GID}" =~ ^[0-9]+$ ]] || die "--agent-gid must be numeric" +[[ "${AGENT_UID}" -ge 501 ]] || die "--agent-uid must be 501 or greater" +[[ "${AGENT_USER}" =~ ^[A-Za-z_][A-Za-z0-9_-]*$ ]] || die "--agent-user must be a simple local account name" +case "${PROFILE}" in + headless|gui) + ;; + *) + die "--profile must be headless or gui" + ;; +esac +case "${TIMEOUT}" in + ''|*[!0-9]*) + die "--timeout must be a positive integer" + ;; +esac +[[ "${TIMEOUT}" -gt 0 ]] || die "--timeout must be a positive integer" + +if [[ -e "${OUT}" && "${FORCE}" -ne 1 ]]; then + die "output exists; use --force to replace: ${OUT}" +fi + +if [[ ! -x "${AGENT_BIN}" ]]; then + "${ROOT_DIR}/benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh" "${AGENT_BIN}" >/dev/null +fi +[[ -x "${AGENT_BIN}" ]] || die "agent binary is not executable: ${AGENT_BIN}" + +if [[ ! -x "${RUNNER}" ]]; then + "${ROOT_DIR}/benchmarks/darwin-vz/macos-minimal/build-runner.sh" "${RUNNER}" >/dev/null +fi +[[ -x "${RUNNER}" ]] || die "runner binary is not executable: ${RUNNER}" + +OUT_PARENT="$(dirname "${OUT}")" +mkdir -p "${OUT_PARENT}" +TMP_BOOTSTRAP="${OUT_PARENT}/.$(basename "${OUT}").bootstrap.$$" +TMP_METRICS="" +if [[ -n "${METRICS_DIR}" ]]; then + mkdir -p "${METRICS_DIR}" + FINALIZE_METRICS="${METRICS_DIR}/finalize.json" + SMOKE_METRICS="${METRICS_DIR}/smoke.json" + GUI_SMOKE_METRICS="${METRICS_DIR}/gui-smoke.json" +else + TMP_METRICS="$(mktemp -d "${TMPDIR:-/tmp}/cleanroom-macos-finalize-metrics.XXXXXX")" + FINALIZE_METRICS="${TMP_METRICS}/finalize.json" + SMOKE_METRICS="${TMP_METRICS}/smoke.json" + GUI_SMOKE_METRICS="${TMP_METRICS}/gui-smoke.json" +fi + +cleanup() { + if [[ -n "${TMP_METRICS}" && -d "${TMP_METRICS}" ]]; then + rm -rf "${TMP_METRICS}" + fi + if [[ "${KEEP_BOOTSTRAP}" -ne 1 && -d "${TMP_BOOTSTRAP}" ]]; then + rm -rf "${TMP_BOOTSTRAP}" + fi +} +trap cleanup EXIT + +rm -rf "${TMP_BOOTSTRAP}" + +PREPARE_ARGS=( + --base "${BASE}" \ + --out "${TMP_BOOTSTRAP}" \ + --agent-bin "${AGENT_BIN}" \ + --agent-version "${AGENT_VERSION}" \ + --install-mode user-cron \ + --agent-user "${AGENT_USER}" \ + --agent-password "${AGENT_PASSWORD}" \ + --agent-uid "${AGENT_UID}" \ + --agent-gid "${AGENT_GID}" \ + --force +) +if [[ "${PROFILE}" == "headless" ]]; then + PREPARE_ARGS+=(--user-cron-no-autologin) +fi + +"${ROOT_DIR}/benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh" "${PREPARE_ARGS[@]}" + +AGENT_PORT="$( + /usr/bin/python3 - "${TMP_BOOTSTRAP}/bundle.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as f: + manifest = json.load(f) +print(manifest.get("agent", {}).get("port", 10700)) +PY +)" +[[ "${AGENT_PORT}" =~ ^[0-9]+$ && "${AGENT_PORT}" -gt 0 ]] || die "bundle agent port is invalid: ${AGENT_PORT}" +if [[ -z "${USER_AGENT_PORT}" ]]; then + USER_AGENT_PORT="$((AGENT_PORT + 1))" +fi +case "${USER_AGENT_PORT}" in + ''|*[!0-9]*) + die "--user-agent-port must be a positive integer" + ;; +esac +[[ "${USER_AGENT_PORT}" -gt 0 ]] || die "--user-agent-port must be a positive integer" +[[ "${USER_AGENT_PORT}" != "${AGENT_PORT}" ]] || die "--user-agent-port must differ from the root agent port" + +PASSWORD_HEX="$(printf '%s' "${AGENT_PASSWORD}" | /usr/bin/xxd -p -c 256)" + +FINALIZER_SCRIPT="$(cat <<'EOF_FINALIZER' +set -eu +label="__LABEL__" +bootstrap_user="__AGENT_USER__" +password_hex="__PASSWORD_HEX__" +password="$(printf '%s' "${password_hex}" | /usr/bin/xxd -r -p)" +plist="/tmp/${label}.plist" +root_script="/tmp/${label}.finalize-root.$$" + +cat > "${plist}" <<'PLIST' + + + + + Label + __LABEL__ + ProgramArguments + + /usr/local/bin/cleanroom-macos-guest-agent + + EnvironmentVariables + + CLEANROOM_VSOCK_PORT + __AGENT_PORT__ + + RunAtLoad + + KeepAlive + + StandardOutPath + /var/log/cleanroom-macos-guest-agent.log + StandardErrorPath + /var/log/cleanroom-macos-guest-agent.err.log + + +PLIST + +cat > "${root_script}" <<'ROOT_SCRIPT' +#!/bin/sh +set -eu + +label="__LABEL__" +bootstrap_user="__AGENT_USER__" +profile="__PROFILE__" +user_agent_port="__USER_AGENT_PORT__" +marker="__FINALIZED_MARKER__" +agent_src="/Users/${bootstrap_user}/bin/cleanroom-macos-guest-agent" +agent_dst="/usr/local/bin/cleanroom-macos-guest-agent" +plist_src="/tmp/${label}.plist" +plist_dst="/Library/LaunchDaemons/${label}.plist" +bootstrap_agent="/Users/${bootstrap_user}/Library/LaunchAgents/${label}.plist" +bootstrap_cron="/private/var/at/tabs/${bootstrap_user}" + +test -x "${agent_src}" +test -f "${plist_src}" +/usr/bin/install -d -o root -g wheel -m 0755 /usr/local/bin /Library/LaunchDaemons /var/log +/bin/mkdir -p "$(/usr/bin/dirname "${marker}")" +/usr/bin/install -o root -g wheel -m 0755 "${agent_src}" "${agent_dst}" +/usr/bin/install -o root -g wheel -m 0644 "${plist_src}" "${plist_dst}" +/usr/bin/xattr -c "${agent_dst}" "${plist_dst}" >/dev/null 2>&1 || true + +if [ -f "${bootstrap_cron}" ] && ! /usr/bin/grep -q "cleanroom-macos-guest-agent.finalized" "${bootstrap_cron}"; then + echo "bootstrap cron is missing finalized marker check: ${bootstrap_cron}" >&2 + exit 1 +fi + +/usr/bin/touch "${marker}" +/usr/sbin/chown root:wheel "${marker}" +/bin/chmod 0644 "${marker}" + +/bin/rm -f "${bootstrap_agent}" >/dev/null 2>&1 || true +if [ "${profile}" = "gui" ]; then + /bin/mkdir -p "$(/usr/bin/dirname "${bootstrap_agent}")" + cat > "${bootstrap_agent}" < + + + + Label + ${label} + ProgramArguments + + ${agent_dst} + + EnvironmentVariables + + CLEANROOM_VSOCK_PORT + ${user_agent_port} + + RunAtLoad + + KeepAlive + + StandardOutPath + /Users/${bootstrap_user}/Library/Logs/cleanroom-macos-guest-agent.log + StandardErrorPath + /Users/${bootstrap_user}/Library/Logs/cleanroom-macos-guest-agent.err.log + + +USER_AGENT_PLIST + /usr/sbin/chown "${bootstrap_user}:staff" "${bootstrap_agent}" >/dev/null 2>&1 || true + /bin/chmod 0644 "${bootstrap_agent}" +else + /usr/bin/defaults delete /Library/Preferences/com.apple.loginwindow autoLoginUser >/dev/null 2>&1 || true + /bin/rm -f /private/etc/kcpassword >/dev/null 2>&1 || true +fi +if /usr/bin/dscl . -read "/Users/${bootstrap_user}" >/dev/null 2>&1; then + /usr/sbin/dseditgroup -o edit -d "${bootstrap_user}" -t user admin >/dev/null 2>&1 || true +fi + +/usr/sbin/chown root:wheel "${agent_dst}" "${plist_dst}" +/bin/chmod 0755 "${agent_dst}" +/bin/chmod 0644 "${plist_dst}" +/usr/bin/stat -f "%Su:%Sg %Lp %N" "${agent_dst}" "${plist_dst}" "${marker}" +/bin/sync +/bin/sleep 5 +ROOT_SCRIPT + +/bin/chmod 0700 "${root_script}" +printf '%s\n' "${password}" | /usr/bin/sudo -S -p '' /bin/sh "${root_script}" +/bin/rm -f "${root_script}" "${plist}" +EOF_FINALIZER +)" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__LABEL__/${LABEL}}" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__AGENT_USER__/${AGENT_USER}}" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__AGENT_PORT__/${AGENT_PORT}}" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__USER_AGENT_PORT__/${USER_AGENT_PORT}}" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__PROFILE__/${PROFILE}}" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__PASSWORD_HEX__/${PASSWORD_HEX}}" +FINALIZER_SCRIPT="${FINALIZER_SCRIPT//__FINALIZED_MARKER__/${FINALIZED_MARKER}}" + +SMOKE_SCRIPT="$(cat <<'EOF_SMOKE' +set -eu +label="__LABEL__" +bootstrap_user="__AGENT_USER__" +profile="__PROFILE__" +user_agent_port="__USER_AGENT_PORT__" +marker="__FINALIZED_MARKER__" +uid="$(id -u)" +user="$(id -un)" +printf "user=%s uid=%s cwd=%s\n" "${user}" "${uid}" "${PWD}" +test "${uid}" = "0" +/bin/launchctl print "system/${label}" >/dev/null +test -f "${marker}" +/usr/bin/stat -f "%Su:%Sg %Lp %N" /usr/local/bin/cleanroom-macos-guest-agent "/Library/LaunchDaemons/${label}.plist" "${marker}" +test ! -e "/private/var/at/tabs/${bootstrap_user}" +echo "bootstrap_cron=absent" +if [ "${profile}" = "gui" ]; then + /usr/bin/id "${bootstrap_user}" >/dev/null + if /usr/bin/dsmemberutil checkmembership -U "${bootstrap_user}" -G admin | /usr/bin/grep -q "is a member"; then + echo "bootstrap_user=admin" >&2 + exit 1 + fi + echo "bootstrap_admin=absent" + test -f "/Users/${bootstrap_user}/Library/LaunchAgents/${label}.plist" + /usr/bin/grep -q "${user_agent_port}" "/Users/${bootstrap_user}/Library/LaunchAgents/${label}.plist" + echo "bootstrap_user=present" +else + if /usr/bin/id "${bootstrap_user}" >/dev/null 2>&1; then + echo "bootstrap_user=present" >&2 + exit 1 + fi + test ! -e "/Users/${bootstrap_user}" + echo "bootstrap_user=absent" +fi +EOF_SMOKE +)" +SMOKE_SCRIPT="${SMOKE_SCRIPT//__LABEL__/${LABEL}}" +SMOKE_SCRIPT="${SMOKE_SCRIPT//__AGENT_USER__/${AGENT_USER}}" +SMOKE_SCRIPT="${SMOKE_SCRIPT//__PROFILE__/${PROFILE}}" +SMOKE_SCRIPT="${SMOKE_SCRIPT//__USER_AGENT_PORT__/${USER_AGENT_PORT}}" +SMOKE_SCRIPT="${SMOKE_SCRIPT//__FINALIZED_MARKER__/${FINALIZED_MARKER}}" + +GUI_SMOKE_SCRIPT="$(cat <<'EOF_GUI_SMOKE' +set -eu +label="__LABEL__" +expected_user="__AGENT_USER__" +uid="$(id -u)" +user="$(id -un)" +screenshot="/tmp/cleanroom-gui-smoke.png" +printf "gui_user=%s uid=%s cwd=%s\n" "${user}" "${uid}" "${PWD}" +test "${user}" = "${expected_user}" +test "${uid}" != "0" +/bin/launchctl print "gui/${uid}/${label}" >/dev/null +/usr/bin/killall TextEdit >/dev/null 2>&1 || true +/bin/rm -f "${screenshot}" +/usr/bin/open -a TextEdit +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do + if /usr/bin/pgrep -x TextEdit >/dev/null 2>&1; then + break + fi + /bin/sleep 1 +done +/usr/bin/pgrep -x TextEdit >/dev/null +if screenshot_output="$(/usr/sbin/screencapture -x "${screenshot}" 2>&1)"; then + test -s "${screenshot}" + /usr/bin/stat -f "gui_screenshot=%z %N" "${screenshot}" +else + echo "gui_screenshot=unavailable" + if [ -n "${screenshot_output}" ]; then + printf "%s\n" "${screenshot_output}" >&2 + fi +fi +/usr/bin/killall TextEdit >/dev/null 2>&1 || true +EOF_GUI_SMOKE +)" +GUI_SMOKE_SCRIPT="${GUI_SMOKE_SCRIPT//__LABEL__/${LABEL}}" +GUI_SMOKE_SCRIPT="${GUI_SMOKE_SCRIPT//__AGENT_USER__/${AGENT_USER}}" + +finalize_status=0 +"${RUNNER}" \ + --bundle "${TMP_BOOTSTRAP}" \ + --timeout "${TIMEOUT}" \ + --metrics "${FINALIZE_METRICS}" \ + -- /bin/sh -lc "${FINALIZER_SCRIPT}" || finalize_status=$? +if [[ -f "${FINALIZE_METRICS}" ]]; then + redact_finalize_metrics "${FINALIZE_METRICS}" +fi +if [[ "${finalize_status}" -ne 0 ]]; then + exit "${finalize_status}" +fi +remove_bootstrap_user_offline "${TMP_BOOTSTRAP}" "${AGENT_USER}" "${PROFILE}" +write_bundle_profile_metadata + +"${RUNNER}" \ + --bundle "${TMP_BOOTSTRAP}" \ + --timeout "${TIMEOUT}" \ + --metrics "${SMOKE_METRICS}" \ + -- /bin/sh -lc "${SMOKE_SCRIPT}" + +if [[ "${PROFILE}" == "gui" ]]; then + "${RUNNER}" \ + --bundle "${TMP_BOOTSTRAP}" \ + --agent user \ + --timeout "${TIMEOUT}" \ + --metrics "${GUI_SMOKE_METRICS}" \ + -- /bin/sh -lc "${GUI_SMOKE_SCRIPT}" +fi + +if [[ -e "${OUT}" ]]; then + if [[ "${FORCE}" -ne 1 ]]; then + die "output appeared during finalization; use --force to replace: ${OUT}" + fi + rm -rf "${OUT}" +fi +mv "${TMP_BOOTSTRAP}" "${OUT}" +TMP_BOOTSTRAP="" + +echo "finalized bundle: ${OUT}" +SUMMARY_METRICS=("${FINALIZE_METRICS}" "${SMOKE_METRICS}") +if [[ "${PROFILE}" == "gui" ]]; then + SUMMARY_METRICS+=("${GUI_SMOKE_METRICS}") +fi +/usr/bin/python3 - "${SUMMARY_METRICS[@]}" <<'PY' +import json +import sys + +for label, path in zip(["finalize", "smoke", "gui_smoke"], sys.argv[1:]): + with open(path, "r", encoding="utf-8") as f: + metrics = json.load(f) + print( + f"{label}: exit_code={metrics.get('exit_code')} " + f"vsock_connect_ms={metrics.get('vsock_connect_ms')} " + f"exec_response_ms={metrics.get('exec_response_ms')}" + ) +PY diff --git a/benchmarks/darwin-vz/macos-minimal/helper-runner.go b/benchmarks/darwin-vz/macos-minimal/helper-runner.go new file mode 100644 index 00000000..b00fe6d6 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/helper-runner.go @@ -0,0 +1,671 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/buildkite/cleanroom/internal/vsockexec" +) + +const helperBinaryName = "cleanroom-darwin-vz" + +type options struct { + bundlePath string + helperPath string + agentName string + metricsPath string + runDir string + timeout time.Duration + validateOnly bool + command []string +} + +type bundleManifest struct { + SchemaVersion int `json:"schema_version"` + OS string `json:"os"` + Arch string `json:"arch"` + MacOSVersion string `json:"macos_version,omitempty"` + MacOSBuild string `json:"macos_build,omitempty"` + VCPUs int64 `json:"vcpus"` + MemoryMiB int64 `json:"memory_mib"` + Disk string `json:"disk"` + AuxiliaryStorage string `json:"auxiliary_storage"` + HardwareModel string `json:"hardware_model"` + MachineIdentifier string `json:"machine_identifier"` + Agent agentManifest `json:"agent"` + UserAgent *agentManifest `json:"user_agent,omitempty"` + Display displayManifest `json:"display,omitempty"` +} + +type agentManifest struct { + Transport string `json:"transport"` + Port uint32 `json:"port"` + Version string `json:"version"` + User string `json:"user,omitempty"` +} + +type displayManifest struct { + WidthPx int64 `json:"width_px,omitempty"` + HeightPx int64 `json:"height_px,omitempty"` + PixelsPerInch int64 `json:"pixels_per_inch,omitempty"` +} + +type resolvedBundle struct { + ManifestURL string + Manifest bundleManifest + DiskPath string + AuxiliaryStoragePath string + HardwareModelPath string + MachineIdentifierPath string + SelectedAgent agentManifest + SelectedAgentName string +} + +type controlRequest struct { + Op string `json:"op"` + DiskPath string `json:"disk_path,omitempty"` + AuxiliaryStoragePath string `json:"auxiliary_storage_path,omitempty"` + HardwareModelPath string `json:"hardware_model_path,omitempty"` + MachineIdentifierPath string `json:"machine_identifier_path,omitempty"` + NetworkMode string `json:"network_mode,omitempty"` + VCPUs int64 `json:"vcpus,omitempty"` + MemoryMiB int64 `json:"memory_mib,omitempty"` + GuestPort uint32 `json:"guest_port,omitempty"` + LaunchSeconds int64 `json:"launch_seconds,omitempty"` + RunDir string `json:"run_dir,omitempty"` + ProxySocketPath string `json:"proxy_socket_path,omitempty"` + VMID string `json:"vm_id,omitempty"` + DisplayWidthPx int64 `json:"display_width_px,omitempty"` + DisplayHeightPx int64 `json:"display_height_px,omitempty"` + DisplayPixelsPerInch int64 `json:"display_pixels_per_inch,omitempty"` +} + +type controlResponse struct { + OK bool `json:"ok"` + Error string `json:"error,omitempty"` + VMID string `json:"vm_id,omitempty"` + ProxySocketPath string `json:"proxy_socket_path,omitempty"` + TimingMS map[string]int64 `json:"timing_ms,omitempty"` +} + +type smokeResult struct { + Bundle string `json:"bundle"` + Command []string `json:"command"` + Helper string `json:"helper,omitempty"` + StartedVM bool `json:"started_vm"` + StartMS int64 `json:"start_ms,omitempty"` + ProxyConnectMS int64 `json:"proxy_connect_ms,omitempty"` + ExecResponseMS int64 `json:"exec_response_ms,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + Error string `json:"error,omitempty"` + SelectedAgent string `json:"selected_agent"` + MacOSVersion string `json:"macos_version,omitempty"` + MacOSBuild string `json:"macos_build,omitempty"` + AgentVersion string `json:"agent_version"` + VCPUs int64 `json:"vcpus"` + MemoryMiB int64 `json:"memory_mib"` + VMID string `json:"vm_id,omitempty"` + HelperTimingMS map[string]int64 `json:"helper_timing_ms,omitempty"` +} + +type helperSession struct { + cmd *exec.Cmd + socketPath string + + stderr bytes.Buffer + done chan error + + conn net.Conn + enc *json.Encoder + dec *json.Decoder + mu sync.Mutex +} + +func main() { + code := 1 + if err := run(os.Args[1:], os.Stdout, os.Stderr, &code); err != nil { + fmt.Fprintf(os.Stderr, "darwin-vz-macos-helper-runner: %v\n", err) + } + os.Exit(code) +} + +func run(args []string, stdout io.Writer, stderr io.Writer, exitCode *int) error { + opts, err := parseOptions(args) + if err != nil { + if errors.Is(err, flag.ErrHelp) { + *exitCode = 0 + return nil + } + *exitCode = 2 + return err + } + + bundle, err := loadBundle(opts.bundlePath, opts.agentName) + if err != nil { + *exitCode = 1 + return err + } + helperPath, err := resolveHelperPath(opts.helperPath) + if err != nil && !opts.validateOnly { + *exitCode = 1 + return err + } + + result := smokeResult{ + Bundle: bundle.ManifestURL, + Command: opts.command, + Helper: helperPath, + SelectedAgent: bundle.SelectedAgentName, + MacOSVersion: bundle.Manifest.MacOSVersion, + MacOSBuild: bundle.Manifest.MacOSBuild, + AgentVersion: bundle.SelectedAgent.Version, + VCPUs: bundle.Manifest.VCPUs, + MemoryMiB: bundle.Manifest.MemoryMiB, + } + defer func() { + if writeErr := writeMetrics(result, opts.metricsPath); writeErr != nil { + fmt.Fprintf(stderr, "write metrics: %v\n", writeErr) + } + }() + + if opts.validateOnly { + *exitCode = 0 + return nil + } + + runDir, cleanup, err := prepareRunDir(opts.runDir) + if err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), opts.timeout) + defer cancel() + + helper, err := startHelper(ctx, helperPath, filepath.Join(runDir, "vz-helper.sock")) + if err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + defer helper.close() + + startedAt := time.Now() + startRes, err := helper.request(ctx, controlRequest{ + Op: "StartMacOSVM", + DiskPath: bundle.DiskPath, + AuxiliaryStoragePath: bundle.AuxiliaryStoragePath, + HardwareModelPath: bundle.HardwareModelPath, + MachineIdentifierPath: bundle.MachineIdentifierPath, + NetworkMode: "none", + VCPUs: bundle.Manifest.VCPUs, + MemoryMiB: bundle.Manifest.MemoryMiB, + GuestPort: bundle.SelectedAgent.Port, + LaunchSeconds: int64(opts.timeout.Seconds()), + RunDir: runDir, + ProxySocketPath: filepath.Join(runDir, "vz-proxy.sock"), + DisplayWidthPx: bundle.Manifest.Display.WidthPx, + DisplayHeightPx: bundle.Manifest.Display.HeightPx, + DisplayPixelsPerInch: bundle.Manifest.Display.PixelsPerInch, + }) + result.StartMS = time.Since(startedAt).Milliseconds() + if err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + result.StartedVM = true + result.VMID = startRes.VMID + result.HelperTimingMS = startRes.TimingMS + defer stopHelperVM(helper, startRes.VMID, stderr) + + proxyPath := strings.TrimSpace(startRes.ProxySocketPath) + if proxyPath == "" { + proxyPath = filepath.Join(runDir, "vz-proxy.sock") + } + proxyConn, err := dialUnix(ctx, proxyPath) + result.ProxyConnectMS = time.Since(startedAt).Milliseconds() + if err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + defer proxyConn.Close() + if deadline, ok := ctx.Deadline(); ok { + if err := proxyConn.SetDeadline(deadline); err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + defer proxyConn.SetDeadline(time.Time{}) + } + + if err := vsockexec.EncodeRequest(proxyConn, vsockexec.ExecRequest{Command: opts.command}); err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + if err := vsockexec.EncodeInputFrame(proxyConn, vsockexec.ExecInputFrame{Type: "eof"}); err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + execRes, err := vsockexec.DecodeStreamResponse(proxyConn, vsockexec.StreamCallbacks{ + OnStdout: func(b []byte) { _, _ = stdout.Write(b) }, + OnStderr: func(b []byte) { _, _ = stderr.Write(b) }, + }) + result.ExecResponseMS = time.Since(startedAt).Milliseconds() + if err != nil { + result.Error = err.Error() + *exitCode = 1 + return err + } + result.ExitCode = &execRes.ExitCode + result.Error = execRes.Error + *exitCode = execRes.ExitCode + return nil +} + +func parseOptions(args []string) (options, error) { + opts := options{ + agentName: "root", + timeout: 120 * time.Second, + command: []string{"/usr/bin/sw_vers"}, + } + fs := flag.NewFlagSet("darwin-vz-macos-helper-runner", flag.ContinueOnError) + fs.StringVar(&opts.bundlePath, "bundle", "", "bundle.json or bundle directory") + fs.StringVar(&opts.helperPath, "helper", "", "cleanroom-darwin-vz binary or .app path") + fs.StringVar(&opts.agentName, "agent", opts.agentName, "agent endpoint: root or user") + fs.StringVar(&opts.metricsPath, "metrics", "", "write result JSON to path; omit to write to stderr") + fs.StringVar(&opts.runDir, "run-dir", "", "helper run directory; defaults to a temporary directory") + fs.BoolVar(&opts.validateOnly, "validate-only", false, "validate bundle metadata without starting the helper") + timeoutSeconds := fs.Int("timeout", int(opts.timeout.Seconds()), "helper, VM, and command timeout in seconds") + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage: %s --bundle [options] [-- [args...]]\n", fs.Name()) + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + return options{}, err + } + if strings.TrimSpace(opts.bundlePath) == "" { + return options{}, errors.New("missing --bundle") + } + if opts.agentName != "root" && opts.agentName != "user" { + return options{}, errors.New("--agent must be root or user") + } + if *timeoutSeconds <= 0 { + return options{}, errors.New("--timeout must be greater than zero") + } + opts.timeout = time.Duration(*timeoutSeconds) * time.Second + if fs.NArg() > 0 { + opts.command = append([]string(nil), fs.Args()...) + } + if len(opts.command) == 0 || strings.TrimSpace(opts.command[0]) == "" { + return options{}, errors.New("command after -- must not be empty") + } + return opts, nil +} + +func loadBundle(path string, agentName string) (resolvedBundle, error) { + manifestPath, err := manifestPath(path) + if err != nil { + return resolvedBundle{}, err + } + data, err := os.ReadFile(manifestPath) + if err != nil { + return resolvedBundle{}, fmt.Errorf("read bundle manifest: %w", err) + } + var manifest bundleManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return resolvedBundle{}, fmt.Errorf("decode bundle manifest: %w", err) + } + if err := validateManifest(manifest); err != nil { + return resolvedBundle{}, err + } + agent, err := selectAgent(manifest, agentName) + if err != nil { + return resolvedBundle{}, err + } + baseDir := filepath.Dir(manifestPath) + resolved := resolvedBundle{ + ManifestURL: manifestPath, + Manifest: manifest, + DiskPath: resolveBundlePath(baseDir, manifest.Disk), + AuxiliaryStoragePath: resolveBundlePath(baseDir, manifest.AuxiliaryStorage), + HardwareModelPath: resolveBundlePath(baseDir, manifest.HardwareModel), + MachineIdentifierPath: resolveBundlePath(baseDir, manifest.MachineIdentifier), + SelectedAgent: agent, + SelectedAgentName: agentName, + } + for field, path := range map[string]string{ + "disk": resolved.DiskPath, + "auxiliary_storage": resolved.AuxiliaryStoragePath, + "hardware_model": resolved.HardwareModelPath, + "machine_identifier": resolved.MachineIdentifierPath, + } { + if err := requireFile(path); err != nil { + return resolvedBundle{}, fmt.Errorf("%s: %w", field, err) + } + } + return resolved, nil +} + +func manifestPath(path string) (string, error) { + expanded := expandPath(path) + info, err := os.Stat(expanded) + if err != nil { + return "", fmt.Errorf("bundle path: %w", err) + } + if info.IsDir() { + expanded = filepath.Join(expanded, "bundle.json") + } + if err := requireFile(expanded); err != nil { + return "", fmt.Errorf("bundle manifest: %w", err) + } + return filepath.Abs(expanded) +} + +func validateManifest(manifest bundleManifest) error { + if manifest.SchemaVersion != 1 { + return fmt.Errorf("unsupported schema_version %d", manifest.SchemaVersion) + } + if manifest.OS != "macos" { + return errors.New("bundle os must be macos") + } + if manifest.Arch != "arm64" { + return errors.New("bundle arch must be arm64") + } + if manifest.VCPUs <= 0 { + return errors.New("vcpus must be greater than zero") + } + if manifest.MemoryMiB < 1024 { + return errors.New("memory_mib must be at least 1024") + } + if err := validateAgent(manifest.Agent, "agent"); err != nil { + return err + } + if manifest.UserAgent != nil { + if err := validateAgent(*manifest.UserAgent, "user_agent"); err != nil { + return err + } + if manifest.UserAgent.Port == manifest.Agent.Port { + return errors.New("user_agent.port must differ from agent.port") + } + } + if manifest.Display.WidthPx < 0 || manifest.Display.HeightPx < 0 || manifest.Display.PixelsPerInch < 0 { + return errors.New("display dimensions must not be negative") + } + return nil +} + +func validateAgent(agent agentManifest, field string) error { + if agent.Transport != "virtio_socket" { + return fmt.Errorf("%s.transport must be virtio_socket", field) + } + if agent.Port == 0 { + return fmt.Errorf("%s.port must be greater than zero", field) + } + if strings.TrimSpace(agent.Version) == "" { + return fmt.Errorf("%s.version must not be empty", field) + } + return nil +} + +func selectAgent(manifest bundleManifest, name string) (agentManifest, error) { + switch name { + case "root": + return manifest.Agent, nil + case "user": + if manifest.UserAgent == nil { + return agentManifest{}, errors.New("bundle does not declare user_agent") + } + return *manifest.UserAgent, nil + default: + return agentManifest{}, errors.New("--agent must be root or user") + } +} + +func resolveBundlePath(baseDir string, path string) string { + expanded := expandPath(path) + if filepath.IsAbs(expanded) { + return expanded + } + return filepath.Join(baseDir, expanded) +} + +func resolveHelperPath(path string) (string, error) { + candidates := []string{} + if strings.TrimSpace(path) != "" { + candidates = append(candidates, path) + } + if env := strings.TrimSpace(os.Getenv("CLEANROOM_DARWIN_VZ_HELPER")); env != "" { + candidates = append(candidates, env) + } + candidates = append(candidates, + filepath.Join("dist", helperBinaryName+".app"), + filepath.Join("dist", helperBinaryName+"-test.app"), + filepath.Join("dist", helperBinaryName), + ) + if found, err := exec.LookPath(helperBinaryName); err == nil { + candidates = append(candidates, found) + } + for _, candidate := range candidates { + resolved := helperExecutablePath(expandPath(candidate)) + if err := requireExecutable(resolved); err == nil { + return filepath.Abs(resolved) + } + } + return "", errors.New("cleanroom-darwin-vz helper not found; pass --helper or set CLEANROOM_DARWIN_VZ_HELPER") +} + +func helperExecutablePath(path string) string { + if strings.HasSuffix(path, ".app") { + return filepath.Join(path, "Contents", "MacOS", helperBinaryName) + } + return path +} + +func prepareRunDir(path string) (string, func(), error) { + if strings.TrimSpace(path) == "" { + dir, err := os.MkdirTemp("/tmp", "crmhr-*") + if err != nil { + return "", func() {}, err + } + return dir, func() { _ = os.RemoveAll(dir) }, nil + } + dir, err := filepath.Abs(expandPath(path)) + if err != nil { + return "", func() {}, err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", func() {}, err + } + return dir, func() {}, nil +} + +func startHelper(ctx context.Context, helperPath string, socketPath string) (*helperSession, error) { + _ = os.Remove(socketPath) + cmd := exec.Command(helperPath, "--socket", socketPath) + session := &helperSession{ + cmd: cmd, + socketPath: socketPath, + done: make(chan error, 1), + } + cmd.Stderr = &session.stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start helper: %w", err) + } + go func() { + session.done <- cmd.Wait() + close(session.done) + }() + conn, err := waitForHelper(ctx, socketPath, session.done) + if err != nil { + _ = session.close() + return nil, fmt.Errorf("connect helper control socket: %w", session.decorateError(err)) + } + session.conn = conn + session.enc = json.NewEncoder(conn) + session.dec = json.NewDecoder(conn) + return session, nil +} + +func waitForHelper(ctx context.Context, socketPath string, helperDone <-chan error) (net.Conn, error) { + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + dialer := net.Dialer{} + for { + conn, err := dialer.DialContext(ctx, "unix", socketPath) + if err == nil { + return conn, nil + } + select { + case doneErr := <-helperDone: + if doneErr == nil { + return nil, errors.New("helper exited before control socket was ready") + } + return nil, fmt.Errorf("helper exited before control socket was ready: %w", doneErr) + case <-ctx.Done(): + return nil, fmt.Errorf("timed out waiting for helper control socket: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func (s *helperSession) request(ctx context.Context, req controlRequest) (controlResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if deadline, ok := ctx.Deadline(); ok { + _ = s.conn.SetDeadline(deadline) + defer s.conn.SetDeadline(time.Time{}) + } + if err := s.enc.Encode(req); err != nil { + return controlResponse{}, s.decorateError(fmt.Errorf("send helper request %q: %w", req.Op, err)) + } + var res controlResponse + if err := s.dec.Decode(&res); err != nil { + return controlResponse{}, s.decorateError(fmt.Errorf("decode helper response %q: %w", req.Op, err)) + } + if !res.OK { + msg := strings.TrimSpace(res.Error) + if msg == "" { + msg = "unknown helper error" + } + return controlResponse{}, s.decorateError(fmt.Errorf("helper %s failed: %s", req.Op, msg)) + } + return res, nil +} + +func (s *helperSession) close() error { + if s == nil { + return nil + } + if s.conn != nil { + _ = s.conn.Close() + } + if s.cmd != nil && s.cmd.Process != nil { + _ = s.cmd.Process.Signal(os.Interrupt) + select { + case <-s.done: + case <-time.After(2 * time.Second): + _ = s.cmd.Process.Kill() + <-s.done + } + } + _ = os.Remove(s.socketPath) + return nil +} + +func (s *helperSession) decorateError(err error) error { + stderr := strings.TrimSpace(s.stderr.String()) + if stderr == "" { + return err + } + return fmt.Errorf("%w; helper stderr: %s", err, stderr) +} + +func stopHelperVM(helper *helperSession, vmID string, stderr io.Writer) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if _, err := helper.request(ctx, controlRequest{Op: "StopVM", VMID: vmID}); err != nil { + fmt.Fprintf(stderr, "stop helper vm: %v\n", err) + } +} + +func dialUnix(ctx context.Context, socketPath string) (net.Conn, error) { + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + dialer := net.Dialer{} + for { + conn, err := dialer.DialContext(ctx, "unix", socketPath) + if err == nil { + return conn, nil + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("timed out waiting for unix socket %q: %w", socketPath, ctx.Err()) + case <-ticker.C: + } + } +} + +func writeMetrics(result smokeResult, path string) error { + encoded, err := json.Marshal(result) + if err != nil { + return err + } + if strings.TrimSpace(path) == "" { + fmt.Fprintln(os.Stderr, string(encoded)) + return nil + } + return os.WriteFile(expandPath(path), append(encoded, '\n'), 0o644) +} + +func requireFile(path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if info.IsDir() { + return fmt.Errorf("%s is a directory", path) + } + return nil +} + +func requireExecutable(path string) error { + if err := requireFile(path); err != nil { + return err + } + info, err := os.Stat(path) + if err != nil { + return err + } + if info.Mode()&0o111 == 0 { + return fmt.Errorf("%s is not executable", path) + } + return nil +} + +func expandPath(path string) string { + if strings.HasPrefix(path, "~/") { + home, err := os.UserHomeDir() + if err == nil { + return filepath.Join(home, strings.TrimPrefix(path, "~/")) + } + } + return path +} diff --git a/benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh b/benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh new file mode 100755 index 00000000..0ebadd10 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh @@ -0,0 +1,517 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +AGENT_BIN="${ROOT_DIR}/dist/cleanroom-macos-guest-agent" +AGENT_VERSION="0.1.0" +BASE="" +OUT="" +FORCE=0 +ALLOW_UNVERIFIED_OWNERSHIP=0 +INSTALL_MODE="launchdaemon" +AGENT_USER="cleanroom" +AGENT_PASSWORD="cleanroom" +AGENT_UID="$(id -u)" +AGENT_GID="$(id -g)" +USER_CRON_AUTOLOGIN=1 + +usage() { + cat <<'EOF' +Usage: + prepare-agent-bundle --base --out [options] + +Options: + --agent-bin Agent binary to install. Default: dist/cleanroom-macos-guest-agent. + --agent-version Version to write to bundle.json. Default: 0.1.0. + --install-mode Agent startup mode: launchdaemon or user-cron. + Default: launchdaemon. + --agent-user User to create for user-cron mode. Default: cleanroom. + --agent-password Password for the user-cron user. Default: cleanroom. + --agent-uid UID for the user-cron user. Default: current host UID. + --agent-gid GID for the user-cron user. Default: current host GID. + --user-cron-no-autologin + Do not configure autologin or a user LaunchAgent + for user-cron mode. + --allow-unverified-ownership + Continue when root ownership cannot be set. The + resulting bundle is for inspection only until a + live smoke proves launchd starts the agent. + --force Replace an existing output directory. + -h, --help Show this help. + +The script clones a local macOS VM bundle, mounts the APFS Data volume from the +cloned disk image, installs the Cleanroom macOS guest agent, updates +bundle.json, and leaves the base bundle untouched. +EOF +} + +die() { + echo "prepare-agent-bundle: $*" >&2 + exit 1 +} + +copy_bundle_file() { + local src="$1" + local dst="$2" + cp -c "${src}" "${dst}" 2>/dev/null || cp "${src}" "${dst}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --base) + [[ $# -ge 2 ]] || die "missing value for --base" + BASE="$2" + shift 2 + ;; + --out) + [[ $# -ge 2 ]] || die "missing value for --out" + OUT="$2" + shift 2 + ;; + --agent-bin) + [[ $# -ge 2 ]] || die "missing value for --agent-bin" + AGENT_BIN="$2" + shift 2 + ;; + --agent-version) + [[ $# -ge 2 ]] || die "missing value for --agent-version" + AGENT_VERSION="$2" + shift 2 + ;; + --install-mode) + [[ $# -ge 2 ]] || die "missing value for --install-mode" + INSTALL_MODE="$2" + shift 2 + ;; + --agent-user) + [[ $# -ge 2 ]] || die "missing value for --agent-user" + AGENT_USER="$2" + shift 2 + ;; + --agent-password) + [[ $# -ge 2 ]] || die "missing value for --agent-password" + AGENT_PASSWORD="$2" + shift 2 + ;; + --agent-uid) + [[ $# -ge 2 ]] || die "missing value for --agent-uid" + AGENT_UID="$2" + shift 2 + ;; + --agent-gid) + [[ $# -ge 2 ]] || die "missing value for --agent-gid" + AGENT_GID="$2" + shift 2 + ;; + --user-cron-no-autologin) + USER_CRON_AUTOLOGIN=0 + shift + ;; + --force) + FORCE=1 + shift + ;; + --allow-unverified-ownership) + ALLOW_UNVERIFIED_OWNERSHIP=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "${BASE}" ]] || die "missing --base" +[[ -n "${OUT}" ]] || die "missing --out" +[[ -d "${BASE}" ]] || die "base bundle does not exist: ${BASE}" +[[ -f "${BASE}/bundle.json" ]] || die "base bundle missing bundle.json: ${BASE}" +case "${INSTALL_MODE}" in + launchdaemon|user-cron) + ;; + *) + die "unsupported --install-mode ${INSTALL_MODE}; expected launchdaemon or user-cron" + ;; +esac +[[ "${AGENT_UID}" =~ ^[0-9]+$ ]] || die "--agent-uid must be numeric" +[[ "${AGENT_GID}" =~ ^[0-9]+$ ]] || die "--agent-gid must be numeric" +[[ -n "${AGENT_USER}" ]] || die "--agent-user must not be empty" +if [[ "${INSTALL_MODE}" == "user-cron" && "${AGENT_UID}" -lt 501 ]]; then + die "--agent-uid must be 501 or greater for user-cron mode" +fi + +if [[ ! -x "${AGENT_BIN}" ]]; then + "${ROOT_DIR}/benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh" "${AGENT_BIN}" >/dev/null +fi + +[[ -x "${AGENT_BIN}" ]] || die "agent binary is not executable: ${AGENT_BIN}" + +if [[ -e "${OUT}" && "${FORCE}" -ne 1 ]]; then + die "output exists; use --force to replace: ${OUT}" +fi + +OUT_PARENT="$(dirname "${OUT}")" +mkdir -p "${OUT_PARENT}" +TMP_OUT="${OUT_PARENT}/.$(basename "${OUT}").tmp.$$" +ATTACHED_DISK="" +DATA_VOLUME="" + +cleanup() { + if [[ -n "${DATA_VOLUME}" ]]; then + diskutil unmount "${DATA_VOLUME}" >/dev/null 2>&1 || true + fi + if [[ -n "${ATTACHED_DISK}" ]]; then + hdiutil detach "${ATTACHED_DISK}" >/dev/null 2>&1 || true + fi + if [[ -d "${TMP_OUT}" ]]; then + rm -rf "${TMP_OUT}" + fi +} +trap cleanup EXIT + +rm -rf "${TMP_OUT}" +mkdir -p "${TMP_OUT}" + +for name in bundle.json disk.img auxiliary.storage hardware-model.bin machine-identifier.bin; do + [[ -f "${BASE}/${name}" ]] || die "base bundle missing ${name}: ${BASE}" + copy_bundle_file "${BASE}/${name}" "${TMP_OUT}/${name}" +done + +read -r AGENT_PORT MACOS_VERSION MACOS_BUILD < <( + /usr/bin/python3 - "${TMP_OUT}/bundle.json" <<'PY' +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as f: + manifest = json.load(f) +print( + manifest.get("agent", {}).get("port", 10700), + manifest.get("macos_version", ""), + manifest.get("macos_build", ""), +) +PY +) + +ATTACH_OUTPUT="$(hdiutil attach -nomount "${TMP_OUT}/disk.img")" +ATTACHED_DISK="$(printf '%s\n' "${ATTACH_OUTPUT}" | awk '/GUID_partition_scheme/ {print $1; exit}')" +[[ -n "${ATTACHED_DISK}" ]] || die "could not identify attached disk" + +MAIN_CONTAINER="$(diskutil list "${ATTACHED_DISK}" | awk ' + /Apple_APFS Container disk/ && $0 !~ /ISC|Recovery/ { + for (i = 1; i <= NF; i++) { + if ($i ~ /^disk[0-9]+$/ && $(i - 1) == "Container") { + print $i + exit + } + } + } +')" +[[ -n "${MAIN_CONTAINER}" ]] || die "could not identify main APFS container" + +DATA_VOLUME="$(diskutil apfs list "${MAIN_CONTAINER}" | awk ' + /APFS Volume Disk \(Role\):/ && /\(Data\)/ { + for (i = 1; i <= NF; i++) { + if ($i == "(Role):" && (i + 1) <= NF) { + print $(i + 1) + exit + } + } + } +')" +[[ -n "${DATA_VOLUME}" ]] || die "could not identify APFS Data volume" + +diskutil mount "${DATA_VOLUME}" >/dev/null +MOUNT_POINT="$(diskutil info "${DATA_VOLUME}" | awk -F: '/Mount Point/ {sub(/^[[:space:]]+/, "", $2); print $2; exit}')" +[[ -n "${MOUNT_POINT}" && "${MOUNT_POINT}" != "Not mounted" ]] || die "could not identify Data volume mount point" + +if [[ "${INSTALL_MODE}" == "launchdaemon" ]]; then + install -d \ + "${MOUNT_POINT}/usr/local/bin" \ + "${MOUNT_POINT}/Library/LaunchDaemons" \ + "${MOUNT_POINT}/private/var/db" \ + "${MOUNT_POINT}/private/var/log" + install -m 0755 "${AGENT_BIN}" "${MOUNT_POINT}/usr/local/bin/cleanroom-macos-guest-agent" + + PLIST_PATH="${MOUNT_POINT}/Library/LaunchDaemons/com.buildkite.cleanroom.macos-guest-agent.plist" + sed "s/10700<\\/string>/${AGENT_PORT}<\\/string>/" \ + "${ROOT_DIR}/cmd/cleanroom-macos-guest-agent/com.buildkite.cleanroom.macos-guest-agent.plist" > "${PLIST_PATH}" + chmod 0644 "${PLIST_PATH}" + touch "${MOUNT_POINT}/private/var/db/.AppleSetupDone" + chmod 0644 "${MOUNT_POINT}/private/var/db/.AppleSetupDone" + + xattr -c "${MOUNT_POINT}/usr/local/bin/cleanroom-macos-guest-agent" "${PLIST_PATH}" 2>/dev/null || true + + if ! chown 0:0 "${MOUNT_POINT}/usr/local/bin/cleanroom-macos-guest-agent" "${PLIST_PATH}" 2>/dev/null; then + if [[ "${ALLOW_UNVERIFIED_OWNERSHIP}" -ne 1 ]]; then + die "could not set root ownership on installed files; rerun with privileges, or pass --allow-unverified-ownership for an inspection-only bundle" + fi + echo "prepare-agent-bundle: warning: could not set root ownership on installed files; bundle is inspection-only until live smoke proves launchd starts the agent" >&2 + fi +else + if [[ "${AGENT_UID}" != "$(id -u)" || "${AGENT_GID}" != "$(id -g)" ]]; then + echo "prepare-agent-bundle: warning: user-cron mode works rootlessly only when --agent-uid/--agent-gid match the current host user; ownership must be verified by live smoke" >&2 + fi + chmod u+rwx \ + "${MOUNT_POINT}/private/var/db/dslocal/nodes/Default" \ + "${MOUNT_POINT}/private/var/db/dslocal/nodes/Default/users" \ + "${MOUNT_POINT}/private/var/db/dslocal/nodes/Default/groups" 2>/dev/null || true + + /usr/bin/python3 - "${MOUNT_POINT}" "${AGENT_USER}" "${AGENT_PASSWORD}" "${AGENT_UID}" "${AGENT_GID}" "${AGENT_PORT}" "${AGENT_VERSION}" "${MACOS_VERSION}" "${MACOS_BUILD}" "${USER_CRON_AUTOLOGIN}" <<'PY' +import hashlib +import os +import plistlib +import sys +import uuid +from pathlib import Path + +mount = Path(sys.argv[1]) +user = sys.argv[2] +password = sys.argv[3].encode() +uid = sys.argv[4] +gid = sys.argv[5] +port = sys.argv[6] +version = sys.argv[7] +macos_version = sys.argv[8] +macos_build = sys.argv[9] +autologin = sys.argv[10] == "1" +guid = str(uuid.uuid4()).upper() + +target = mount / "private/var/db/dslocal/nodes/Default" +users_dir = target / "users" +groups_dir = target / "groups" +if not users_dir.is_dir(): + raise SystemExit(f"dslocal users directory not found: {users_dir}") +if not groups_dir.is_dir(): + raise SystemExit(f"dslocal groups directory not found: {groups_dir}") +if (users_dir / f"{user}.plist").exists(): + raise SystemExit(f"guest user already exists: {user}") +for existing_user_path in users_dir.glob("*.plist"): + try: + with existing_user_path.open("rb") as f: + existing_user = plistlib.load(f) + except Exception: + continue + if uid in existing_user.get("uid", []): + existing_name = existing_user_path.stem + raise SystemExit(f"guest uid {uid} already belongs to {existing_name}") + +salt = os.urandom(32) +entropy = hashlib.pbkdf2_hmac("sha512", password, salt, 35000, dklen=128) +shadow = plistlib.dumps({ + "SALTED-SHA512-PBKDF2": { + "entropy": entropy, + "salt": salt, + "iterations": 35000, + } +}, fmt=plistlib.FMT_BINARY) + +user_record = { + "ShadowHashData": [shadow], + "authentication_authority": [";ShadowHash;HASHLIST:"], + "generateduid": [guid], + "gid": [gid], + "home": [f"/Users/{user}"], + "name": [user], + "passwd": ["********"], + "realname": ["Cleanroom CI"], + "shell": ["/bin/zsh"], + "uid": [uid], +} +user_path = users_dir / f"{user}.plist" +with user_path.open("wb") as f: + plistlib.dump(user_record, f, fmt=plistlib.FMT_BINARY) +os.chmod(user_path, 0o600) + +admin_path = groups_dir / "admin.plist" +with admin_path.open("rb") as f: + admin = plistlib.load(f) +admin.setdefault("users", []) +admin.setdefault("groupmembers", []) +if user not in admin["users"]: + admin["users"].append(user) +if guid not in admin["groupmembers"]: + admin["groupmembers"].append(guid) +with admin_path.open("wb") as f: + plistlib.dump(admin, f, fmt=plistlib.FMT_BINARY) +os.chmod(admin_path, 0o600) + +(mount / "private/var/db/.AppleSetupDone").touch() +os.chmod(mount / "private/var/db/.AppleSetupDone", 0o644) + +def write_plist(path, values): + path.parent.mkdir(parents=True, exist_ok=True) + current = {} + if path.exists(): + with path.open("rb") as f: + current = plistlib.load(f) + current.update(values) + with path.open("wb") as f: + plistlib.dump(current, f, fmt=plistlib.FMT_BINARY) + os.chmod(path, 0o644) + +setup_values = { + "DidSeeAccessibility": True, + "DidSeeActivationLock": True, + "DidSeeAppearanceSetup": True, + "DidSeeApplePaySetup": True, + "DidSeeAppStore": True, + "DidSeeCloudSetup": True, + "DidSeeiCloudLoginForStorageServices": True, + "DidSeeLockdownMode": True, + "DidSeePrivacy": True, + "DidSeeScreenTime": True, + "DidSeeSiriSetup": True, + "DidSeeSyncSetup": True, + "DidSeeSyncSetup2": True, + "DidSeeTermsOfAddress": True, + "DidSeeTouchIDSetup": True, + "DidSeeTrueTonePrivacy": True, + "GestureMovieSeen": "none", + "InitialAccountOnMac": True, + "LastSeenBuddyBuildVersion": macos_build, + "LastSeenCloudProductVersion": "99.99", + "MiniBuddyLaunchedPostMigration": True, + "MiniBuddyLaunchReason": 0, + "MiniBuddyShouldLaunchToResumeSetup": False, + "PreviousBuildVersion": macos_build, + "PreviousSystemVersion": macos_version, + "SkipExpressSettingsUpdating": True, + "SkipFirstLoginOptimization": True, +} +if autologin: + for rel in [ + f"Users/{user}/Library/Preferences/com.apple.SetupAssistant.plist", + "Library/Preferences/com.apple.SetupAssistant.plist", + ]: + write_plist(mount / rel, setup_values) + + login_values = { + "autoLoginUser": user, + "GuestEnabled": False, + "lastUser": "loggedIn", + "lastUserName": user, + "MiniBuddyLaunch": False, + "MiniBuddyLaunchCount": 0, + "oneTimeSSMigrationComplete": True, + "RecentUsers": [user], + } + write_plist(mount / "Library/Preferences/com.apple.loginwindow.plist", login_values) + write_plist(mount / f"Users/{user}/Library/Preferences/com.apple.loginwindow.plist", { + "MiniBuddyLaunch": False, + "MiniBuddyLaunchCount": 0, + "oneTimeSSMigrationComplete": True, + }) + +software_update_values = { + "AutomaticCheckEnabled": False, + "AutomaticDownload": False, + "AutomaticallyInstallMacOSUpdates": False, + "ConfigDataInstall": False, + "CriticalUpdateInstall": False, + "PostSuccessfulMinorUpdatePostLogOutNotification": False, + "RecommendedUpdates": [], +} +if macos_build and macos_version: + software_update_values["LastAttemptBuildVersion"] = f"{macos_version} ({macos_build})" + software_update_values["LastAttemptSystemVersion"] = f"{macos_version} ({macos_build})" +write_plist(mount / "Library/Preferences/com.apple.SoftwareUpdate.plist", software_update_values) +write_plist(mount / f"Users/{user}/Library/Preferences/com.apple.SoftwareUpdate.plist", software_update_values) + +if autologin: + key = bytes([0x7D, 0x89, 0x52, 0x23, 0xD2, 0xBC, 0xDD, 0xEA, 0xA3, 0xB9, 0x1F]) + plain = bytearray(password + b"\x00") + while len(plain) % len(key) != 0: + plain.append(0) + encoded = bytes(b ^ key[i % len(key)] for i, b in enumerate(plain)) + kcpassword = mount / "private/etc/kcpassword" + kcpassword.parent.mkdir(parents=True, exist_ok=True) + kcpassword.write_bytes(encoded) + os.chmod(kcpassword, 0o600) + +for rel in [ + f"Users/{user}/bin", + f"Users/{user}/Library/Logs", + f"Users/{user}/Library/LaunchAgents", + f"Users/{user}/Desktop", + f"Users/{user}/Documents", + f"Users/{user}/Downloads", + "private/var/at/tabs", +]: + (mount / rel).mkdir(parents=True, exist_ok=True) + +cron = mount / f"private/var/at/tabs/{user}" +cron.write_text( + "SHELL=/bin/sh\n" + f"CLEANROOM_VSOCK_PORT={port}\n" + f"PATH=/usr/bin:/bin:/usr/sbin:/sbin:/Users/{user}/bin\n" + f"* * * * * test -f /private/var/db/cleanroom-macos-guest-agent.finalized && exit 0; " + f"/usr/bin/pgrep -f '/Users/{user}/bin/cleanroom-macos-guest-agent' >/dev/null || " + f"/Users/{user}/bin/cleanroom-macos-guest-agent " + f">>/Users/{user}/Library/Logs/cleanroom-macos-guest-agent.cron.log " + f"2>>/Users/{user}/Library/Logs/cleanroom-macos-guest-agent.cron.err\n", + encoding="utf-8", +) +os.chmod(cron, 0o600) + +if autologin: + launch_agent = mount / f"Users/{user}/Library/LaunchAgents/com.buildkite.cleanroom.macos-guest-agent.plist" + with launch_agent.open("wb") as f: + plistlib.dump({ + "Label": "com.buildkite.cleanroom.macos-guest-agent", + "ProgramArguments": [ + "/bin/sh", + "-lc", + f"test -f /private/var/db/cleanroom-macos-guest-agent.finalized || exec /Users/{user}/bin/cleanroom-macos-guest-agent", + ], + "EnvironmentVariables": {"CLEANROOM_VSOCK_PORT": port}, + "RunAtLoad": True, + "KeepAlive": True, + "StandardOutPath": f"/Users/{user}/Library/Logs/cleanroom-macos-guest-agent.log", + "StandardErrorPath": f"/Users/{user}/Library/Logs/cleanroom-macos-guest-agent.err.log", + }, f, fmt=plistlib.FMT_BINARY) + os.chmod(launch_agent, 0o644) + +print(f"configured user-cron agent user={user} uid={uid} gid={gid} port={port} version={version}") +PY + + install -m 0755 "${AGENT_BIN}" "${MOUNT_POINT}/Users/${AGENT_USER}/bin/cleanroom-macos-guest-agent" + xattr -cr \ + "${MOUNT_POINT}/Users/${AGENT_USER}/bin/cleanroom-macos-guest-agent" \ + "${MOUNT_POINT}/Users/${AGENT_USER}/Library/LaunchAgents/com.buildkite.cleanroom.macos-guest-agent.plist" \ + "${MOUNT_POINT}/private/var/at/tabs/${AGENT_USER}" 2>/dev/null || true +fi + +/usr/bin/python3 - "${TMP_OUT}/bundle.json" "${AGENT_VERSION}" <<'PY' +import json +import sys + +path = sys.argv[1] +version = sys.argv[2] + +with open(path, "r", encoding="utf-8") as f: + manifest = json.load(f) + +agent = manifest.setdefault("agent", {}) +agent["transport"] = "virtio_socket" +agent["version"] = version + +with open(path, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2, sort_keys=True) + f.write("\n") +PY + +diskutil unmount "${DATA_VOLUME}" >/dev/null +DATA_VOLUME="" +hdiutil detach "${ATTACHED_DISK}" >/dev/null +ATTACHED_DISK="" + +if [[ -e "${OUT}" ]]; then + rm -rf "${OUT}" +fi +mv "${TMP_OUT}" "${OUT}" +TMP_OUT="" + +echo "prepared bundle: ${OUT}" diff --git a/benchmarks/darwin-vz/macos-minimal/runner.swift b/benchmarks/darwin-vz/macos-minimal/runner.swift new file mode 100644 index 00000000..a93b7ca5 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/runner.swift @@ -0,0 +1,738 @@ +import Darwin +import Foundation +import Virtualization + +private struct Options { + var bundlePath = "" + var command: [String] = ["/usr/bin/sw_vers"] + var metricsPath = "" + var validateOnly = false + var timeoutSeconds = 120.0 + var connectIntervalMS: useconds_t = 50 + var agentName = "root" +} + +private struct BundleManifest: Decodable { + let schemaVersion: Int + let os: String + let arch: String + let macOSVersion: String? + let macOSBuild: String? + let vcpus: Int + let memoryMiB: UInt64 + let disk: String + let auxiliaryStorage: String + let hardwareModel: String + let machineIdentifier: String + let agent: AgentManifest + let userAgent: AgentManifest? + let display: DisplayManifest? + + enum CodingKeys: String, CodingKey { + case schemaVersion = "schema_version" + case os + case arch + case macOSVersion = "macos_version" + case macOSBuild = "macos_build" + case vcpus + case memoryMiB = "memory_mib" + case disk + case auxiliaryStorage = "auxiliary_storage" + case hardwareModel = "hardware_model" + case machineIdentifier = "machine_identifier" + case agent + case userAgent = "user_agent" + case display + } +} + +private struct AgentManifest: Decodable { + let transport: String + let port: UInt32 + let version: String +} + +private struct DisplayManifest: Decodable { + let widthPx: Int? + let heightPx: Int? + let pixelsPerInch: Int? + + enum CodingKeys: String, CodingKey { + case widthPx = "width_px" + case heightPx = "height_px" + case pixelsPerInch = "pixels_per_inch" + } +} + +private struct ResolvedBundle { + let manifestURL: URL + let manifest: BundleManifest + let diskURL: URL + let auxiliaryStorageURL: URL + let hardwareModel: VZMacHardwareModel + let machineIdentifier: VZMacMachineIdentifier +} + +private func validateAgent(_ agent: AgentManifest, field: String) throws { + guard agent.transport == "virtio_socket" else { + throw RunnerError.invalid("\(field).transport must be virtio_socket") + } + guard agent.port > 0 else { + throw RunnerError.invalid("\(field).port must be greater than zero") + } + guard !agent.version.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw RunnerError.invalid("\(field).version must not be empty") + } +} + +private func selectedAgent(bundle: ResolvedBundle, name: String) throws -> AgentManifest { + switch name { + case "root": + return bundle.manifest.agent + case "user": + guard let userAgent = bundle.manifest.userAgent else { + throw RunnerError.invalid("bundle does not declare user_agent") + } + return userAgent + default: + throw RunnerError.invalid("--agent must be root or user") + } +} + +private struct ExecRequest: Encodable { + let command: [String] + let env: [String] + let dir: String? + + enum CodingKeys: String, CodingKey { + case command + case env + case dir + } +} + +private struct ExecInputFrame: Encodable { + let type: String +} + +private struct ExecFrame: Decodable { + let type: String + let data: Data? + let exitCode: Int? + let error: String? + + enum CodingKeys: String, CodingKey { + case type + case data + case exitCode = "exit_code" + case error + } +} + +private struct ProbeResult: Encodable { + let bundle: String + let command: [String] + let startedVM: Bool + let startMS: Double? + let vsockConnectMS: Double? + let execResponseMS: Double? + let exitCode: Int? + let error: String? + let selectedAgent: String + let macOSVersion: String? + let macOSBuild: String? + let agentVersion: String + let vcpus: Int + let memoryMiB: UInt64 + + enum CodingKeys: String, CodingKey { + case bundle + case command + case startedVM = "started_vm" + case startMS = "start_ms" + case vsockConnectMS = "vsock_connect_ms" + case execResponseMS = "exec_response_ms" + case exitCode = "exit_code" + case error + case selectedAgent = "selected_agent" + case macOSVersion = "macos_version" + case macOSBuild = "macos_build" + case agentVersion = "agent_version" + case vcpus + case memoryMiB = "memory_mib" + } +} + +private enum RunnerError: LocalizedError { + case usage(String) + case invalid(String) + case timeout(String) + case posix(String, Int32) + case vm(String) + + var errorDescription: String? { + switch self { + case .usage(let message), .invalid(let message), .timeout(let message), .vm(let message): + return message + case .posix(let op, let code): + return "\(op): \(String(cString: strerror(code)))" + } + } +} + +private final class VMHandle { + let vm: VZVirtualMachine + let queue: DispatchQueue + + init(vm: VZVirtualMachine, queue: DispatchQueue) { + self.vm = vm + self.queue = queue + } + + func stop() { + if currentState() == .stopped { + return + } + if requestGracefulStop(), waitUntilStopped(timeoutSeconds: 20) { + return + } + forceStop(timeoutSeconds: 10) + } + + private func currentState() -> VZVirtualMachine.State? { + let sem = DispatchSemaphore(value: 0) + var state: VZVirtualMachine.State? + queue.async { + state = self.vm.state + sem.signal() + } + guard sem.wait(timeout: .now() + .seconds(3)) == .success else { + return nil + } + return state + } + + private func requestGracefulStop() -> Bool { + let sem = DispatchSemaphore(value: 0) + var requested = false + queue.async { + if self.vm.state == .stopped { + requested = true + } else if self.vm.canRequestStop { + do { + try self.vm.requestStop() + requested = true + } catch { + requested = false + } + } + sem.signal() + } + guard sem.wait(timeout: .now() + .seconds(3)) == .success else { + return false + } + return requested + } + + private func waitUntilStopped(timeoutSeconds: Double) -> Bool { + let deadline = Date().addingTimeInterval(timeoutSeconds) + while Date() < deadline { + if currentState() == .stopped { + return true + } + Thread.sleep(forTimeInterval: 0.2) + } + return currentState() == .stopped + } + + private func forceStop(timeoutSeconds: Double) { + let sem = DispatchSemaphore(value: 0) + queue.async { + if self.vm.state == .stopped { + sem.signal() + } else if self.vm.canStop { + self.vm.stop { _ in sem.signal() } + } else { + sem.signal() + } + } + _ = sem.wait(timeout: .now() + .milliseconds(Int(timeoutSeconds * 1_000))) + } +} + +private func usage() -> String { + """ + Usage: + darwin-vz-macos-minimal --bundle [options] [-- [args...]] + + Options: + --metrics Write result JSON to path. If omitted, JSON is written to stderr. + --agent Agent endpoint to use. Default: root. + --validate-only Validate the bundle and host support without starting the VM. + --timeout VM start, connect, and command timeout. Default: 120. + -h, --help Show this help. + + The default command is /usr/bin/sw_vers. + """ +} + +private func parseOptions(_ args: [String]) throws -> Options { + var opts = Options() + var i = 0 + while i < args.count { + let arg = args[i] + if arg == "--" { + opts.command = Array(args.dropFirst(i + 1)) + break + } + + func value() throws -> String { + guard i + 1 < args.count else { + throw RunnerError.usage("missing value for \(arg)\n\n\(usage())") + } + i += 1 + return args[i] + } + + switch arg { + case "--bundle": + opts.bundlePath = try value() + case "--metrics": + opts.metricsPath = try value() + if opts.metricsPath == "-" { + throw RunnerError.usage("--metrics - is not supported because guest stdout is streamed on stdout; write metrics to a file or omit --metrics") + } + case "--agent": + opts.agentName = try value() + guard opts.agentName == "root" || opts.agentName == "user" else { + throw RunnerError.invalid("--agent must be root or user") + } + case "--validate-only": + opts.validateOnly = true + case "--timeout": + guard let timeout = Double(try value()), timeout > 0 else { + throw RunnerError.invalid("invalid --timeout") + } + opts.timeoutSeconds = timeout + case "-h", "--help": + throw RunnerError.usage(usage()) + default: + throw RunnerError.usage("unknown argument: \(arg)\n\n\(usage())") + } + i += 1 + } + + if opts.bundlePath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw RunnerError.usage("missing --bundle\n\n\(usage())") + } + if opts.command.isEmpty { + throw RunnerError.invalid("command after -- must not be empty") + } + if opts.command.first?.isEmpty == true { + throw RunnerError.invalid("command after -- must not be empty") + } + return opts +} + +private func now() -> UInt64 { + DispatchTime.now().uptimeNanoseconds +} + +private func msSince(_ start: UInt64) -> Double { + Double(now() - start) / 1_000_000.0 +} + +private func absoluteURL(_ path: String, relativeTo baseURL: URL) -> URL { + let expanded = NSString(string: path).expandingTildeInPath + if expanded.hasPrefix("/") { + return URL(fileURLWithPath: expanded) + } + return baseURL.appendingPathComponent(expanded) +} + +private func manifestURL(from path: String) throws -> URL { + let expanded = NSString(string: path).expandingTildeInPath + var isDir: ObjCBool = false + guard FileManager.default.fileExists(atPath: expanded, isDirectory: &isDir) else { + throw RunnerError.invalid("bundle path does not exist: \(expanded)") + } + if isDir.boolValue { + return URL(fileURLWithPath: expanded).appendingPathComponent("bundle.json") + } + return URL(fileURLWithPath: expanded) +} + +private func requireReadableFile(_ url: URL, field: String) throws { + var isDir: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDir), !isDir.boolValue else { + throw RunnerError.invalid("\(field) does not exist or is not a file: \(url.path)") + } +} + +private func loadBundle(path: String) throws -> ResolvedBundle { + let url = try manifestURL(from: path) + try requireReadableFile(url, field: "bundle manifest") + let baseURL = url.deletingLastPathComponent() + let manifest = try JSONDecoder().decode(BundleManifest.self, from: Data(contentsOf: url)) + + guard manifest.schemaVersion == 1 else { + throw RunnerError.invalid("unsupported schema_version \(manifest.schemaVersion)") + } + guard manifest.os == "macos" else { + throw RunnerError.invalid("bundle os must be macos") + } + guard manifest.arch == "arm64" else { + throw RunnerError.invalid("bundle arch must be arm64") + } + guard manifest.vcpus > 0 else { + throw RunnerError.invalid("vcpus must be greater than zero") + } + guard manifest.memoryMiB >= 1024 else { + throw RunnerError.invalid("memory_mib must be at least 1024") + } + guard manifest.memoryMiB <= UInt64.max / 1024 / 1024 else { + throw RunnerError.invalid("memory_mib is too large") + } + try validateAgent(manifest.agent, field: "agent") + if let userAgent = manifest.userAgent { + try validateAgent(userAgent, field: "user_agent") + if userAgent.port == manifest.agent.port { + throw RunnerError.invalid("user_agent.port must differ from agent.port") + } + } + if let display = manifest.display { + if let width = display.widthPx, width <= 0 { + throw RunnerError.invalid("display.width_px must be greater than zero") + } + if let height = display.heightPx, height <= 0 { + throw RunnerError.invalid("display.height_px must be greater than zero") + } + if let pixelsPerInch = display.pixelsPerInch, pixelsPerInch <= 0 { + throw RunnerError.invalid("display.pixels_per_inch must be greater than zero") + } + } + + let diskURL = absoluteURL(manifest.disk, relativeTo: baseURL) + let auxiliaryURL = absoluteURL(manifest.auxiliaryStorage, relativeTo: baseURL) + let hardwareModelURL = absoluteURL(manifest.hardwareModel, relativeTo: baseURL) + let machineIdentifierURL = absoluteURL(manifest.machineIdentifier, relativeTo: baseURL) + try requireReadableFile(diskURL, field: "disk") + try requireReadableFile(auxiliaryURL, field: "auxiliary_storage") + try requireReadableFile(hardwareModelURL, field: "hardware_model") + try requireReadableFile(machineIdentifierURL, field: "machine_identifier") + + guard let hardwareModel = VZMacHardwareModel(dataRepresentation: try Data(contentsOf: hardwareModelURL)) else { + throw RunnerError.invalid("hardware_model is not a valid VZMacHardwareModel data representation") + } + guard hardwareModel.isSupported else { + throw RunnerError.invalid("hardware_model is not supported by this host") + } + guard let machineIdentifier = VZMacMachineIdentifier(dataRepresentation: try Data(contentsOf: machineIdentifierURL)) else { + throw RunnerError.invalid("machine_identifier is not a valid VZMacMachineIdentifier data representation") + } + + return ResolvedBundle( + manifestURL: url, + manifest: manifest, + diskURL: diskURL, + auxiliaryStorageURL: auxiliaryURL, + hardwareModel: hardwareModel, + machineIdentifier: machineIdentifier + ) +} + +private func writeAll(fd: Int32, bytes: [UInt8]) throws { + var written = 0 + while written < bytes.count { + let n = bytes.withUnsafeBytes { raw -> Int in + guard let base = raw.baseAddress else { return 0 } + return Darwin.write(fd, base.advanced(by: written), bytes.count - written) + } + if n < 0 { + if errno == EINTR { + continue + } + throw RunnerError.posix("write", errno) + } + if n == 0 { + throw RunnerError.posix("write", EIO) + } + written += n + } +} + +private func waitForReadable(fd: Int32, deadline: Date) throws { + while true { + let remaining = deadline.timeIntervalSinceNow + if remaining <= 0 { + throw RunnerError.timeout("timed out waiting for guest frame") + } + let timeoutMS = Int32(max(1, min(remaining * 1000, Double(Int32.max)))) + var pfd = pollfd(fd: fd, events: Int16(POLLIN), revents: 0) + let n = Darwin.poll(&pfd, 1, timeoutMS) + if n < 0 { + if errno == EINTR { + continue + } + throw RunnerError.posix("poll", errno) + } + if n == 0 { + throw RunnerError.timeout("timed out waiting for guest frame") + } + if (pfd.revents & Int16(POLLIN | POLLHUP | POLLERR | POLLNVAL)) != 0 { + return + } + } +} + +private func readLine(fd: Int32, buffer: inout Data, deadline: Date) throws -> Data { + var chunk = [UInt8](repeating: 0, count: 4096) + while true { + if let newline = buffer.firstIndex(of: 0x0A) { + let line = buffer.subdata(in: 0.. Int in + guard let base = raw.baseAddress else { return 0 } + return Darwin.read(fd, base, chunkCount) + } + if n < 0 { + if errno == EINTR { + continue + } + throw RunnerError.posix("read", errno) + } + if n == 0 { + throw RunnerError.vm("guest connection closed before exit frame") + } + buffer.append(contentsOf: chunk[0.. VMHandle { + guard VZVirtualMachine.isSupported else { + throw RunnerError.vm("Virtualization.framework is not supported on this host") + } + + let config = VZVirtualMachineConfiguration() + config.bootLoader = VZMacOSBootLoader() + config.cpuCount = bundle.manifest.vcpus + config.memorySize = bundle.manifest.memoryMiB * 1024 * 1024 + + let platform = VZMacPlatformConfiguration() + platform.hardwareModel = bundle.hardwareModel + platform.machineIdentifier = bundle.machineIdentifier + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: bundle.auxiliaryStorageURL) + config.platform = platform + + let display = bundle.manifest.display + let graphics = VZMacGraphicsDeviceConfiguration() + graphics.displays = [ + VZMacGraphicsDisplayConfiguration( + widthInPixels: display?.widthPx ?? 1024, + heightInPixels: display?.heightPx ?? 768, + pixelsPerInch: display?.pixelsPerInch ?? 72 + ) + ] + config.graphicsDevices = [graphics] + config.keyboards = [VZUSBKeyboardConfiguration()] + config.pointingDevices = [VZUSBScreenCoordinatePointingDeviceConfiguration()] + + let disk = try VZDiskImageStorageDeviceAttachment( + url: bundle.diskURL, + readOnly: false, + cachingMode: .automatic, + synchronizationMode: .full + ) + config.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)] + config.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + config.socketDevices = [VZVirtioSocketDeviceConfiguration()] + + try config.validate() + return VMHandle(vm: VZVirtualMachine(configuration: config, queue: queue), queue: queue) +} + +private func startVM(_ handle: VMHandle, timeoutSeconds: Double) throws { + let sem = DispatchSemaphore(value: 0) + var startError: Error? + handle.queue.async { + handle.vm.start { result in + if case .failure(let error) = result { + startError = error + } + sem.signal() + } + } + if sem.wait(timeout: .now() + timeoutSeconds) == .timedOut { + throw RunnerError.timeout("timed out waiting for VM start") + } + if let startError { + throw RunnerError.vm("VM start failed: \(startError)") + } +} + +private func connectVsock(handle: VMHandle, port: UInt32, timeoutSeconds: Double, intervalMS: useconds_t) throws -> VZVirtioSocketConnection { + guard let socketDevice = handle.vm.socketDevices.first as? VZVirtioSocketDevice else { + throw RunnerError.vm("VM has no virtio socket device") + } + let deadline = Date().addingTimeInterval(timeoutSeconds) + var lastError: Error? + while Date() < deadline { + let sem = DispatchSemaphore(value: 0) + var connection: VZVirtioSocketConnection? + var connectError: Error? + handle.queue.async { + socketDevice.connect(toPort: port) { result in + switch result { + case .success(let conn): + connection = conn + case .failure(let error): + connectError = error + } + sem.signal() + } + } + _ = sem.wait(timeout: .now() + .milliseconds(500)) + if let connection { + return connection + } + if let connectError { + lastError = connectError + } + usleep(intervalMS * 1000) + } + if let lastError { + throw RunnerError.timeout("timed out connecting to guest vsock port \(port): \(lastError)") + } + throw RunnerError.timeout("timed out connecting to guest vsock port \(port)") +} + +private func writeExecRequest(connection: VZVirtioSocketConnection, command: [String]) throws { + let request = ExecRequest(command: command, env: [], dir: nil) + let encoded = try JSONEncoder().encode(request) + try writeAll(fd: connection.fileDescriptor, bytes: Array(encoded) + [0x0A]) + let eof = try JSONEncoder().encode(ExecInputFrame(type: "eof")) + try writeAll(fd: connection.fileDescriptor, bytes: Array(eof) + [0x0A]) +} + +private func runGuestCommand(connection: VZVirtioSocketConnection, command: [String], timeoutSeconds: Double) throws -> (Int, String?) { + try writeExecRequest(connection: connection, command: command) + let deadline = Date().addingTimeInterval(timeoutSeconds) + var buffer = Data() + while true { + let line = try readLine(fd: connection.fileDescriptor, buffer: &buffer, deadline: deadline) + if line.isEmpty { + continue + } + let frame = try JSONDecoder().decode(ExecFrame.self, from: line) + switch frame.type { + case "stdout": + if let data = frame.data { + try writeAll(fd: STDOUT_FILENO, bytes: Array(data)) + } + case "stderr": + if let data = frame.data { + try writeAll(fd: STDERR_FILENO, bytes: Array(data)) + } + case "exit": + let exitCode = frame.exitCode ?? 0 + if exitCode < 0 { + return (1, frame.error ?? "guest command exited without a status") + } + guard (0...255).contains(exitCode) else { + throw RunnerError.vm("guest exit_code must be between 0 and 255") + } + return (exitCode, frame.error) + default: + throw RunnerError.vm("unknown guest frame type \(frame.type)") + } + } +} + +private func writeMetrics(_ result: ProbeResult, path: String) throws { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let encoded = try encoder.encode(result) + if path == "-" { + print(String(data: encoded, encoding: .utf8)!) + return + } + if path.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + fputs(String(data: encoded, encoding: .utf8)! + "\n", stderr) + return + } + try encoded.write(to: URL(fileURLWithPath: NSString(string: path).expandingTildeInPath)) +} + +do { + let opts = try parseOptions(Array(CommandLine.arguments.dropFirst())) + let bundle = try loadBundle(path: opts.bundlePath) + let agent = try selectedAgent(bundle: bundle, name: opts.agentName) + if opts.validateOnly { + let queue = DispatchQueue(label: "cleanroom.benchmark.darwin-vz-macos-minimal.validate") + _ = try buildVM(bundle: bundle, queue: queue) + let result = ProbeResult( + bundle: bundle.manifestURL.path, + command: opts.command, + startedVM: false, + startMS: nil, + vsockConnectMS: nil, + execResponseMS: nil, + exitCode: nil, + error: nil, + selectedAgent: opts.agentName, + macOSVersion: bundle.manifest.macOSVersion, + macOSBuild: bundle.manifest.macOSBuild, + agentVersion: agent.version, + vcpus: bundle.manifest.vcpus, + memoryMiB: bundle.manifest.memoryMiB + ) + try writeMetrics(result, path: opts.metricsPath) + Foundation.exit(0) + } + + let queue = DispatchQueue(label: "cleanroom.benchmark.darwin-vz-macos-minimal.vm") + let t0 = now() + let handle = try buildVM(bundle: bundle, queue: queue) + defer { handle.stop() } + + try startVM(handle, timeoutSeconds: opts.timeoutSeconds) + let startMS = msSince(t0) + let connection = try connectVsock( + handle: handle, + port: agent.port, + timeoutSeconds: opts.timeoutSeconds, + intervalMS: opts.connectIntervalMS + ) + defer { connection.close() } + let connectMS = msSince(t0) + let outcome = try runGuestCommand(connection: connection, command: opts.command, timeoutSeconds: opts.timeoutSeconds) + let execResponseMS = msSince(t0) + let result = ProbeResult( + bundle: bundle.manifestURL.path, + command: opts.command, + startedVM: true, + startMS: startMS, + vsockConnectMS: connectMS, + execResponseMS: execResponseMS, + exitCode: outcome.0, + error: outcome.1, + selectedAgent: opts.agentName, + macOSVersion: bundle.manifest.macOSVersion, + macOSBuild: bundle.manifest.macOSBuild, + agentVersion: agent.version, + vcpus: bundle.manifest.vcpus, + memoryMiB: bundle.manifest.memoryMiB + ) + try writeMetrics(result, path: opts.metricsPath) + Foundation.exit(Int32(outcome.0)) +} catch RunnerError.usage(let message) { + fputs(message + "\n", stderr) + Foundation.exit(message.hasPrefix("Usage:") ? 0 : 2) +} catch { + fputs("darwin-vz-macos-minimal: \(error.localizedDescription)\n", stderr) + Foundation.exit(1) +} diff --git a/benchmarks/darwin-vz/macos-minimal/viewer.swift b/benchmarks/darwin-vz/macos-minimal/viewer.swift new file mode 100644 index 00000000..b3d91451 --- /dev/null +++ b/benchmarks/darwin-vz/macos-minimal/viewer.swift @@ -0,0 +1,457 @@ +import AppKit +import Foundation +import Virtualization + +private struct Options { + var bundlePath = "" + var sharedDirectoryPath: String? + var validateOnly = false + var screenshotPath: String? + var screenshotAfterSeconds = 30.0 +} + +private struct BundleManifest: Decodable { + let schemaVersion: Int + let os: String + let arch: String + let macOSVersion: String? + let macOSBuild: String? + let vcpus: Int + let memoryMiB: UInt64 + let disk: String + let auxiliaryStorage: String + let hardwareModel: String + let machineIdentifier: String + let display: DisplayManifest? + + enum CodingKeys: String, CodingKey { + case schemaVersion = "schema_version" + case os + case arch + case macOSVersion = "macos_version" + case macOSBuild = "macos_build" + case vcpus + case memoryMiB = "memory_mib" + case disk + case auxiliaryStorage = "auxiliary_storage" + case hardwareModel = "hardware_model" + case machineIdentifier = "machine_identifier" + case display + } +} + +private struct DisplayManifest: Decodable { + let widthPx: Int? + let heightPx: Int? + let pixelsPerInch: Int? + + enum CodingKeys: String, CodingKey { + case widthPx = "width_px" + case heightPx = "height_px" + case pixelsPerInch = "pixels_per_inch" + } +} + +private struct ResolvedBundle { + let manifestURL: URL + let manifest: BundleManifest + let diskURL: URL + let auxiliaryStorageURL: URL + let hardwareModel: VZMacHardwareModel + let machineIdentifier: VZMacMachineIdentifier +} + +private enum ViewerError: LocalizedError { + case usage(String) + case invalid(String) + case vm(String) + + var errorDescription: String? { + switch self { + case .usage(let message), .invalid(let message), .vm(let message): + return message + } + } +} + +private func usage() -> String { + """ + Usage: + darwin-vz-macos-viewer --bundle [options] + + Options: + --shared-directory Expose a host directory read-only with the macOS guest automount tag. + --validate-only Validate the bundle and optional share without starting the VM. + --screenshot Write a viewer-owned PNG snapshot after the VM starts. + --screenshot-after + Delay before writing --screenshot. Default: 30. + -h, --help Show this help. + """ +} + +private func parseOptions(_ args: [String]) throws -> Options { + var opts = Options() + var i = 0 + while i < args.count { + let arg = args[i] + + func value() throws -> String { + guard i + 1 < args.count else { + throw ViewerError.usage("missing value for \(arg)\n\n\(usage())") + } + i += 1 + return args[i] + } + + switch arg { + case "--bundle": + opts.bundlePath = try value() + case "--shared-directory": + opts.sharedDirectoryPath = try value() + case "--validate-only": + opts.validateOnly = true + case "--screenshot": + opts.screenshotPath = try value() + case "--screenshot-after": + guard let seconds = Double(try value()), seconds >= 0 else { + throw ViewerError.invalid("invalid --screenshot-after") + } + opts.screenshotAfterSeconds = seconds + case "-h", "--help": + throw ViewerError.usage(usage()) + default: + throw ViewerError.usage("unknown argument: \(arg)\n\n\(usage())") + } + i += 1 + } + + if opts.bundlePath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + throw ViewerError.usage("missing --bundle\n\n\(usage())") + } + return opts +} + +private func absoluteURL(_ path: String, relativeTo baseURL: URL) -> URL { + let expanded = NSString(string: path).expandingTildeInPath + if expanded.hasPrefix("/") { + return URL(fileURLWithPath: expanded) + } + return baseURL.appendingPathComponent(expanded) +} + +private func fileExists(_ url: URL, description: String) throws { + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), !isDirectory.boolValue else { + throw ViewerError.invalid("\(description) does not exist or is a directory: \(url.path)") + } +} + +private func directoryExists(_ url: URL, description: String) throws { + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), isDirectory.boolValue else { + throw ViewerError.invalid("\(description) does not exist or is not a directory: \(url.path)") + } +} + +private func resolveBundle(_ path: String) throws -> ResolvedBundle { + let inputURL = URL(fileURLWithPath: NSString(string: path).expandingTildeInPath) + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: inputURL.path, isDirectory: &isDirectory) else { + throw ViewerError.invalid("bundle path does not exist: \(inputURL.path)") + } + + let manifestURL = isDirectory.boolValue ? inputURL.appendingPathComponent("bundle.json") : inputURL + let baseURL = manifestURL.deletingLastPathComponent() + let data = try Data(contentsOf: manifestURL) + let decoder = JSONDecoder() + let manifest = try decoder.decode(BundleManifest.self, from: data) + + guard manifest.schemaVersion == 1 else { + throw ViewerError.invalid("unsupported schema_version: \(manifest.schemaVersion)") + } + guard manifest.os == "macos" else { + throw ViewerError.invalid("bundle os must be macos") + } + guard manifest.arch == "arm64" else { + throw ViewerError.invalid("bundle arch must be arm64") + } + guard manifest.vcpus > 0 else { + throw ViewerError.invalid("vcpus must be positive") + } + guard manifest.memoryMiB > 0 else { + throw ViewerError.invalid("memory_mib must be positive") + } + + let diskURL = absoluteURL(manifest.disk, relativeTo: baseURL) + let auxiliaryStorageURL = absoluteURL(manifest.auxiliaryStorage, relativeTo: baseURL) + let hardwareModelURL = absoluteURL(manifest.hardwareModel, relativeTo: baseURL) + let machineIdentifierURL = absoluteURL(manifest.machineIdentifier, relativeTo: baseURL) + + try fileExists(diskURL, description: "disk") + try fileExists(auxiliaryStorageURL, description: "auxiliary_storage") + try fileExists(hardwareModelURL, description: "hardware_model") + try fileExists(machineIdentifierURL, description: "machine_identifier") + + guard let hardwareModel = VZMacHardwareModel(dataRepresentation: try Data(contentsOf: hardwareModelURL)) else { + throw ViewerError.invalid("hardware_model is not a valid VZMacHardwareModel data representation") + } + guard hardwareModel.isSupported else { + throw ViewerError.invalid("hardware_model is not supported on this host") + } + guard let machineIdentifier = VZMacMachineIdentifier(dataRepresentation: try Data(contentsOf: machineIdentifierURL)) else { + throw ViewerError.invalid("machine_identifier is not a valid VZMacMachineIdentifier data representation") + } + + return ResolvedBundle( + manifestURL: manifestURL, + manifest: manifest, + diskURL: diskURL, + auxiliaryStorageURL: auxiliaryStorageURL, + hardwareModel: hardwareModel, + machineIdentifier: machineIdentifier + ) +} + +private func buildVMConfiguration(bundle: ResolvedBundle, sharedDirectoryPath: String?) throws -> VZVirtualMachineConfiguration { + guard VZVirtualMachine.isSupported else { + throw ViewerError.vm("Virtualization.framework is not supported on this host") + } + + let config = VZVirtualMachineConfiguration() + config.bootLoader = VZMacOSBootLoader() + config.cpuCount = bundle.manifest.vcpus + config.memorySize = bundle.manifest.memoryMiB * 1024 * 1024 + + let platform = VZMacPlatformConfiguration() + platform.hardwareModel = bundle.hardwareModel + platform.machineIdentifier = bundle.machineIdentifier + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: bundle.auxiliaryStorageURL) + config.platform = platform + + let display = bundle.manifest.display + let graphics = VZMacGraphicsDeviceConfiguration() + graphics.displays = [ + VZMacGraphicsDisplayConfiguration( + widthInPixels: display?.widthPx ?? 1024, + heightInPixels: display?.heightPx ?? 768, + pixelsPerInch: display?.pixelsPerInch ?? 72 + ) + ] + config.graphicsDevices = [graphics] + config.keyboards = [VZUSBKeyboardConfiguration()] + config.pointingDevices = [VZUSBScreenCoordinatePointingDeviceConfiguration()] + + let disk = try VZDiskImageStorageDeviceAttachment( + url: bundle.diskURL, + readOnly: false, + cachingMode: .automatic, + synchronizationMode: .full + ) + config.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)] + config.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + config.socketDevices = [VZVirtioSocketDeviceConfiguration()] + + if let sharedDirectoryPath { + guard #available(macOS 13.0, *) else { + throw ViewerError.vm("--shared-directory requires macOS 13 or newer on the host") + } + let currentDirectoryURL = URL(fileURLWithPath: FileManager.default.currentDirectoryPath, isDirectory: true) + let sharedDirectoryURL = absoluteURL(sharedDirectoryPath, relativeTo: currentDirectoryURL) + try directoryExists(sharedDirectoryURL, description: "shared directory") + + let sharedDirectory = VZSharedDirectory(url: sharedDirectoryURL, readOnly: true) + let sharingDevice = VZVirtioFileSystemDeviceConfiguration(tag: VZVirtioFileSystemDeviceConfiguration.macOSGuestAutomountTag) + sharingDevice.share = VZSingleDirectoryShare(directory: sharedDirectory) + config.directorySharingDevices = [sharingDevice] + } + + try config.validate() + return config +} + +private final class ViewerAppDelegate: NSObject, NSApplicationDelegate, NSWindowDelegate { + private var window: NSWindow? + private var vm: VZVirtualMachine? + private let queue = DispatchQueue(label: "com.buildkite.cleanroom.macos-viewer.vm") + private var stopping = false + + func applicationDidFinishLaunching(_ notification: Notification) { + do { + let opts = try parseOptions(Array(CommandLine.arguments.dropFirst())) + let bundle = try resolveBundle(opts.bundlePath) + let config = try buildVMConfiguration(bundle: bundle, sharedDirectoryPath: opts.sharedDirectoryPath) + if opts.validateOnly { + print("bundle metadata validated: \(bundle.manifestURL.path)") + if let sharedDirectoryPath = opts.sharedDirectoryPath { + print("shared directory validated: \(sharedDirectoryPath)") + } + exit(0) + } + + let vm = VZVirtualMachine(configuration: config, queue: queue) + self.vm = vm + + let display = bundle.manifest.display + let width = CGFloat(display?.widthPx ?? 1024) + let height = CGFloat(display?.heightPx ?? 768) + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: width, height: height), + styleMask: [.titled, .closable, .miniaturizable, .resizable], + backing: .buffered, + defer: false + ) + window.title = "Cleanroom macOS VM" + window.delegate = self + + let view = VZVirtualMachineView(frame: NSRect(x: 0, y: 0, width: width, height: height)) + view.autoresizingMask = [.width, .height] + view.capturesSystemKeys = true + if #available(macOS 14.0, *) { + view.automaticallyReconfiguresDisplay = true + } + view.virtualMachine = vm + window.contentView = view + + if let sharedDirectoryPath = opts.sharedDirectoryPath { + fputs("shared directory: \(sharedDirectoryPath)\n", stderr) + fputs("macOS guest mount: /Volumes/My Shared Files\n", stderr) + } + + self.window = window + window.center() + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + + queue.async { + vm.start { result in + switch result { + case .failure(let error): + DispatchQueue.main.async { + fputs("darwin-vz-macos-viewer: VM start failed: \(error)\n", stderr) + NSApp.terminate(nil) + } + case .success: + guard let screenshotPath = opts.screenshotPath else { + return + } + DispatchQueue.main.asyncAfter(deadline: .now() + opts.screenshotAfterSeconds) { + do { + try writeSnapshot(of: view, to: screenshotPath) + fputs("wrote screenshot: \(screenshotPath)\n", stderr) + } catch { + fputs("darwin-vz-macos-viewer: screenshot failed: \(error.localizedDescription)\n", stderr) + } + } + } + } + } + } catch ViewerError.usage(let message) { + print(message) + exit(message == usage() ? 0 : 64) + } catch { + fputs("darwin-vz-macos-viewer: \(error.localizedDescription)\n", stderr) + exit(1) + } + } + + func windowWillClose(_ notification: Notification) { + NSApp.terminate(nil) + } + + func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply { + stopVM() + return .terminateNow + } + + private func stopVM() { + guard !stopping, let vm else { + return + } + stopping = true + if currentState(of: vm) == .stopped { + return + } + if requestGracefulStop(of: vm), waitUntilStopped(vm, timeoutSeconds: 20) { + return + } + forceStop(vm, timeoutSeconds: 10) + } + + private func currentState(of vm: VZVirtualMachine) -> VZVirtualMachine.State? { + let sem = DispatchSemaphore(value: 0) + var state: VZVirtualMachine.State? + queue.async { + state = vm.state + sem.signal() + } + guard sem.wait(timeout: .now() + .seconds(3)) == .success else { + return nil + } + return state + } + + private func requestGracefulStop(of vm: VZVirtualMachine) -> Bool { + let sem = DispatchSemaphore(value: 0) + var requested = false + queue.async { + if vm.state == .stopped { + requested = true + } else if vm.canRequestStop { + do { + try vm.requestStop() + requested = true + } catch { + requested = false + } + } + sem.signal() + } + guard sem.wait(timeout: .now() + .seconds(3)) == .success else { + return false + } + return requested + } + + private func waitUntilStopped(_ vm: VZVirtualMachine, timeoutSeconds: Double) -> Bool { + let deadline = Date().addingTimeInterval(timeoutSeconds) + while Date() < deadline { + if currentState(of: vm) == .stopped { + return true + } + Thread.sleep(forTimeInterval: 0.2) + } + return currentState(of: vm) == .stopped + } + + private func forceStop(_ vm: VZVirtualMachine, timeoutSeconds: Double) { + let sem = DispatchSemaphore(value: 0) + queue.async { + if vm.state == .stopped { + sem.signal() + } else if vm.canStop { + vm.stop { _ in sem.signal() } + } else { + sem.signal() + } + } + _ = sem.wait(timeout: .now() + .milliseconds(Int(timeoutSeconds * 1_000))) + } +} + +private func writeSnapshot(of view: NSView, to path: String) throws { + let url = absoluteURL(path, relativeTo: URL(fileURLWithPath: FileManager.default.currentDirectoryPath, isDirectory: true)) + guard let rep = view.bitmapImageRepForCachingDisplay(in: view.bounds) else { + throw ViewerError.vm("could not create bitmap representation") + } + view.cacheDisplay(in: view.bounds, to: rep) + guard let data = rep.representation(using: .png, properties: [:]) else { + throw ViewerError.vm("could not encode PNG") + } + try data.write(to: url) +} + +let app = NSApplication.shared +private let delegate = ViewerAppDelegate() +app.delegate = delegate +app.setActivationPolicy(.regular) +app.run() diff --git a/cmd/cleanroom-darwin-vz/main.swift b/cmd/cleanroom-darwin-vz/main.swift index 3dd9ddd1..d814431b 100644 --- a/cmd/cleanroom-darwin-vz/main.swift +++ b/cmd/cleanroom-darwin-vz/main.swift @@ -11,6 +11,10 @@ private struct ControlRequest: Decodable { let kernelPath: String? let rootFSPath: String? let sidecarDiskPaths: [String]? + let diskPath: String? + let auxiliaryStoragePath: String? + let hardwareModelPath: String? + let machineIdentifierPath: String? let bootArgs: String? let networkMode: String? let vmnetSubnetCIDR: String? @@ -30,12 +34,19 @@ private struct ControlRequest: Decodable { let proxySocketPath: String? let consoleLogPath: String? let vmID: String? + let displayWidthPx: Int? + let displayHeightPx: Int? + let displayPixelsPerInch: Int? enum CodingKeys: String, CodingKey { case op case kernelPath = "kernel_path" case rootFSPath = "rootfs_path" case sidecarDiskPaths = "sidecar_disk_paths" + case diskPath = "disk_path" + case auxiliaryStoragePath = "auxiliary_storage_path" + case hardwareModelPath = "hardware_model_path" + case machineIdentifierPath = "machine_identifier_path" case bootArgs = "boot_args" case networkMode = "network_mode" case vmnetSubnetCIDR = "vmnet_subnet_cidr" @@ -55,6 +66,9 @@ private struct ControlRequest: Decodable { case proxySocketPath = "proxy_socket_path" case consoleLogPath = "console_log_path" case vmID = "vm_id" + case displayWidthPx = "display_width_px" + case displayHeightPx = "display_height_px" + case displayPixelsPerInch = "display_pixels_per_inch" } } @@ -618,6 +632,157 @@ private final class VMRuntime { ) } + func startMacOS(from req: ControlRequest) throws -> ControlResponse { + guard VZVirtualMachine.isSupported else { + throw HelperError.vm("virtualization is not supported on this host") + } + + let diskPath = try requireAbsolutePath(req.diskPath, field: "disk_path") + let auxiliaryStoragePath = try requireAbsolutePath(req.auxiliaryStoragePath, field: "auxiliary_storage_path") + let hardwareModelPath = try requireAbsolutePath(req.hardwareModelPath, field: "hardware_model_path") + let machineIdentifierPath = try requireAbsolutePath(req.machineIdentifierPath, field: "machine_identifier_path") + let runDir = try requireAbsolutePath(req.runDir, field: "run_dir") + let proxySocketPath = try requireAbsolutePath(req.proxySocketPath, field: "proxy_socket_path") + + try requireFile(diskPath, field: "disk_path") + try requireFile(auxiliaryStoragePath, field: "auxiliary_storage_path") + try requireFile(hardwareModelPath, field: "hardware_model_path") + try requireFile(machineIdentifierPath, field: "machine_identifier_path") + try ensureDirectory(runDir) + try ensureDirectory((proxySocketPath as NSString).deletingLastPathComponent) + + if req.kernelPath?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false { + throw HelperError.invalidRequest("StartMacOSVM does not support kernel_path") + } + if req.rootFSPath?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false { + throw HelperError.invalidRequest("StartMacOSVM does not support rootfs_path") + } + if !(req.sidecarDiskPaths ?? []).isEmpty { + throw HelperError.invalidRequest("StartMacOSVM does not support sidecar_disk_paths") + } + if req.bootArgs?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false { + throw HelperError.invalidRequest("StartMacOSVM does not support boot_args") + } + if req.consoleLogPath?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false { + throw HelperError.invalidRequest("StartMacOSVM does not support console_log_path yet") + } + + let networkMode = req.networkMode?.trimmingCharacters(in: .whitespacesAndNewlines) + if let networkMode, !networkMode.isEmpty, networkMode != "none" { + throw HelperError.invalidRequest("StartMacOSVM currently supports only network_mode none") + } + if req.vmnetSubnetCIDR?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false { + throw HelperError.invalidRequest("StartMacOSVM does not support vmnet_subnet_cidr yet") + } + if let fileHandleSocketPath = req.fileHandleSocketPath?.trimmingCharacters(in: .whitespacesAndNewlines), + !fileHandleSocketPath.isEmpty { + throw HelperError.invalidRequest("StartMacOSVM does not support filehandle_socket_path yet") + } + if (req.vmnetExternalInterface?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false) + || req.vmnetDisableNAT44 == true + || req.vmnetDisableNAT66 == true + || req.vmnetDisableDNSProxy == true + || req.vmnetDisableRouterAdvertisement == true { + throw HelperError.invalidRequest("StartMacOSVM does not support vmnet-specific network settings") + } + if req.initialMemoryBalloonTargetMiB ?? 0 > 0 { + throw HelperError.invalidRequest("StartMacOSVM does not support initial_memory_balloon_target_mib") + } + + if let requestedVCPUs = req.vcpus, requestedVCPUs <= 0 { + throw HelperError.invalidRequest("vcpus must be greater than zero") + } + if let requestedMemoryMiB = req.memoryMiB, requestedMemoryMiB < 1024 { + throw HelperError.invalidRequest("memory_mib must be at least 1024 for StartMacOSVM") + } + let vcpus = req.vcpus ?? 4 + let memoryMiB = req.memoryMiB ?? 4096 + guard memoryMiB <= Int64.max / Int64(1024 * 1024) else { + throw HelperError.invalidRequest("memory_mib is too large") + } + if let requestedGuestPort = req.guestPort, requestedGuestPort == 0 { + throw HelperError.invalidRequest("guest_port must be greater than zero") + } + if let requestedLaunchSeconds = req.launchSeconds, requestedLaunchSeconds <= 0 { + throw HelperError.invalidRequest("launch_seconds must be greater than zero") + } + let guestPort = req.guestPort ?? 10_700 + let launchSeconds = req.launchSeconds ?? 120 + + lock.lock() + if vm != nil { + lock.unlock() + throw HelperError.invalidRequest("vm is already running") + } + lock.unlock() + + let startedAt = DispatchTime.now() + let vmQueue = DispatchQueue(label: "cleanroom.darwin-vz.vm.macos") + let vmID = UUID().uuidString + + let vm = try buildMacOSVM( + diskPath: diskPath, + auxiliaryStoragePath: auxiliaryStoragePath, + hardwareModelPath: hardwareModelPath, + machineIdentifierPath: machineIdentifierPath, + vcpus: vcpus, + memoryMiB: memoryMiB, + displayWidthPx: req.displayWidthPx, + displayHeightPx: req.displayHeightPx, + displayPixelsPerInch: req.displayPixelsPerInch, + queue: vmQueue + ) + let configBuiltAt = DispatchTime.now() + + let vzStartStartedAt = DispatchTime.now() + var stopVMOnFailure = true + defer { + if stopVMOnFailure { + try? stopVM(vm, queue: vmQueue) + } + } + try startVM(vm, queue: vmQueue, timeoutSeconds: launchSeconds) + let vzStartedAt = DispatchTime.now() + + let proxyReadyStartedAt = DispatchTime.now() + let proxy = try ProxyServer(path: proxySocketPath) + self.guestPort = guestPort + self.launchTimeout = TimeInterval(launchSeconds) + self.serialChannel = nil + self.vmQueue = vmQueue + self.vm = vm + self.vmID = vmID + self.proxy = proxy + self.memoryMiB = memoryMiB + self.fileHandleNetworkAttachment = nil + stopVMOnFailure = false + proxy.start { [weak self] in + guard let self else { + throw HelperError.vm("vm runtime no longer available") + } + return try self.connectGuestChannel() + } + let proxyReadyAt = DispatchTime.now() + + let timingMS = [ + "config_build": elapsedMilliseconds(from: startedAt, to: configBuiltAt), + "vz_start": elapsedMilliseconds(from: vzStartStartedAt, to: vzStartedAt), + "proxy_ready": elapsedMilliseconds(from: proxyReadyStartedAt, to: proxyReadyAt), + "vm_ready": elapsedMilliseconds(from: startedAt, to: proxyReadyAt), + ] + return ControlResponse( + ok: true, + error: nil, + vmID: vmID, + proxySocketPath: proxySocketPath, + vmnetSubnetCIDR: nil, + vmnetGuestIPv4: nil, + vmnetGatewayIPv4: nil, + vmnetPrefixLen: nil, + timingMS: timingMS + ) + } + func stop(vmID requestedID: String?) throws { lock.lock() let currentID = vmID @@ -1062,6 +1227,80 @@ private final class VMRuntime { return (VZVirtualMachine(configuration: config, queue: queue), channel, networkDetails, fileHandleNetworkAttachment) } + private func buildMacOSVM( + diskPath: String, + auxiliaryStoragePath: String, + hardwareModelPath: String, + machineIdentifierPath: String, + vcpus: Int, + memoryMiB: Int64, + displayWidthPx: Int?, + displayHeightPx: Int?, + displayPixelsPerInch: Int?, + queue: DispatchQueue + ) throws -> VZVirtualMachine { + let diskURL = URL(fileURLWithPath: diskPath) + let auxiliaryStorageURL = URL(fileURLWithPath: auxiliaryStoragePath) + guard let hardwareModel = VZMacHardwareModel(dataRepresentation: try Data(contentsOf: URL(fileURLWithPath: hardwareModelPath))) else { + throw HelperError.invalidRequest("hardware_model_path is not a valid VZMacHardwareModel data representation") + } + guard hardwareModel.isSupported else { + throw HelperError.invalidRequest("hardware_model_path is not supported by this host") + } + guard let machineIdentifier = VZMacMachineIdentifier(dataRepresentation: try Data(contentsOf: URL(fileURLWithPath: machineIdentifierPath))) else { + throw HelperError.invalidRequest("machine_identifier_path is not a valid VZMacMachineIdentifier data representation") + } + + let widthPx = displayWidthPx ?? 1024 + let heightPx = displayHeightPx ?? 768 + let pixelsPerInch = displayPixelsPerInch ?? 72 + guard widthPx > 0 else { + throw HelperError.invalidRequest("display_width_px must be greater than zero") + } + guard heightPx > 0 else { + throw HelperError.invalidRequest("display_height_px must be greater than zero") + } + guard pixelsPerInch > 0 else { + throw HelperError.invalidRequest("display_pixels_per_inch must be greater than zero") + } + + let config = VZVirtualMachineConfiguration() + config.bootLoader = VZMacOSBootLoader() + config.cpuCount = vcpus + config.memorySize = UInt64(memoryMiB) * 1024 * 1024 + + let platform = VZMacPlatformConfiguration() + platform.hardwareModel = hardwareModel + platform.machineIdentifier = machineIdentifier + platform.auxiliaryStorage = VZMacAuxiliaryStorage(url: auxiliaryStorageURL) + config.platform = platform + + let graphics = VZMacGraphicsDeviceConfiguration() + graphics.displays = [ + VZMacGraphicsDisplayConfiguration( + widthInPixels: widthPx, + heightInPixels: heightPx, + pixelsPerInch: pixelsPerInch + ) + ] + config.graphicsDevices = [graphics] + config.keyboards = [VZUSBKeyboardConfiguration()] + config.pointingDevices = [VZUSBScreenCoordinatePointingDeviceConfiguration()] + + let disk = try VZDiskImageStorageDeviceAttachment( + url: diskURL, + readOnly: false, + cachingMode: .automatic, + synchronizationMode: .full + ) + config.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)] + config.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + config.socketDevices = [VZVirtioSocketDeviceConfiguration()] + + try config.validate() + return VZVirtualMachine(configuration: config, queue: queue) + } + private func applyMemoryBalloonTarget( vm: VZVirtualMachine, queue: DispatchQueue, @@ -1212,8 +1451,14 @@ private final class VMRuntime { usleep(10_000) } if let lastError { + guard serialChannel != nil else { + throw HelperError.timeout("timed out connecting to guest vsock port \(guestPort): \(lastError)") + } fputs("cleanroom-darwin-vz: vsock connect fallback to serial after error: \(lastError)\n", stderr) } else { + guard serialChannel != nil else { + throw HelperError.timeout("timed out connecting to guest vsock port \(guestPort)") + } fputs("cleanroom-darwin-vz: vsock connect timed out, falling back to serial\n", stderr) } } @@ -1279,6 +1524,8 @@ private final class HelperService { switch req.op { case "StartVM": return try vmRuntime.start(from: req) + case "StartMacOSVM": + return try vmRuntime.startMacOS(from: req) case "StopVM": try vmRuntime.stop(vmID: req.vmID) return ControlResponse(ok: true, error: nil, vmID: nil, proxySocketPath: nil, vmnetSubnetCIDR: nil, vmnetGuestIPv4: nil, vmnetGatewayIPv4: nil, vmnetPrefixLen: nil, timingMS: nil) diff --git a/cmd/cleanroom-macos-guest-agent/com.buildkite.cleanroom.macos-guest-agent.plist b/cmd/cleanroom-macos-guest-agent/com.buildkite.cleanroom.macos-guest-agent.plist new file mode 100644 index 00000000..d82de053 --- /dev/null +++ b/cmd/cleanroom-macos-guest-agent/com.buildkite.cleanroom.macos-guest-agent.plist @@ -0,0 +1,25 @@ + + + + + Label + com.buildkite.cleanroom.macos-guest-agent + ProgramArguments + + /usr/local/bin/cleanroom-macos-guest-agent + + EnvironmentVariables + + CLEANROOM_VSOCK_PORT + 10700 + + RunAtLoad + + KeepAlive + + StandardOutPath + /var/log/cleanroom-macos-guest-agent.log + StandardErrorPath + /var/log/cleanroom-macos-guest-agent.err.log + + diff --git a/cmd/cleanroom-macos-guest-agent/main.go b/cmd/cleanroom-macos-guest-agent/main.go new file mode 100644 index 00000000..42f99095 --- /dev/null +++ b/cmd/cleanroom-macos-guest-agent/main.go @@ -0,0 +1,421 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "os" + "os/exec" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "syscall" + + "github.com/buildkite/cleanroom/internal/vsockexec" +) + +const ( + agentVersion = "0.1.0" + defaultHome = "/var/root" + defaultPath = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" +) + +var errListenerClosed = errors.New("listener closed") + +type streamListener interface { + Accept() (io.ReadWriteCloser, error) + Close() error +} + +type requestEnvelope struct { + Type string `json:"type,omitempty"` +} + +type probeExecRequest struct { + Type string `json:"type,omitempty"` + Command []string `json:"command"` + Dir string `json:"dir,omitempty"` + WorkingDirectory *string `json:"working_directory,omitempty"` + Env []string `json:"env,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + ClosedEnv bool `json:"closed_env,omitempty"` +} + +type controlResponse struct { + Type string `json:"type"` + Version string `json:"version"` + OS string `json:"os"` + Arch string `json:"arch"` + Capabilities []string `json:"capabilities,omitempty"` +} + +func main() { + if err := run(os.Args[1:], os.Stderr, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "cleanroom-macos-guest-agent: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, stderr io.Writer, stdout io.Writer) error { + fs := flag.NewFlagSet("cleanroom-macos-guest-agent", flag.ContinueOnError) + fs.SetOutput(stderr) + port := fs.Uint64("port", uint64(vsockexec.DefaultPort), "virtio socket port to listen on") + stdio := fs.Bool("stdio", false, "serve one request over stdin/stdout") + showVersion := fs.Bool("version", false, "print version and exit") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %s", strings.Join(fs.Args(), " ")) + } + if *showVersion { + fmt.Fprintf(stdout, "cleanroom-macos-guest-agent %s\n", agentVersion) + return nil + } + if *stdio { + handleConn(stdioConn{r: os.Stdin, w: os.Stdout}) + return nil + } + + portExplicit := false + fs.Visit(func(f *flag.Flag) { + if f.Name == "port" { + portExplicit = true + } + }) + listenPort, err := resolveListenPort(*port, portExplicit) + if err != nil { + return err + } + + ln, err := listenVsock(listenPort) + if err != nil { + return err + } + defer ln.Close() + return serve(ln, stderr) +} + +func resolveListenPort(flagPort uint64, explicit bool) (uint32, error) { + if !explicit { + return defaultPortFromEnv() + } + if flagPort == 0 || flagPort > math.MaxUint32 { + return 0, fmt.Errorf("port must be between 1 and %d", uint64(math.MaxUint32)) + } + return uint32(flagPort), nil +} + +func defaultPortFromEnv() (uint32, error) { + raw := strings.TrimSpace(os.Getenv("CLEANROOM_VSOCK_PORT")) + if raw == "" { + return vsockexec.DefaultPort, nil + } + port, err := strconv.ParseUint(raw, 10, 32) + if err != nil || port == 0 { + return 0, fmt.Errorf("invalid CLEANROOM_VSOCK_PORT %q", raw) + } + return uint32(port), nil +} + +func serve(ln streamListener, stderr io.Writer) error { + for { + conn, err := ln.Accept() + if err != nil { + if errors.Is(err, errListenerClosed) { + return nil + } + fmt.Fprintf(stderr, "accept: %v\n", err) + continue + } + go handleConn(conn) + } +} + +func handleConn(conn io.ReadWriteCloser) { + defer conn.Close() + + dec := json.NewDecoder(conn) + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + sendErrorResponse(conn, err) + return + } + + var envelope requestEnvelope + if err := json.Unmarshal(raw, &envelope); err != nil { + sendErrorResponse(conn, err) + return + } + + switch envelope.Type { + case "ready", "version": + sendControlResponse(conn, envelope.Type) + return + case "", "exec": + req, err := decodeExecRequest(raw) + if err != nil { + sendErrorResponse(conn, err) + return + } + handleExec(conn, dec, req) + default: + sendErrorResponse(conn, fmt.Errorf("unsupported request type %q", envelope.Type)) + } +} + +func sendControlResponse(w io.Writer, typ string) { + if typ == "" { + typ = "version" + } + _ = json.NewEncoder(w).Encode(controlResponse{ + Type: typ, + Version: agentVersion, + OS: runtime.GOOS, + Arch: runtime.GOARCH, + Capabilities: []string{ + "ready", + "version", + "exec", + "stdin", + "stdout", + "stderr", + "exit_code", + "env", + "working_directory", + }, + }) +} + +func decodeExecRequest(raw json.RawMessage) (vsockexec.ExecRequest, error) { + var req probeExecRequest + if err := json.Unmarshal(raw, &req); err != nil { + return vsockexec.ExecRequest{}, err + } + if len(req.Command) == 0 { + return vsockexec.ExecRequest{}, errors.New("missing command") + } + if strings.TrimSpace(req.Command[0]) == "" { + return vsockexec.ExecRequest{}, errors.New("missing command executable") + } + + dir := req.Dir + if req.WorkingDirectory != nil { + dir = *req.WorkingDirectory + } + + return vsockexec.ExecRequest{ + Command: append([]string(nil), req.Command...), + Dir: dir, + Env: mergeEnv(req.Env, req.Environment), + ClosedEnv: req.ClosedEnv, + }, nil +} + +func mergeEnv(entries []string, mapped map[string]string) []string { + if len(mapped) == 0 { + return append([]string(nil), entries...) + } + values := make(map[string]string, len(entries)+len(mapped)) + for _, entry := range entries { + key, value, _ := strings.Cut(entry, "=") + values[key] = value + } + for key, value := range mapped { + values[key] = value + } + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]string, 0, len(keys)) + for _, key := range keys { + out = append(out, key+"="+values[key]) + } + return out +} + +func handleExec(conn io.Writer, dec *json.Decoder, req vsockexec.ExecRequest) { + cmd := exec.Command(req.Command[0], req.Command[1:]...) + if req.Dir != "" { + cmd.Dir = req.Dir + } + cmd.Env = buildCommandEnv(req.Env, !req.ClosedEnv) + + stdin, err := cmd.StdinPipe() + if err != nil { + sendErrorResponse(conn, err) + return + } + stdout, err := cmd.StdoutPipe() + if err != nil { + sendErrorResponse(conn, err) + return + } + stderr, err := cmd.StderrPipe() + if err != nil { + sendErrorResponse(conn, err) + return + } + if err := cmd.Start(); err != nil { + sendErrorResponse(conn, err) + return + } + + sender := newFrameSender(conn) + go readInputFrames(dec, stdin) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _, _ = io.Copy(streamFrameWriter{send: sender.Send, kind: "stdout"}, stdout) + }() + go func() { + defer wg.Done() + _, _ = io.Copy(streamFrameWriter{send: sender.Send, kind: "stderr"}, stderr) + }() + + wg.Wait() + waitErr := cmd.Wait() + sendExitResult(sender, waitErr) +} + +func readInputFrames(dec *json.Decoder, stdin io.WriteCloser) { + defer stdin.Close() + for { + var frame vsockexec.ExecInputFrame + if err := dec.Decode(&frame); err != nil { + return + } + switch frame.Type { + case "stdin": + if len(frame.Data) > 0 { + if _, err := stdin.Write(frame.Data); err != nil { + return + } + } + case "eof": + return + } + } +} + +func buildCommandEnv(requestEnv []string, inheritAmbient bool) []string { + base := map[string]string{} + if inheritAmbient { + for _, entry := range os.Environ() { + key, value, _ := strings.Cut(entry, "=") + base[key] = value + } + } + if _, ok := base["HOME"]; !ok { + base["HOME"] = defaultHome + } + if _, ok := base["PATH"]; !ok { + base["PATH"] = defaultPath + } + for _, entry := range requestEnv { + key, value, _ := strings.Cut(entry, "=") + base[key] = value + } + + keys := make([]string, 0, len(base)) + for key := range base { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]string, 0, len(keys)) + for _, key := range keys { + out = append(out, key+"="+base[key]) + } + return out +} + +func sendErrorResponse(w io.Writer, err error) { + msg := err.Error() + sender := newFrameSender(w) + if sendErr := sender.Send(vsockexec.ExecStreamFrame{ + Type: "stderr", + Data: []byte(msg + "\n"), + }); sendErr != nil { + return + } + _ = sender.Send(vsockexec.ExecStreamFrame{ + Type: "exit", + ExitCode: 1, + Error: msg, + }) +} + +func sendExitResult(sender *frameSender, waitErr error) { + exitCode, errMsg := exitResult(waitErr) + _ = sender.Send(vsockexec.ExecStreamFrame{ + Type: "exit", + ExitCode: exitCode, + Error: errMsg, + }) +} + +func exitResult(waitErr error) (int, string) { + if waitErr == nil { + return 0, "" + } + if exitErr, ok := waitErr.(*exec.ExitError); ok { + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() { + return 128 + int(status.Signal()), status.Signal().String() + } + if code := exitErr.ExitCode(); code >= 0 { + return code, "" + } + } + return 1, waitErr.Error() +} + +type frameSender struct { + w io.Writer + mu sync.Mutex +} + +func newFrameSender(w io.Writer) *frameSender { + return &frameSender{w: w} +} + +func (s *frameSender) Send(frame vsockexec.ExecStreamFrame) error { + s.mu.Lock() + defer s.mu.Unlock() + return vsockexec.EncodeStreamFrame(s.w, frame) +} + +type streamFrameWriter struct { + send func(vsockexec.ExecStreamFrame) error + kind string +} + +func (w streamFrameWriter) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if err := w.send(vsockexec.ExecStreamFrame{ + Type: w.kind, + Data: append([]byte(nil), p...), + }); err != nil { + return 0, err + } + return len(p), nil +} + +type stdioConn struct { + r io.Reader + w io.Writer +} + +func (c stdioConn) Read(p []byte) (int, error) { return c.r.Read(p) } +func (c stdioConn) Write(p []byte) (int, error) { return c.w.Write(p) } +func (stdioConn) Close() error { return nil } diff --git a/cmd/cleanroom-macos-guest-agent/main_test.go b/cmd/cleanroom-macos-guest-agent/main_test.go new file mode 100644 index 00000000..b3808bab --- /dev/null +++ b/cmd/cleanroom-macos-guest-agent/main_test.go @@ -0,0 +1,277 @@ +package main + +import ( + "bytes" + "encoding/json" + "net" + "path/filepath" + "strings" + "testing" + + "github.com/buildkite/cleanroom/internal/vsockexec" +) + +func TestHandleExecStreamsOutputAndExit(t *testing.T) { + stdout, stderr, res := runExecRequest(t, vsockexec.ExecRequest{ + Command: []string{"/bin/sh", "-c", "printf out; printf err >&2"}, + }, nil) + + if got, want := stdout, "out"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + if got, want := stderr, "err"; got != want { + t.Fatalf("stderr = %q, want %q", got, want) + } + if got, want := res.ExitCode, 0; got != want { + t.Fatalf("exit code = %d, want %d", got, want) + } + if res.Error != "" { + t.Fatalf("unexpected error: %q", res.Error) + } +} + +func TestHandleExecUsesDirEnvAndStdin(t *testing.T) { + dir := t.TempDir() + wantDir := canonicalPath(t, dir) + stdout, stderr, res := runExecRequest(t, vsockexec.ExecRequest{ + Command: []string{"/bin/sh", "-c", "printf '%s:%s:' \"$PWD\" \"$CLEANROOM_TEST_VALUE\"; cat"}, + Dir: dir, + Env: []string{"CLEANROOM_TEST_VALUE=ok"}, + }, []byte("input")) + + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + if got, want := stdout, wantDir+":ok:input"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + if got, want := res.ExitCode, 0; got != want { + t.Fatalf("exit code = %d, want %d", got, want) + } +} + +func TestHandleExecReportsNonZeroExit(t *testing.T) { + stdout, stderr, res := runExecRequest(t, vsockexec.ExecRequest{ + Command: []string{"/bin/sh", "-c", "printf bad >&2; exit 7"}, + }, nil) + + if stdout != "" { + t.Fatalf("stdout = %q, want empty", stdout) + } + if got, want := stderr, "bad"; got != want { + t.Fatalf("stderr = %q, want %q", got, want) + } + if got, want := res.ExitCode, 7; got != want { + t.Fatalf("exit code = %d, want %d", got, want) + } + if res.Error != "" { + t.Fatalf("unexpected error: %q", res.Error) + } +} + +func TestHandleExecReportsSignaledExit(t *testing.T) { + stdout, stderr, res := runExecRequest(t, vsockexec.ExecRequest{ + Command: []string{"/bin/sh", "-c", "kill -TERM $$"}, + }, nil) + + if stdout != "" { + t.Fatalf("stdout = %q, want empty", stdout) + } + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + if got, want := res.ExitCode, 143; got != want { + t.Fatalf("exit code = %d, want %d", got, want) + } + if res.Error == "" { + t.Fatal("expected signal error") + } +} + +func TestHandleExecAcceptsProbeRequestShape(t *testing.T) { + client, done := startTestAgent(t) + defer client.Close() + + dir := t.TempDir() + wantDir := canonicalPath(t, dir) + if err := json.NewEncoder(client).Encode(map[string]any{ + "type": "exec", + "command": []string{"/bin/sh", "-c", "printf '%s:%s' \"$PWD\" \"$CLEANROOM_TEST_VALUE\""}, + "environment": map[string]string{"CLEANROOM_TEST_VALUE": "ok"}, + "working_directory": dir, + }); err != nil { + t.Fatalf("encode request: %v", err) + } + if err := vsockexec.EncodeInputFrame(client, vsockexec.ExecInputFrame{Type: "eof"}); err != nil { + t.Fatalf("encode eof: %v", err) + } + + var stdout strings.Builder + res, err := vsockexec.DecodeStreamResponse(client, vsockexec.StreamCallbacks{ + OnStdout: func(chunk []byte) { stdout.Write(chunk) }, + }) + if err != nil { + t.Fatalf("decode stream response: %v", err) + } + if got, want := stdout.String(), wantDir+":ok"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + if got, want := res.ExitCode, 0; got != want { + t.Fatalf("exit code = %d, want %d", got, want) + } + + client.Close() + <-done +} + +func TestHandleVersionRequest(t *testing.T) { + client, done := startTestAgent(t) + defer client.Close() + + if err := json.NewEncoder(client).Encode(map[string]string{"type": "version"}); err != nil { + t.Fatalf("encode request: %v", err) + } + + var res controlResponse + if err := json.NewDecoder(client).Decode(&res); err != nil { + t.Fatalf("decode response: %v", err) + } + if got, want := res.Type, "version"; got != want { + t.Fatalf("type = %q, want %q", got, want) + } + if got, want := res.Version, agentVersion; got != want { + t.Fatalf("version = %q, want %q", got, want) + } + if !contains(res.Capabilities, "exec") { + t.Fatalf("capabilities = %#v, want exec", res.Capabilities) + } + + client.Close() + <-done +} + +func TestDefaultPortFromEnv(t *testing.T) { + t.Setenv("CLEANROOM_VSOCK_PORT", "12000") + port, err := defaultPortFromEnv() + if err != nil { + t.Fatalf("defaultPortFromEnv returned error: %v", err) + } + if got, want := port, uint32(12000); got != want { + t.Fatalf("port = %d, want %d", got, want) + } +} + +func TestRunVersionIgnoresInvalidPortEnv(t *testing.T) { + t.Setenv("CLEANROOM_VSOCK_PORT", "bogus") + var stderr, stdout bytes.Buffer + if err := run([]string{"--version"}, &stderr, &stdout); err != nil { + t.Fatalf("run --version returned error: %v", err) + } + if got, want := stdout.String(), "cleanroom-macos-guest-agent "+agentVersion+"\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q, want empty", stderr.String()) + } +} + +func TestResolveListenPortExplicitOverridesInvalidPortEnv(t *testing.T) { + t.Setenv("CLEANROOM_VSOCK_PORT", "bogus") + port, err := resolveListenPort(12000, true) + if err != nil { + t.Fatalf("resolveListenPort returned error: %v", err) + } + if got, want := port, uint32(12000); got != want { + t.Fatalf("port = %d, want %d", got, want) + } +} + +func TestBuildCommandEnvClosedEnvUsesMacOSDefaults(t *testing.T) { + t.Setenv("HOME", "/ambient") + env := buildCommandEnv([]string{"EXAMPLE=value"}, false) + got := envMap(env) + if got["HOME"] != defaultHome { + t.Fatalf("HOME = %q, want %q", got["HOME"], defaultHome) + } + if got["PATH"] != defaultPath { + t.Fatalf("PATH = %q, want %q", got["PATH"], defaultPath) + } + if got["EXAMPLE"] != "value" { + t.Fatalf("EXAMPLE = %q, want value", got["EXAMPLE"]) + } +} + +func runExecRequest(t *testing.T, req vsockexec.ExecRequest, stdin []byte) (string, string, vsockexec.ExecResponse) { + t.Helper() + + client, done := startTestAgent(t) + defer client.Close() + + if err := vsockexec.EncodeRequest(client, req); err != nil { + t.Fatalf("encode request: %v", err) + } + if len(stdin) > 0 { + if err := vsockexec.EncodeInputFrame(client, vsockexec.ExecInputFrame{Type: "stdin", Data: stdin}); err != nil { + t.Fatalf("encode stdin: %v", err) + } + } + if err := vsockexec.EncodeInputFrame(client, vsockexec.ExecInputFrame{Type: "eof"}); err != nil { + t.Fatalf("encode eof: %v", err) + } + + var stdout, stderr strings.Builder + res, err := vsockexec.DecodeStreamResponse(client, vsockexec.StreamCallbacks{ + OnStdout: func(chunk []byte) { stdout.Write(chunk) }, + OnStderr: func(chunk []byte) { stderr.Write(chunk) }, + }) + if err != nil { + t.Fatalf("decode stream response: %v", err) + } + client.Close() + <-done + return stdout.String(), stderr.String(), res +} + +func startTestAgent(t *testing.T) (net.Conn, <-chan struct{}) { + t.Helper() + + server, client := net.Pipe() + done := make(chan struct{}) + go func() { + defer close(done) + handleConn(server) + }() + return client, done +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func envMap(entries []string) map[string]string { + out := make(map[string]string, len(entries)) + for _, entry := range entries { + key, value, ok := strings.Cut(entry, "=") + if !ok { + out[entry] = "" + continue + } + out[key] = value + } + return out +} + +func canonicalPath(t *testing.T, path string) string { + t.Helper() + + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + t.Fatalf("eval symlinks: %v", err) + } + return resolved +} diff --git a/cmd/cleanroom-macos-guest-agent/vsock_darwin.go b/cmd/cleanroom-macos-guest-agent/vsock_darwin.go new file mode 100644 index 00000000..8aeb9cd2 --- /dev/null +++ b/cmd/cleanroom-macos-guest-agent/vsock_darwin.go @@ -0,0 +1,100 @@ +//go:build darwin + +package main + +import ( + "fmt" + "io" + "os" + "sync/atomic" + "syscall" + + "golang.org/x/sys/unix" +) + +const darwinVsockCIDAny = 0xffffffff + +type vsockListener struct { + fd int + closed atomic.Bool +} + +func listenVsock(port uint32) (streamListener, error) { + fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM, 0) + if err != nil { + return nil, fmt.Errorf("open vsock listener: %w", err) + } + unix.CloseOnExec(fd) + if err := unix.SetNonblock(fd, true); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("set vsock listener nonblocking: %w", err) + } + if err := unix.Bind(fd, &unix.SockaddrVM{CID: darwinVsockCIDAny, Port: port}); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("bind vsock port %d: %w", port, err) + } + if err := unix.Listen(fd, 128); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("listen vsock port %d: %w", port, err) + } + return &vsockListener{fd: fd}, nil +} + +func (l *vsockListener) Accept() (io.ReadWriteCloser, error) { + for { + fd := -1 + var err error + syscall.ForkLock.RLock() + fd, _, err = unix.Accept(l.fd) + if err == nil { + unix.CloseOnExec(fd) + err = unix.SetNonblock(fd, false) + } + syscall.ForkLock.RUnlock() + if err == unix.EINTR { + continue + } + if err == unix.EAGAIN || err == unix.EWOULDBLOCK { + if l.closed.Load() { + return nil, errListenerClosed + } + if err := l.waitReadable(); err != nil { + if l.closed.Load() { + return nil, errListenerClosed + } + return nil, err + } + continue + } + if err != nil { + if fd >= 0 { + _ = unix.Close(fd) + } + if l.closed.Load() { + return nil, errListenerClosed + } + return nil, err + } + return os.NewFile(uintptr(fd), "cleanroom-macos-guest-agent-vsock"), nil + } +} + +func (l *vsockListener) waitReadable() error { + for { + _, err := unix.Poll([]unix.PollFd{{Fd: int32(l.fd), Events: unix.POLLIN}}, 1000) + if err == unix.EINTR { + continue + } + return err + } +} + +func (l *vsockListener) Close() error { + if l.closed.Swap(true) { + return nil + } + if err := unix.Close(l.fd); err != nil { + return fmt.Errorf("close vsock listener: %w", err) + } + return nil +} diff --git a/cmd/cleanroom-macos-guest-agent/vsock_unsupported.go b/cmd/cleanroom-macos-guest-agent/vsock_unsupported.go new file mode 100644 index 00000000..7e0d2af0 --- /dev/null +++ b/cmd/cleanroom-macos-guest-agent/vsock_unsupported.go @@ -0,0 +1,12 @@ +//go:build !darwin + +package main + +import ( + "fmt" + "runtime" +) + +func listenVsock(_ uint32) (streamListener, error) { + return nil, fmt.Errorf("macOS guest agent vsock listener is only supported on darwin (current: %s)", runtime.GOOS) +} diff --git a/docs/backend/darwin-vz.md b/docs/backend/darwin-vz.md index bedd5d3c..61cc7667 100644 --- a/docs/backend/darwin-vz.md +++ b/docs/backend/darwin-vz.md @@ -16,6 +16,8 @@ Implemented: - launched execution on macOS via `Virtualization.framework` - interactive and non-interactive command execution via existing `internal/vsockexec` protocol - helper-managed VM lifecycle (`StartVM` / `StopVM` / `PauseVM` / `ResumeVM` / `SetMemoryBalloonTarget`) +- experimental helper-only macOS guest launch operation (`StartMacOSVM`) for + prepared local bundles; this is not wired into the production Go adapter yet - `filehandle` network mode with a Cleanroom-owned guest gateway and stable guest IP - TCP allowlist egress filtering for the active effective policy in `filehandle` mode - allow-all egress for repo-agnostic sandboxes created with `cleanroom sandbox create --dangerously-allow-all` @@ -40,7 +42,7 @@ Control plane: - socket: `/vz-helper.sock` - protocol: newline-delimited JSON request/response -- operations: `StartVM`, `StopVM`, `PauseVM`, `ResumeVM`, `SetMemoryBalloonTarget`, `Ping` +- operations: `StartVM`, `StartMacOSVM`, `StopVM`, `PauseVM`, `ResumeVM`, `SetMemoryBalloonTarget`, `Ping` Data plane: @@ -68,6 +70,34 @@ High-level flow: - `proxy_socket_path` - `console_log_path` +`StartMacOSVM` is an experimental helper operation for a prepared Apple +Silicon macOS guest bundle. It intentionally shares the same helper binary and +proxy socket model as the Linux path, but uses macOS-specific bundle artifacts +instead of Linux kernel/rootfs fields. + +`StartMacOSVM` request fields: + +- `disk_path` absolute path to the writable macOS disk image clone +- `auxiliary_storage_path` absolute path to `VZMacAuxiliaryStorage` +- `hardware_model_path` absolute path to the `VZMacHardwareModel` data + representation +- `machine_identifier_path` absolute path to the `VZMacMachineIdentifier` data + representation +- `vcpus`, `memory_mib`, `guest_port`, `launch_seconds` +- `run_dir` +- `proxy_socket_path` +- optional `display_width_px`, `display_height_px`, and + `display_pixels_per_inch` +- optional `network_mode`, which currently must be omitted or set to `none` + +The helper starts the VM with `VZMacOSBootLoader`, `VZMacPlatformConfiguration`, +the configured disk, graphics/input devices required by macOS guests, and a +virtio socket device for the Cleanroom macOS guest agent. It rejects filehandle +networking and vmnet-specific settings until macOS guest networking is wired +through the Cleanroom policy gateway. The production adapter must not advertise +macOS guest networking, file operations, snapshotting, or workspace support +until those paths are implemented and reported through capabilities. + `vcpus` and `memory_mib` are effective VM launch ceilings after runtime config and policy resource minimums are merged. They are not an exact host reservation contract. diff --git a/docs/plans/macos-cleanrooms.md b/docs/plans/macos-cleanrooms.md new file mode 100644 index 00000000..8b1d90b3 --- /dev/null +++ b/docs/plans/macos-cleanrooms.md @@ -0,0 +1,763 @@ +# macOS Cleanrooms Tart Replacement Plan + +**Status:** Proposed +**Last reviewed:** 2026-06-02 +**Spec references:** `docs/backends.md`, `docs/backend/darwin-vz.md`, `benchmarks/darwin-vz/minimal/README.md`, `docs/research.md` + +## Summary + +Cleanroom can plausibly replace Tart for macOS CI workloads, but not by +turning the current `darwin-vz` backend from Linux into macOS. The current +macOS host backend is a Linux guest backend: Go resolves an OCI-derived ext4 +rootfs and Linux kernel, the Swift helper starts that guest with +`VZLinuxBootLoader`, and the guest-side command protocol is implemented by the +Linux `cleanroom-guest-agent`. + +The replacement path is a parallel macOS guest platform under the same +Virtualization.framework helper and control-plane shape. Apple provides the +required primitives for macOS guests on Apple Silicon, and Tart's source and +docs prove the operational model is practical: install from IPSW, persist a +disk plus auxiliary storage, run with `VZMacOSBootLoader`, expose host +directories through VirtioFS, and execute commands through a guest agent over a +virtio socket. + +The first useful target should be narrow: run a Buildkite-style command in an +ephemeral macOS VM cloned from a prepared image, without invoking `tart`. Full +Cleanroom parity requires more work: macOS VM image lifecycle, a macOS guest +agent, file transfer/workspace setup, policy-grade networking, OCI +distribution or Tart-image import, and real host validation for Xcode/keychain +workloads. + +## Problem + +Tart currently solves three operational problems for macOS CI: distributing +macOS VM images, cloning and starting those images quickly, and running a job +command in the guest. It does not solve Cleanroom's stronger contract around +repository-scoped policy, host-held credentials, auditability, and fail-closed +capability reporting. + +Cleanroom already has a macOS host backend, but that backend only runs Linux +guests. If we present that backend as "macOS Cleanrooms" without a separate +macOS guest path, we will either preserve Tart as the real execution layer or +ship a VM runner that bypasses the security and policy behavior that makes +Cleanroom worth using. + +## Goals + +- Replace Tart at runtime for a first Buildkite macOS CI workflow: clone, + start, exec, stream logs, collect exit code, and tear down. +- Keep the public contract backend-neutral. Users ask for a macOS guest + platform and an image, not a new backend name such as `darwin-vz-macos`. +- Reuse the existing `darwin-vz` helper/control split where it helps, but keep + the Linux guest path unchanged until the macOS path proves itself. +- Support command execution without SSH by installing a Cleanroom macOS guest + agent into the image. +- Preserve Cleanroom's policy story before declaring replacement parity: + deny-by-default networking, stage-scoped policy swaps, gateway mediation, and + machine-readable capability gaps. +- Make image identity explicit enough for CI: macOS build, hardware model, + disk/auxiliary storage digests, guest-agent version, and clone semantics. + +## Non-Goals + +- Do not implement every Tart command or flag. +- Do not build Orchard-style cluster orchestration. +- Do not make GUI automation, VNC, audio, clipboard, or Screen Sharing part of + the first supported surface. +- Do not support Intel macOS guests or cross-architecture emulation. +- Do not make VZ NAT plus SSH the definition of a Cleanroom sandbox. +- Do not make Tart's OCI media types the native Cleanroom image contract, + although an importer is useful for migration. + +## Evidence + +### Current Cleanroom State + +`docs/backends.md` describes Cleanroom as running Linux microVMs on macOS and +Linux. The macOS backend is `darwin-vz`, but the guest is still Linux. + +`docs/backend/darwin-vz.md` says the helper request takes `kernel_path` and +`rootfs_path`, and the implemented flow resolves a Linux kernel, derives an +ext4 rootfs, injects the Linux guest runtime, and starts the helper-managed VM. + +`cmd/cleanroom-darwin-vz/main.swift` confirms this mechanically: + +- `StartVM` requires an absolute Linux kernel path and rootfs path. +- `buildVM` constructs `VZLinuxBootLoader(kernelURL:)`. +- boot args pass `root=/dev/vda`, `init=/sbin/cleanroom-init`, and the + Cleanroom guest port. +- networking relies on a static guest IP carried through Linux boot args. + +That code is a useful foundation because the helper process, lifecycle calls, +filehandle networking, and socket proxy model already exist. It is not a +macOS guest implementation. + +### Apple Virtualization.framework + +The macOS 26.5 SDK on this host includes the macOS guest APIs Cleanroom would +need: + +- `VZMacOSBootLoader` +- `VZMacPlatformConfiguration` +- `VZMacOSRestoreImage` +- `VZMacOSInstaller` +- `VZMacAuxiliaryStorage` +- `VZMacOSVirtualMachineStartOptions` +- `VZVirtioFileSystemDeviceConfiguration` +- `VZVirtioSocketDeviceConfiguration` + +The local host is Apple Silicon on macOS 26.3, and +`VZVirtualMachine.isSupported` returned `true`. Fetching Apple's latest restore +image catalog failed locally with `VZErrorRestoreImageCatalogLoadFailed`, so +the prototype should not depend on that catalog path as its only image source. +It should accept a local IPSW or a prebuilt VM bundle. + +Apple's public docs describe the same model: macOS guests use +`VZMacPlatformConfiguration`, `VZMacOSBootLoader`, CPU/memory requirements from +`VZMacOSConfigurationRequirements`, a main storage device, and input/graphics +devices. Installing from a restore image is done with `VZMacOSInstaller`. +VirtioFS can automount shared host directories inside macOS guests under +`/Volumes/My Shared Files`. + +### Tart Replacement Surface + +Tart is the right comparison point because it already packages this stack for +CI: + +- it publishes macOS 12 through macOS 26 VM images, including base and Xcode + variants +- its Buildkite integration runs a pipeline command inside a cloned macOS VM +- it can create macOS VMs from IPSW +- it stores and pulls/pushes VM images through OCI-compatible registries +- it has a guest agent for `tart exec`, clipboard, and disk resize +- it uses VirtioFS directory sharing for host workspace mounts + +I checked Tart source at commit +`5287b597a14773eb9b627ccd0545c675ef8a59f5`. The relevant implementation shape: + +- `Sources/tart/Platform/Darwin.swift` maps macOS guests to + `VZMacOSBootLoader`, `VZMacPlatformConfiguration`, + `VZMacAuxiliaryStorage`, `VZMacGraphicsDeviceConfiguration`, and macOS input + devices. +- `Sources/tart/VM.swift` installs from IPSW with `VZMacOSRestoreImage` and + `VZMacOSInstaller`, then builds a VM configuration with disk, network, + directory sharing, console devices, and a virtio socket. +- `Sources/tart/ControlSocket.swift` bridges a host Unix socket to the guest + agent over `VZVirtioSocketDevice`. +- `Sources/tart/VMDirectory+OCI.swift` pulls and pushes VM config, disk, and + NVRAM layers with Tart-specific media types. + +The conclusion is that the VM mechanics are feasible. The hard work is fitting +them into Cleanroom's stronger sandbox contract. + +## Replacement Boundary + +"Replace Tart" should mean: + +- Buildkite/macOS jobs no longer invoke the `tart` binary. +- Cleanroom owns clone, start, exec, file transfer or workspace mount, + teardown, events, and diagnostics. +- Images are either Cleanroom-native macOS VM images or imported from an + existing Tart-compatible image source. +- The user-facing API stays Cleanroom-shaped: repository policy, runtime + config, capabilities, `cleanroom exec`, and the control API. + +It should not mean: + +- implement every Tart CLI flag +- implement Orchard-style orchestration +- support GUI-first workflows, VNC, audio, clipboard, or Screen Sharing in the + first slice +- support Intel macOS guests or x86 emulation +- copy Tart source code or make Tart's image format the native Cleanroom + contract + +## Target Model + +The public contract should eventually separate guest platform from host +backend. A possible future policy shape: + +```yaml +sandbox: + platform: + os: macos + arch: arm64 + image: + ref: ghcr.io/buildkite/cleanroom/macos-sequoia-xcode@sha256:... +``` + +This is illustrative, not a schema proposal to implement in the first PR. The +important rule is that `macos` is a guest platform, not a new user-facing +backend. Backend-specific details stay in runtime config and adapter internals. + +The first implementation does not need to land the final policy schema. It can +start with an experimental runner that reads a local bundle path, then graduate +to runtime config, and only then expose a repository policy surface once the +backend can enforce the advertised capabilities. That avoids documenting a +copyable `sandbox.platform` example before the loader and capability checks +exist. + +The macOS VM image record should include: + +- macOS version and build +- hardware model data +- minimum CPU and memory +- disk format +- guest-agent version and capabilities +- disk image digest +- auxiliary-storage digest +- optional source IPSW metadata + +Each sandbox should clone the base disk and auxiliary storage into a +sandbox-owned bundle before boot. It should generate unique runtime identity +where Apple requires it, including MAC address and machine identifier handling, +and reject concurrent launches that would reuse unsafe identity. + +### Local Bundle Contract + +The first harness should accept a local bundle rather than OCI. The bundle is a +directory with a small metadata file and paths relative to that directory. A +strawman shape: + +```json +{ + "schema_version": 1, + "os": "macos", + "arch": "arm64", + "macos_version": "15.5", + "macos_build": "24F74", + "vcpus": 4, + "memory_mib": 8192, + "disk": "disk.img", + "auxiliary_storage": "auxiliary.storage", + "hardware_model": "hardware-model.bin", + "machine_identifier": "machine-identifier.bin", + "agent": { + "transport": "virtio_socket", + "port": 10700, + "version": "0.1.0" + }, + "display": { + "width_px": 1024, + "height_px": 768, + "pixels_per_inch": 72 + } +} +``` + +The exact fields can change after the first probe, but the runner should fail +closed when required identity, disk, auxiliary storage, or agent metadata is +missing. Keeping this metadata explicit gives the later OCI and image-import +work a concrete target instead of burying assumptions in filenames. + +### Helper Boundary + +The production helper should grow an explicit macOS start operation instead of +overloading the existing Linux `StartVM` request: + +- `StartLinuxVM` or the current `StartVM` path keeps `kernel_path`, + `rootfs_path`, Linux boot args, and Linux guest networking. +- `StartMacOSVM` takes bundle-derived disk, auxiliary storage, hardware model, + machine identity policy, directory shares, socket-device port, and macOS + start options. + +The first benchmark harness can duplicate a small amount of Swift setup code to +learn quickly. The production integration should converge on one helper binary +so signing, entitlements, lifecycle operations, proxy sockets, and diagnostics +stay in one place. + +## Guest Agent + +The Linux `cleanroom-guest-agent` cannot be reused as-is because it owns Linux +init behavior, ext4 rootfs assumptions, and Linux service startup. macOS needs +a LaunchDaemon or LaunchAgent package installed into the image. + +The first macOS agent should support: + +- readiness probe +- non-interactive command execution +- stdout/stderr streaming +- stdin streaming +- environment and working directory +- exit code reporting +- path stat/read/write operations needed for workspace setup and artifact copy + +TTY, clipboard, GUI session automation, keychain operations, and disk resize can +come later. + +The host transport should prefer the existing virtio-socket pattern. If the +macOS agent can implement the existing `vsockexec` framing cleanly, the Go +control path stays smaller. If gRPC is faster to implement on macOS, hide it +behind the backend adapter and keep the control-service contract unchanged. + +Tart's guest-agent design is useful evidence here: recent Tart images run +commands without SSH by using a Go guest agent and a bidirectional gRPC exec +stream over the VZ socket path. Cleanroom does not need to copy that protocol, +but it should copy the product boundary: runtime command execution must be a +guest-agent capability, not an SSH/network side channel. + +## Networking + +Networking is the biggest product-risk area. + +Tart can use NAT, bridged networking, or Softnet. Cleanroom's value is stronger +than that: repo-scoped policy, deny-by-default egress, host-held credentials, +and gateway mediation. + +The current `darwin-vz` filehandle gateway is the right long-term direction +because it already owns DNS, TCP proxying, gateway access, and policy swapping. +The missing macOS-guest piece is address configuration. Today the Linux guest +gets static network details through boot args. A macOS guest cannot use that +path. We need one of: + +- DHCP support in the filehandle gateway +- a guest-agent network setup operation before workload execution +- a preconfigured static network profile in the base image, parameterized per + sandbox + +Using VZ NAT for the first boot/exec prototype is acceptable only if the +backend reports degraded capabilities and requires an explicit allow-all mode. +It should not be called a Cleanroom replacement for Tart until the filehandle +path can enforce the same policy model as Linux Cleanrooms. + +## Image And Storage Strategy + +There are two viable tracks: + +1. Cleanroom-native macOS VM images. + - Define Cleanroom media types for VM metadata, disk, and auxiliary storage. + - Require digest-pinned refs for reproducible CI. + - Use APFS clonefile or ASIF/raw sparse disk copy for fast local clones. + - Keep image build/import tooling in Cleanroom. + +2. Tart image import for migration. + - Read Tart OCI media types as an importer. + - Convert to a Cleanroom-native local bundle before execution. + - Do not make Tart's format the long-term Cleanroom API. + +The first prototype can skip OCI entirely by accepting a local VM bundle. That +keeps the first proof focused on boot, exec, and teardown. + +## Current Progress Snapshot + +Slice 1 is partially implemented in `benchmarks/darwin-vz/macos-minimal`. +Current evidence proves that the host and SDK expose the required +Virtualization.framework APIs, the existing Cleanroom `darwin-vz` backend has +reusable helper/control-plane pieces, Tart proves the macOS VM plus guest-agent +model works in practice, and the standalone macOS harness can compile and +validate bundle metadata paths. The benchmark directory also has a local +IPSW-to-bundle creator that installs macOS, writes VZ identity files, and emits +the runner's `bundle.json` shape. + +The worktree now has a minimal `darwin/arm64` macOS guest agent command, +LaunchDaemon template, package builder, setup viewer, and offline prepare +script that clones a base bundle, mounts the clone's APFS Data volume, installs +the agent, marks setup complete, and updates `bundle.json`. The prepare script +fails closed if it cannot set root ownership, with an explicit inspection-only +override for rootless experiments. On the local macOS 26.5 build 25F71 bundle, +metadata validation succeeds after rootless preparation, but the live +`sw_vers` smoke still times out connecting to the guest vsock port. The +rootless offline install cannot set root ownership on the agent or LaunchDaemon +plist, and the guest did not create agent stdout/stderr logs during boot. + +The package builder now produces `dist/cleanroom-macos-guest-agent.pkg` as a +script-only installer: its postinstall runs inside the guest as root, writes the +agent and LaunchDaemon plist as `root:wheel`, and bootstraps the LaunchDaemon +when the target is the running system. A host-side attempt to apply that package +to a mounted clone's Data volume blocked in `installerauthagent`, so this is +not a noninteractive host-finalization path without administrator +authorization. The setup viewer boots a bundle in a `VZVirtualMachineView` +window and can expose `dist/` through the macOS guest automount tag so the +package is available at `/Volumes/My Shared Files/cleanroom-macos-guest-agent.pkg`. +The viewer-owned screenshot path works and showed the local prepared image +stopping at loginwindow with generic name/password fields. + +The rootless harness now has a repeatable user-session fallback: +`prepare-agent-bundle.sh --install-mode user-cron` creates a local `cleanroom` +admin user, installs the agent under `/Users/cleanroom/bin`, and writes a user +crontab that starts the agent over virtio socket. This avoids the root-owned +LaunchDaemon requirement for the standalone probe. A freshly prepared local +macOS 26.5 build 25F71 bundle passed the live `sw_vers` smoke over the guest +agent with exit code 0, and a shell command confirmed execution as the +`cleanroom` user. This is not the production image-finalization answer yet: +commands run as the CI user, startup can wait for cron's next minute tick, and +the root-owned LaunchDaemon package still needs a privileged in-guest setup +flow before backend integration. + +The next image-prep step is now implemented as +`finalize-agent-bundle.sh`. It creates a temporary rootless `user-cron` +bootstrap bundle, boots it once, uses the bootstrap agent to run `sudo` inside +the guest, installs the agent and LaunchDaemon as `root:wheel`, writes +`/private/var/db/cleanroom-macos-guest-agent.finalized`, and then boots the +bundle again to prove commands are served by the system LaunchDaemon as root. +The finalizer disables autologin for the temporary bootstrap so the bootstrap +user can be removed offline after the root daemon is installed. The offline +cleanup removes the bootstrap dslocal record, home directory, and crontab from +the cloned Data volume before the final smoke boot. The bootstrap cron and +LaunchAgent now check the finalized marker before starting the user agent, so a +failed finalization leaves an inert bootstrap path after the marker is written. + +The same finalizer now has a local GUI profile. `--profile gui` keeps the +bootstrap user as a real autologin account, rewrites that user's LaunchAgent to +serve exec on `user_agent.port`, leaves the root LaunchDaemon on `agent.port`, +and removes only the bootstrap cron entry offline. Its smoke path verifies the +root daemon first, then connects to the user agent, launches TextEdit, and +attempts a guest screenshot with `screencapture`. On the headless runner the +screenshot can be unavailable without an attached VZ view, so the hard smoke +assertion is GUI app launch through the user session. This is evidence that a +GUI-session image profile can run macOS apps; production GUI automation, VNC, +clipboard, TCC management, and backend integration remain separate work. + +The production `cleanroom-darwin-vz` helper now has an explicit +`StartMacOSVM` operation boundary. It keeps the existing Linux `StartVM` +request intact, takes macOS bundle primitives (`disk_path`, +`auxiliary_storage_path`, `hardware_model_path`, and +`machine_identifier_path`), starts the guest with `VZMacOSBootLoader` and +`VZMacPlatformConfiguration`, and exposes the existing proxy socket path for +guest-agent exec over a virtio socket. The operation currently rejects +filehandle/vmnet networking and supports only `network_mode: none`, so the Go +adapter still needs explicit experimental runtime config, bundle cloning, +capability reporting, and fail-closed checks before this becomes a Cleanroom +execution path. + +## Delivery Strategy + +### Slice 1: Local macOS VM boot-and-exec probe + +Add a standalone harness under `benchmarks/darwin-vz/macos-minimal`. This is +the macOS analogue of the existing Linux minimal benchmark: a learning tool, +not a backend. + +Expected files: + +- `benchmarks/darwin-vz/macos-minimal/README.md` +- `benchmarks/darwin-vz/macos-minimal/runner.swift` +- `benchmarks/darwin-vz/macos-minimal/build-runner.sh` +- `benchmarks/darwin-vz/macos-minimal/viewer.swift` +- `benchmarks/darwin-vz/macos-minimal/build-viewer.sh` +- `benchmarks/darwin-vz/macos-minimal/example-bundle.json` + +Current status: these files exist. The runner builds and signs as +`dist/darwin-vz-macos-minimal`, `--help` works, and validation fails closed +when the example bundle points at missing artifacts or invalid +Virtualization.framework identity data. The viewer builds and signs as +`dist/darwin-vz-macos-viewer`; `--help` and invalid bundle errors work, it can +validate and attach a read-only VirtioFS directory share for setup-time package +access, and its own `--screenshot` path can capture the VZ window for +diagnostics. The repository Go suite also passes locally with this harness +present. Live VM validation passed on a rootless `user-cron` prepared bundle: +`/usr/bin/sw_vers` returned macOS 26.5 build 25F71 over the virtio-socket guest +agent, and `/bin/sh -lc 'printf "user=$(id -un) cwd=$PWD\n"'` returned +`user=cleanroom cwd=/Users/cleanroom`. + +Scope: + +- read a local bundle metadata file and resolve relative disk/auxiliary paths +- build a `VZVirtualMachineConfiguration` with `VZMacOSBootLoader`, + `VZMacPlatformConfiguration`, storage, socket device, and minimal console + devices +- sign the runner with the existing virtualization entitlement +- provide a viewer path for setup and manual image finalization +- provide a viewer-owned screenshot path for diagnosing guest boot state +- start the VM without invoking `tart` +- connect to the guest agent over virtio socket +- run `/usr/bin/sw_vers` by default, with a flag for an arbitrary command +- stream stdout/stderr, return the exit code, and write machine-readable + timing JSON to `--metrics` or stderr +- stop the VM cleanly + +Definition of done: + +- `--help` and bundle validation work without a VM bundle +- missing disk, auxiliary storage, hardware model, or agent metadata fails with + a clear error +- live smoke works on an Apple Silicon host when + `CLEANROOM_MACOS_VM_BUNDLE=/path/to/bundle.json` points at a prepared image +- Linux `benchmarks/darwin-vz/minimal` behavior is untouched + +This slice may use VZ NAT only for image bring-up or debugging. The command +path itself should not depend on SSH or guest networking. + +### Slice 2: macOS guest agent package + +Add a minimal guest agent that can be installed into a macOS image. Prefer a +separate `cmd/cleanroom-macos-guest-agent` command at first so Linux init, +Docker, overlay-capture, and ext4 assumptions stay out of the macOS binary. + +Scope: + +- LaunchDaemon plist template for boot-time startup +- readiness RPC +- exec RPC with argv, env, working directory, stdin, stdout, stderr, and exit + code +- path stat/read/write operations needed for workspace setup and artifacts +- version/capabilities RPC + +Definition of done: + +- agent builds for `darwin/arm64` +- LaunchDaemon starts the agent in a prepared image +- the Slice 1 runner can execute `sw_vers` and a shell command without SSH +- agent reports version and capability metadata to the host +- unsupported operations return explicit errors rather than silent success + +Current status: `cmd/cleanroom-macos-guest-agent` builds for `darwin/arm64`, +serves the existing newline-delimited exec stream over stdio for tests and +Darwin AF_VSOCK in the guest, supports `ready`/`version` control requests, and +streams stdout, stderr, stdin EOF, environment, working directory, and exit +status. The LaunchDaemon template and package builder exist, but live launchd +startup is not yet proved because the current offline install path cannot set +root ownership in this non-sudo session. + +### Slice 3: Image bundle creation and import + +Create the smallest image workflow needed to repeat the boot-and-exec smoke +without hand-editing VM directories. + +Scope: + +- document a local bundle layout for disk, auxiliary storage, hardware model, + machine identity policy, and agent metadata +- provide a local import or prepare command that validates an existing macOS VM + bundle and writes normalized Cleanroom metadata +- install or verify the guest agent in the image +- clone the base bundle into a sandbox-owned working bundle before boot + +Current status: `benchmarks/darwin-vz/macos-minimal/create-bundle.swift` and +`build-create-bundle.sh` can build a signed installer helper that accepts a +local Apple Silicon IPSW, creates `disk.img`, `auxiliary.storage`, +`hardware-model.bin`, `machine-identifier.bin`, runs `VZMacOSInstaller`, and +writes `bundle.json`. The tool intentionally stops short of claiming the bundle +is command-runnable because the Cleanroom macOS guest agent still needs to be +installed inside the guest. + +`prepare-agent-bundle.sh` now clones the base bundle, installs the local macOS +guest agent into the clone's APFS Data volume, writes the LaunchDaemon plist, +marks setup complete, and updates `bundle.json` to the installed agent version. +It leaves the base bundle untouched and fails closed when root ownership cannot +be set, unless the caller passes the inspection-only rootless override. +Rootless offline LaunchDaemon installation has not produced a launchd-started +agent yet. +`build-guest-agent-pkg.sh` creates a script-only installer package for an +in-guest finalization path. The package avoids host-side AppleDouble payload +entries and uses postinstall to write root-owned files, then tries to bootstrap +and kickstart the LaunchDaemon. `build-viewer.sh`/`viewer.swift` provide the +manual setup boot path for that package by showing the VM and exposing a host +directory with VirtioFS. + +For the standalone rootless harness, `prepare-agent-bundle.sh --install-mode +user-cron` now creates a local guest user, installs the agent in that user's +home directory, writes setup/autologin preferences, and adds a user crontab +that starts the agent. This path produced a command-runnable bundle from the +same base without mutating it. + +`finalize-agent-bundle.sh` turns that bootstrap into a root-owned +LaunchDaemon-backed bundle without Tart, SSH, Packer, GUI automation, or host +sudo. It leaves the original base untouched, validates the finalized bundle +with a second boot, and fails if the smoke command is not running as uid 0 or +the temporary bootstrap user still exists. Its bootstrap path uses cron only, +not autologin, and it removes the temporary dslocal record offline before the +final smoke boot because macOS can mark the first local account with a secure +token and refuse an in-guest delete. + +`finalize-agent-bundle.sh --profile gui` is the local GUI-session variant. It +keeps a non-admin autologin user, writes `user_agent` metadata into +`bundle.json`, proves the root daemon and user LaunchAgent can both serve exec, +and uses the user agent to launch TextEdit. It attempts guest-side screenshot +capture, but the headless runner does not treat that as required because no VZ +view is attached. This does not change the first production target: +Buildkite-style command execution still comes before GUI automation in the +backend. + +Definition of done: + +- repeated local runs clone from the same base without mutating it +- concurrent runs cannot reuse unsafe identity state +- bundle validation rejects missing or mismatched metadata before VM start +- clone time and disk usage are measured on APFS + +### Slice 4: Experimental backend integration + +Wire the macOS guest path into `darwin-vz` behind explicit experimental +configuration and capability flags. The first runtime config should stay +operator-scoped; repository policy should not advertise macOS guests until +capability checks and validation are in place. + +Possible runtime config: + +```yaml +backends: + darwin-vz: + macos: + enabled: true + bundle: /var/lib/cleanroom/macos-images/sequoia-xcode/bundle.json +``` + +Scope: + +- add a macOS-specific helper operation such as `StartMacOSVM` +- add backend adapter code that selects the macOS path only for explicit + experimental macOS requests +- keep `ProvisionSandbox`, `RunInSandbox`, and `TerminateSandbox` as the public + backend interface +- publish separate capability gaps for macOS guest support in + `cleanroom doctor --json` +- fail closed when a policy or command needs unsupported capabilities + +Definition of done: + +- normal Linux `darwin-vz` behavior and tests are unchanged +- an experimental macOS sandbox can run a command through Cleanroom control + flow without Tart +- unsupported file, network, snapshot, cache-output, or Docker capabilities + fail before execution +- observability includes guest platform, image metadata, startup timings, and + agent version + +Current status: the helper operation boundary is implemented, documented, and +represented in the Go helper-control request type. The benchmark directory now +also has a helper-backed smoke runner that starts `cleanroom-darwin-vz`, sends +`StartMacOSVM`, connects to the helper-managed proxy socket, and can run a +GUI-profile bundle's user-session agent to launch a GUI app. The remaining +slice 4 work is the Go adapter/runtime-config path that clones a prepared local +bundle, sends `StartMacOSVM` from the production backend, runs the macOS guest +agent through the existing proxy socket, reports macOS-specific capability +gaps, and fails closed for unsupported operations. + +### Slice 5: Policy-compatible networking + +Extend the filehandle gateway or guest provisioning path so macOS guests use a +Cleanroom-owned network path. + +Scope: + +- decide between DHCP in the filehandle gateway, guest-agent network setup, or + preconfigured static profiles +- ensure DNS, TCP proxying, gateway access, and stage policy swaps behave like + Linux `darwin-vz` +- close active TCP proxy connections when the active policy changes + +Definition of done: + +- deny-by-default policy blocks arbitrary TCP egress +- allowed hosts work through the existing DNS/TCP authorization model +- stage-scoped policy swaps close active TCP proxy connections +- host gateway access works through a stable name +- conformance tests cover blocked and allowed traffic + +### Slice 6: Image lifecycle and Buildkite migration + +Add enough image lifecycle to replace the Tart Buildkite plugin for the target +workflow. + +Scope: + +- choose Cleanroom-native image publishing as the product path +- add Tart image import only as migration tooling if existing Cirrus/Tart + images are required +- publish or import a prebuilt Xcode image with the Cleanroom guest agent +- run a Buildkite command inside the VM without invoking `tart` +- capture artifacts and workspace writes through the chosen copy/mount model + +Definition of done: + +- the target Buildkite job runs with no `tart` binary in the command path +- startup, clone time, command runtime, cleanup time, and disk usage are + measured against the current Tart workflow +- logs and failure diagnostics are good enough to operate in CI +- remaining capability gaps are visible in `doctor --json` and user-facing + errors + +## Verification Plan + +- Document/static checks: `git diff --check` and a README smoke for every new + command or bundle format. +- Swift compile/signing: `xcrun swiftc -framework Virtualization` and + `codesign --entitlements cmd/cleanroom-darwin-vz/entitlements.plist` for the + experimental runner and helper changes. +- Host support check: fail early when `VZVirtualMachine.isSupported` is false, + the host is not Apple Silicon, or required macOS guest APIs are unavailable. +- Bundle validation tests: missing disk, auxiliary storage, hardware model, + unsupported arch, missing agent port, and unsafe identity reuse. +- Agent tests: exec success, non-zero exit, stdin/stdout/stderr streaming, + env/working directory handling, readiness, and unsupported operation errors. +- Backend unit tests: Linux `darwin-vz` path remains selected by default, + experimental macOS config is required, capability gaps are reported, and + unsupported policy features fail closed. +- Live smoke: with `CLEANROOM_MACOS_VM_BUNDLE` set, start a macOS VM, run + `/usr/bin/sw_vers`, run a shell command that writes an artifact, and stop the + VM cleanly. +- Network conformance: after Slice 5, run allowed/blocked TCP and stage-scoped + policy tests against the macOS guest path before claiming Tart replacement + parity. + +## Key Learnings From Pressure-Testing + +Booting macOS is not the hard part. The risky parts are the Cleanroom-specific +parts Tart does not solve: policy-grade networking, host-side credential +mediation, deterministic image identity, and a backend-neutral control API. + +Treating VZ NAT plus SSH as "macOS Cleanrooms" would be too weak. It may prove +that a VM can boot, but it does not prove Cleanroom can replace Tart for a +policy-governed sandbox. + +Image management can become the largest maintenance burden. If the actual goal +is only to run existing Tart images, an importer is pragmatic. If the goal is a +Cleanroom product surface, use a Cleanroom-native image format and keep Tart +compatibility as migration tooling. + +The first PR should not modify the production `darwin-vz` Linux path. A small +standalone harness gives us a fast way to learn about macOS guest boot, +virtio-socket agent behavior, and image bundle metadata without destabilizing +the existing backend. + +## Resolved Decisions And Defaults + +- First replacement target: Buildkite-style macOS CI command execution. Local + developer sandboxes and GUI workflows can benefit later, but they should not + shape the first supported surface. +- Local GUI proof: a harness-only GUI image profile is acceptable before + backend integration as evidence for app/session mechanics. It should not + imply production support for GUI automation or user-session policy yet. +- First PR boundary: standalone `benchmarks/darwin-vz/macos-minimal` harness, + not production backend integration. +- Runtime dependency: no `tart` binary in the execution path. Temporary use of + Tart-created images is acceptable only as input to an importer or local + bundle migration step. +- Command transport: guest agent over virtio socket, not SSH. +- First probe protocol: keep the existing `vsockexec` newline-delimited JSON + shape for exec, with small `ready` and `version` control requests for the + macOS agent. +- Networking default: VZ NAT is acceptable for a local boot probe, but not for + claimed Cleanroom parity. +- Public shape: guest platform belongs in policy or request shape; backend + selection remains runtime/operator configuration. +- Image direction: prefer Cleanroom-native metadata and digests; add Tart import + as migration tooling if it materially reduces adoption cost. + +## Open Questions + +Questions that block Slice 1: + +- What installation path should make the LaunchDaemon acceptable to launchd? + Current answer: use `finalize-agent-bundle.sh` for the standalone harness. + It uses `--install-mode user-cron` only as a temporary bootstrap, then + performs the root-owned LaunchDaemon install from inside the running guest. + +Questions before backend integration: + +- Should repository policy grow `sandbox.platform` directly, or should the + first integrated path use an internal request/runtime flag? Recommended + default: runtime flag first, policy schema only after capability checks are + enforceable. +- How should macOS guest identity be regenerated or fenced per sandbox? + Recommended default: require clone-time identity handling in the bundle tool + and reject concurrent runs if safety cannot be proven. + +Questions before Tart replacement parity: + +- Do we need to run existing Cirrus/Tart images directly, or is building new + Cleanroom macOS images acceptable? +- What macOS image build pipeline owns Xcode/base image updates? +- What legal and capacity constraints should the scheduler enforce for macOS + guests on shared Apple hardware? diff --git a/internal/backend/darwinvz/helper_client.go b/internal/backend/darwinvz/helper_client.go index 8ee0c664..5d8c47ba 100644 --- a/internal/backend/darwinvz/helper_client.go +++ b/internal/backend/darwinvz/helper_client.go @@ -34,6 +34,10 @@ type helperControlRequest struct { KernelPath string `json:"kernel_path,omitempty"` RootFSPath string `json:"rootfs_path,omitempty"` SidecarDiskPaths []string `json:"sidecar_disk_paths,omitempty"` + DiskPath string `json:"disk_path,omitempty"` + AuxiliaryStoragePath string `json:"auxiliary_storage_path,omitempty"` + HardwareModelPath string `json:"hardware_model_path,omitempty"` + MachineIdentifierPath string `json:"machine_identifier_path,omitempty"` BootArgs string `json:"boot_args,omitempty"` NetworkMode string `json:"network_mode,omitempty"` VMNetSubnetCIDR string `json:"vmnet_subnet_cidr,omitempty"` @@ -53,6 +57,9 @@ type helperControlRequest struct { ProxySocketPath string `json:"proxy_socket_path,omitempty"` ConsoleLogPath string `json:"console_log_path,omitempty"` VMID string `json:"vm_id,omitempty"` + DisplayWidthPx int64 `json:"display_width_px,omitempty"` + DisplayHeightPx int64 `json:"display_height_px,omitempty"` + DisplayPixelsPerInch int64 `json:"display_pixels_per_inch,omitempty"` } type helperControlResponse struct { diff --git a/internal/backend/darwinvz/helper_client_test.go b/internal/backend/darwinvz/helper_client_test.go index d9541c45..182256ac 100644 --- a/internal/backend/darwinvz/helper_client_test.go +++ b/internal/backend/darwinvz/helper_client_test.go @@ -13,6 +13,51 @@ import ( "github.com/buildkite/cleanroom/internal/backend" ) +func TestHelperControlRequestEncodesMacOSVMFields(t *testing.T) { + payload, err := json.Marshal(helperControlRequest{ + Op: "StartMacOSVM", + DiskPath: "/tmp/macos/disk.img", + AuxiliaryStoragePath: "/tmp/macos/auxiliary.storage", + HardwareModelPath: "/tmp/macos/hardware-model.bin", + MachineIdentifierPath: "/tmp/macos/machine-identifier.bin", + NetworkMode: "none", + VCPUs: 4, + MemoryMiB: 8192, + GuestPort: 10700, + LaunchSeconds: 120, + RunDir: "/tmp/macos/run", + ProxySocketPath: "/tmp/macos/run/vz-proxy.sock", + DisplayWidthPx: 1024, + DisplayHeightPx: 768, + DisplayPixelsPerInch: 72, + }) + if err != nil { + t.Fatalf("marshal helper request: %v", err) + } + + var decoded map[string]any + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("decode helper request: %v", err) + } + + for _, key := range []string{ + "disk_path", + "auxiliary_storage_path", + "hardware_model_path", + "machine_identifier_path", + "display_width_px", + "display_height_px", + "display_pixels_per_inch", + } { + if _, ok := decoded[key]; !ok { + t.Fatalf("expected encoded key %q in %s", key, string(payload)) + } + } + if got := decoded["op"]; got != "StartMacOSVM" { + t.Fatalf("op = %v, want StartMacOSVM", got) + } +} + func TestHelperRequestDecodeErrorIsLifecycleIndeterminate(t *testing.T) { clientConn, serverConn := net.Pipe() defer clientConn.Close() diff --git a/mise.toml b/mise.toml index 57165745..aee6ce29 100644 --- a/mise.toml +++ b/mise.toml @@ -16,7 +16,7 @@ run = "go test ./..." [tasks.lint-shell] description = "Lint shell scripts" -run = "shellcheck -x .buildkite/hooks/pre-command scripts/base-image-tag.sh scripts/cleanroom-root-helper.sh scripts/benchmark-tti.sh scripts/build-go.sh scripts/install-go.sh scripts/install.sh scripts/install-global.sh scripts/package-darwin-vz-helper.sh scripts/release.sh scripts/e2e-observability.sh scripts/ci-with-host-lock.sh scripts/ci-go-test-engine.sh scripts/ci-auth-oidc-smoke.sh scripts/ci-example-smoke.sh scripts/ci-examples-firecracker.sh scripts/ci-examples-darwin-vz.sh scripts/ci-cleanroom-e2e.sh scripts/ci-darwin-vz-e2e.sh scripts/ci-darwin-vz-filehandle-e2e.sh scripts/build-macos-release-pkg.sh scripts/notarize-macos-package.sh scripts/ci-macos-release-pkg.sh scripts/ci-buildkite-release.sh" +run = "shellcheck -x .buildkite/hooks/pre-command scripts/base-image-tag.sh scripts/cleanroom-root-helper.sh scripts/benchmark-tti.sh benchmarks/darwin-vz/macos-minimal/build-runner.sh benchmarks/darwin-vz/macos-minimal/build-viewer.sh benchmarks/darwin-vz/macos-minimal/build-create-bundle.sh benchmarks/darwin-vz/macos-minimal/build-guest-agent.sh benchmarks/darwin-vz/macos-minimal/build-guest-agent-pkg.sh benchmarks/darwin-vz/macos-minimal/prepare-agent-bundle.sh benchmarks/darwin-vz/macos-minimal/finalize-agent-bundle.sh scripts/build-go.sh scripts/install-go.sh scripts/install.sh scripts/install-global.sh scripts/package-darwin-vz-helper.sh scripts/release.sh scripts/e2e-observability.sh scripts/ci-with-host-lock.sh scripts/ci-go-test-engine.sh scripts/ci-auth-oidc-smoke.sh scripts/ci-example-smoke.sh scripts/ci-examples-firecracker.sh scripts/ci-examples-darwin-vz.sh scripts/ci-cleanroom-e2e.sh scripts/ci-darwin-vz-e2e.sh scripts/ci-darwin-vz-filehandle-e2e.sh scripts/build-macos-release-pkg.sh scripts/notarize-macos-package.sh scripts/ci-macos-release-pkg.sh scripts/ci-buildkite-release.sh" [tasks.lint-proto] description = "Lint protobuf schemas" diff --git a/scripts/buildkite_pipeline_test.go b/scripts/buildkite_pipeline_test.go index 20ee5182..63fa6b0c 100644 --- a/scripts/buildkite_pipeline_test.go +++ b/scripts/buildkite_pipeline_test.go @@ -281,6 +281,8 @@ func TestMiseLintShellCoversSharedE2EObservabilityHelper(t *testing.T) { `[tasks.lint-shell]`, `.buildkite/hooks/pre-command`, `scripts/base-image-tag.sh`, + `benchmarks/darwin-vz/macos-minimal/build-runner.sh`, + `benchmarks/darwin-vz/macos-minimal/build-viewer.sh`, `scripts/e2e-observability.sh`, `scripts/ci-with-host-lock.sh`, `scripts/ci-go-test-engine.sh`,