diff --git a/.eslintrc.json b/.eslintrc.json index 18724e1..8e67fb9 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -57,7 +57,7 @@ "**/runner-release", "**/runner-release/**" ], - "message": "src/harness is pure: the runner-release verifier (Node crypto) imports it, never the other way." + "message": "src/harness is pure: src/runner-release imports it, never the other way." } ] } @@ -220,7 +220,7 @@ "**/runner-release", "**/runner-release/**" ], - "message": "Runner releases are verified in the main process (src/runner-release); the renderer and preload never import it." + "message": "src/runner-release uses Node built-ins (the main process and the runner import it); the renderer and preload never do." } ] } @@ -257,7 +257,7 @@ "**/channel", "**/channel/**" ], - "message": "src/runner-release may import only src/harness, itself and node:crypto." + "message": "src/runner-release may import only src/harness, itself and the Node built-ins .eslintrc.json allows it (no child_process: nothing here runs a package)." } ] } @@ -266,7 +266,11 @@ "error", { "allow": [ - "node:crypto" + "node:crypto", + "node:fs", + "node:path", + "node:stream", + "node:zlib" ] } ] diff --git a/scripts/package-runner.mjs b/scripts/package-runner.mjs index 289a973..c56ebe5 100644 --- a/scripts/package-runner.mjs +++ b/scripts/package-runner.mjs @@ -16,6 +16,11 @@ * config.sh run.sh svc.sh VERSION README.md LICENSE * bin/node bin/node.LICENSE bin/puck-runner.cjs * + * That layout, with its modes, is RUNNER_PACKAGE_ENTRIES in + * src/harness/runner-releases.ts: the entries below must match it, since + * src/runner-release/archive.ts unpacks nothing else (a unit test reads + * this script's output through it). + * * The Node runtime is the pinned release below, downloaded once into * .cache/runner-node/ and checked against the pinned sha256 before use. * Archives are written by the runner's own ustar writer with fixed owners diff --git a/src/harness/runner-releases.ts b/src/harness/runner-releases.ts index 70d07f7..7152737 100644 --- a/src/harness/runner-releases.ts +++ b/src/harness/runner-releases.ts @@ -20,6 +20,15 @@ * never re-serialises a manifest to check it. The manifest names SHA256SUMS * by digest, so the two agree or the release is refused. * + * A signed release travels through other documents (a server's listing) + * as a record carrying the manifest's exact bytes and its signature, both + * in standard base64 (`SignedRunnerRelease`). The reader here checks the + * encoding and its bounds; the bytes go to the verifier unchanged. + * + * Each package unpacks into the fixed layout `RUNNER_PACKAGE_ENTRIES`, as + * scripts/package-runner.mjs writes it; src/runner-release/archive.ts + * accepts nothing else. + * * Only erasable TypeScript here: the build scripts load this file with * Node's type stripping. */ @@ -39,6 +48,10 @@ export const ED25519_SIGNATURE_BYTES = 64; export const MAX_MANIFEST_BYTES = 256 * 1024; /** Most bytes a SHA256SUMS file may have. */ export const MAX_SUMS_BYTES = 16 * 1024; +/** Most signed releases one listing carries. */ +export const MAX_RELEASE_RECORDS = 16; +/** Most bytes a whole release listing may have on the wire; readers refuse a larger body before parsing it. */ +export const MAX_RELEASE_METADATA_BYTES = 4 * 1024 * 1024; /** * How a runner build treats packages, compiled in by scripts/build-runner.mjs. @@ -74,6 +87,45 @@ export function runnerPackageFile(target: RunnerTarget, version: string): string return `puck-runner-${target.os}-${target.arch}-${version}.tar.gz`; } +/** The Git tag of a release, and so the GitHub Release its assets hang off (RELEASE.md). */ +export const runnerReleaseTag = (version: string): string => `v${version}`; + +/** + * The one mapping from Node's platform and architecture names to the + * manifest's: `darwin` is `macos` in package names. Null for anything + * runners do not run on. + */ +export function runnerTargetFor(platform: string, arch: string): RunnerTarget | null { + const os: RunnerOs | null = platform === 'linux' ? 'linux' : platform === 'darwin' ? 'macos' : null; + const a: RunnerArch | null = arch === 'x64' || arch === 'arm64' ? arch : null; + return os && a ? { os, arch: a } : null; +} + +export interface RunnerPackageEntry { + /** Relative path; a directory's ends with '/'. */ + name: string; + type: 'file' | 'dir'; + /** The permission bits the packager writes and the archive reader restores; never setuid, setgid or sticky. */ + mode: number; +} + +/** + * What a package holds, in the order scripts/package-runner.mjs writes it: + * nine regular files and `bin/`, nothing else, with these exact modes. + */ +export const RUNNER_PACKAGE_ENTRIES: readonly RunnerPackageEntry[] = [ + { name: 'config.sh', type: 'file', mode: 0o755 }, + { name: 'run.sh', type: 'file', mode: 0o755 }, + { name: 'svc.sh', type: 'file', mode: 0o755 }, + { name: 'VERSION', type: 'file', mode: 0o644 }, + { name: 'README.md', type: 'file', mode: 0o644 }, + { name: 'LICENSE', type: 'file', mode: 0o644 }, + { name: 'bin/', type: 'dir', mode: 0o755 }, + { name: 'bin/node', type: 'file', mode: 0o755 }, + { name: 'bin/node.LICENSE', type: 'file', mode: 0o644 }, + { name: 'bin/puck-runner.cjs', type: 'file', mode: 0o644 }, +]; + export interface RunnerReleaseAsset { os: RunnerOs; arch: RunnerArch; @@ -313,3 +365,71 @@ export function readVersionProbe(text: string): RunnerVersionProbe { if (!isPositiveSafeInteger(runnerProtocol)) throw malformed(`The runner reports protocol ${JSON.stringify(runnerProtocol)}.`); return { version, trustMode, runnerProtocol }; } + +/** + * A signed release as a listing carries it: runner-release.json's exact + * bytes and its detached signature, each standard base64 with padding. + */ +export interface SignedRunnerRelease { + manifest: string; + signature: string; +} + +/** Base64 text of a maximal manifest: the byte bound, accounting for the encoding. */ +export const MAX_MANIFEST_BASE64_CHARS = Math.ceil(MAX_MANIFEST_BYTES / 3) * 4; +const SIGNATURE_BASE64_CHARS = Math.ceil(ED25519_SIGNATURE_BYTES / 3) * 4; +const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; +const RECORD_KEYS = ['manifest', 'signature'] as const; + +/** + * Decodes canonical base64: the standard alphabet with padding, no + * whitespace, and exactly one encoding of the bytes (a re-encoding must + * give the text back). `maxChars` bounds the text before it is decoded. + */ +function decodeBase64(value: unknown, maxChars: number, what: string): Uint8Array { + if (typeof value !== 'string') throw malformed(`${what} must be a base64 string.`); + if (value.length > maxChars) throw malformed(`${what} is over ${maxChars} characters of base64.`); + if (!BASE64_RE.test(value)) throw malformed(`${what} is not standard base64.`); + const binary = atob(value); + if (btoa(binary) !== value) throw malformed(`${what} is not canonical base64.`); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} + +/** Standard base64 with padding, the encoding readSignedRunnerRelease accepts. */ +function encodeBase64(bytes: Uint8Array): string { + let binary = ''; + for (let i = 0; i < bytes.length; i += 0x2000) binary += String.fromCharCode(...bytes.subarray(i, i + 0x2000)); + return btoa(binary); +} + +/** + * Reads one signed release record strictly: exactly its two fields, the + * manifest at most MAX_MANIFEST_BYTES once decoded and the signature + * exactly ED25519_SIGNATURE_BYTES. The bytes come back as sent, for the + * verifier (src/runner-release/verify.ts); nothing here trusts them. + */ +export function readSignedRunnerRelease(value: unknown): { manifest: Uint8Array; signature: Uint8Array } { + if (!isObject(value)) throw malformed('A signed release record must be an object.'); + exactKeys(value, RECORD_KEYS, 'A signed release record'); + const manifest = decodeBase64(value.manifest, MAX_MANIFEST_BASE64_CHARS, "A signed release record's manifest"); + if (manifest.length > MAX_MANIFEST_BYTES) throw malformed(`A signed release record's manifest is over ${MAX_MANIFEST_BYTES} bytes.`); + const signature = decodeBase64(value.signature, SIGNATURE_BASE64_CHARS, "A signed release record's signature"); + if (signature.length !== ED25519_SIGNATURE_BYTES) { + throw malformed(`A signed release record's signature is ${signature.length} bytes, not ${ED25519_SIGNATURE_BYTES}.`); + } + return { manifest, signature }; +} + +/** Reads a listing's records: an array of at most MAX_RELEASE_RECORDS, each read strictly. */ +export function readSignedRunnerReleases(value: unknown): { manifest: Uint8Array; signature: Uint8Array }[] { + if (!Array.isArray(value)) throw malformed('Signed release records must be an array.'); + if (value.length > MAX_RELEASE_RECORDS) throw malformed(`A listing carries at most ${MAX_RELEASE_RECORDS} signed releases; this one has ${value.length}.`); + return value.map(readSignedRunnerRelease); +} + +/** The record for a manifest's exact bytes and its signature. */ +export function formatSignedRunnerRelease(manifest: Uint8Array, signature: Uint8Array): SignedRunnerRelease { + return { manifest: encodeBase64(manifest), signature: encodeBase64(signature) }; +} diff --git a/src/main/server/api.ts b/src/main/server/api.ts index e9e4bc3..f77a827 100644 --- a/src/main/server/api.ts +++ b/src/main/server/api.ts @@ -4,6 +4,7 @@ * read leniently (src/harness/server-api.ts). */ +import { MAX_RELEASE_METADATA_BYTES } from '../../harness/runner-releases'; import { readInstance, readReleases, @@ -103,6 +104,7 @@ export async function forgetInstance(envId: string): Promise { await authed('DELETE', `/v1/instances/${encodeURIComponent(envId)}`); } +/** The server's runner releases, read within the listing bound (src/runner-release/download.ts). */ export async function releases(): Promise { - return readReleases(await serverRequest('GET', '/v1/runner/releases')); + return readReleases(await serverRequest('GET', '/v1/runner/releases', { maxBodyBytes: MAX_RELEASE_METADATA_BYTES, refuseContentEncoding: false })); } diff --git a/src/main/server/http.ts b/src/main/server/http.ts index 39dd8ec..fb22c0b 100644 --- a/src/main/server/http.ts +++ b/src/main/server/http.ts @@ -15,6 +15,7 @@ */ import { shell } from 'electron'; +import { readBoundedBody } from '../../runner-release/download'; export const SERVER_URL_ENV = 'PUCK_SERVER_URL'; export const DEFAULT_SERVER_URL = 'http://localhost:8765'; @@ -80,9 +81,22 @@ export interface RequestOptions { body?: unknown; token?: string; timeoutMs?: number; + /** + * Bounds the answer through the runner-release transport: a longer body + * is refused before parsing. Release listings set it; other requests + * read as before. + */ + maxBodyBytes?: number; + /** With maxBodyBytes. Listings pass false and keep a content encoding. */ + refuseContentEncoding?: boolean; } -/** One JSON request; resolves with the parsed body (null for 204), throws ServerApiError otherwise. */ +/** + * One JSON request; resolves with the parsed body (null for 204). + * A non-2xx answer is ServerApiError, and an unreachable server is + * ServerUnreachableError. A bounded body (`maxBodyBytes`) throws + * RunnerDownloadError from the runner-release transport before parsing. + */ export async function serverRequest>(method: string, path: string, opts: RequestOptions = {}): Promise { const base = serverUrl(); const headers: Record = { Accept: 'application/json' }; @@ -99,7 +113,10 @@ export async function serverRequest>(method: string, } catch (err) { throw new ServerUnreachableError(base, err); } - const text = await res.text().catch(() => ''); + const text = + opts.maxBodyBytes === undefined + ? await res.text().catch(() => '') + : new TextDecoder().decode(await readBoundedBody(res, opts.maxBodyBytes, { refuseContentEncoding: opts.refuseContentEncoding })); let parsed: unknown = null; if (text) { try { diff --git a/src/puck-runner/api.ts b/src/puck-runner/api.ts index 239ae1c..5dafd22 100644 --- a/src/puck-runner/api.ts +++ b/src/puck-runner/api.ts @@ -9,11 +9,14 @@ * * Errors carry the server's status and code. Two codes mean "stop and act": * `runner-removed` (403: the runner was removed; exit for good) and - * `runner-outdated` (426: update first). + * `runner-outdated` (426: update first). A release listing over the bound + * throws RunnerDownloadError (src/runner-release/download.ts) before parsing. */ import { Readable } from 'node:stream'; import type { GithubGrant } from '../harness/daemon-protocol'; +import { MAX_RELEASE_METADATA_BYTES } from '../harness/runner-releases'; +import { readBoundedBody } from '../runner-release/download'; import { signAssertion, type RunnerKey } from './identity'; const REQUEST_TIMEOUT_MS = 30_000; @@ -92,7 +95,8 @@ export class ServerApi { private readonly fetchImpl: Fetch = fetch, ) {} - private async call(method: string, path: string, opts: { body?: unknown; token?: string } = {}): Promise { + /** `maxBytes` bounds the answer through the runner-release transport; a longer body is refused unparsed. Release listings set it. */ + private async call(method: string, path: string, opts: { body?: unknown; token?: string; maxBytes?: number; refuseContentEncoding?: boolean } = {}): Promise { let res: Response; try { res = await this.fetchImpl(this.baseUrl + path, { @@ -107,7 +111,10 @@ export class ServerApi { } catch (err) { throw new ApiError(0, 'unreachable', `Cannot reach the Puck server at ${this.baseUrl}: ${err instanceof Error ? err.message : String(err)}`); } - const text = await res.text(); + const text = + opts.maxBytes === undefined + ? await res.text() + : new TextDecoder().decode(await readBoundedBody(res, opts.maxBytes, { refuseContentEncoding: opts.refuseContentEncoding })); let body: Record = {}; try { body = text ? (JSON.parse(text) as Record) : {}; @@ -147,7 +154,7 @@ export class ServerApi { } releases(): Promise { - return this.call('GET', '/v1/runner/releases'); + return this.call('GET', '/v1/runner/releases', { maxBytes: MAX_RELEASE_METADATA_BYTES, refuseContentEncoding: false }); } /** Streams a download; only URLs on this server are fetched. */ diff --git a/src/puck-runner/configure.ts b/src/puck-runner/configure.ts index 566acf4..cb11e99 100644 --- a/src/puck-runner/configure.ts +++ b/src/puck-runner/configure.ts @@ -26,6 +26,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; +import { runnerTargetFor } from '../harness/runner-releases'; import { ApiError, RunnerRemovedError, ServerApi, type Fetch, type RegisterResponse } from './api'; import type { DockerRunner } from './docker/client'; import { dockerHealth } from './docker/health'; @@ -62,10 +63,9 @@ export interface Platform { arch: 'x64' | 'arm64'; } +/** This machine as a package target (the one mapping is runnerTargetFor in src/harness/runner-releases.ts). */ export function currentPlatform(platform: NodeJS.Platform = process.platform, arch: string = process.arch): Platform | null { - const o = platform === 'linux' ? 'linux' : platform === 'darwin' ? 'macos' : null; - const a = arch === 'x64' || arch === 'arm64' ? arch : null; - return o && a ? { os: o, arch: a } : null; + return runnerTargetFor(platform, arch); } const NAME_RE = /^[A-Za-z0-9][A-Za-z0-9 ._()-]{0,63}$/; diff --git a/src/runner-release/archive.ts b/src/runner-release/archive.ts new file mode 100644 index 0000000..eee0053 --- /dev/null +++ b/src/runner-release/archive.ts @@ -0,0 +1,474 @@ +/** + * Reads a runner package (a gzipped ustar archive, as + * scripts/package-runner.mjs writes one) into a private staging directory, + * accepting only the fixed layout RUNNER_PACKAGE_ENTRIES: nine regular + * files and `bin/`, each once, with the packager's exact modes. Node + * built-ins only; system tar is never run, so what was checked is what + * was written. Nothing here runs or installs anything: a caller activates + * a staging directory itself, after its own checks. + * + * The file is one gzip member and nothing else, framed here rather than by + * zlib's gunzip (which joins concatenated members and ignores trailing + * NULs): a fixed ten-byte header with no optional fields, one raw deflate + * stream that must consume every byte up to the eight-byte trailer, and a + * trailer whose CRC-32 and size must match what was inflated. A second + * member, bytes between the stream and the trailer, or bytes after the + * trailer fail the read. + * + * The deflate stream is inflated as a stream and read block by block. + * Every header must be ustar (magic `ustar\0`, version `00`) with a + * verified checksum, octal fields (no base-256), an empty prefix, link + * name and device numbers, owner names terminated like every text field, + * and a type flag of a regular file or a directory: links, devices, + * FIFOs, PAX and GNU extension entries are refused. A name must be exactly + * one of the layout's; absolute, traversing or otherwise unsafe paths are + * refused before that lookup. Modes must equal the layout's, so setuid, + * setgid and sticky bits never pass. After the last entry come two zero + * blocks and nothing else; a further byte, a missing trailer, a member + * seen twice or a member missing fails the read. Inflated bytes are + * counted against MAX_UNPACKED_BYTES and the compressed file against + * MAX_PACKAGE_BYTES. + * + * Entries are written as they are validated, into a fresh directory of + * mode 0700 under the caller's parent, with the layout's modes. VERSION + * must read exactly the expected version. Any failure removes the + * directory and throws RunnerArchiveError. + * + * Only erasable TypeScript here (see verify.ts). + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { Writable } from 'node:stream'; +import { crc32, createInflateRaw } from 'node:zlib'; +import { RUNNER_PACKAGE_ENTRIES, type RunnerPackageEntry } from '../harness/runner-releases'; +import { MAX_PACKAGE_BYTES, writeAll } from './download'; + +/** Most bytes a package may inflate to. */ +export const MAX_UNPACKED_BYTES = 512 * 1024 * 1024; +/** Most bytes VERSION may have; a version line is a few. */ +const MAX_VERSION_BYTES = 64; +const BLOCK = 512; +/** A gzip member: ID1, ID2, CM, FLG, MTIME(4), XFL, OS; then the deflate stream; then CRC-32 and ISIZE, little-endian. */ +const GZIP_HEADER_BYTES = 10; +const GZIP_TRAILER_BYTES = 8; + +export type RunnerArchiveErrorCode = + /** Not one plain gzip member with a matching trailer, or a corrupt deflate stream. */ + | 'corrupt' + /** A header's checksum, magic, version or fields are wrong. */ + | 'bad-header' + /** An entry the layout has no place for: another name, type or extension entry. */ + | 'unexpected-entry' + /** An absolute or traversing path. */ + | 'unsafe-path' + | 'duplicate-entry' + | 'missing-entry' + /** A mode other than the layout's (setuid, setgid and sticky bits included). */ + | 'bad-mode' + /** Over MAX_UNPACKED_BYTES inflated, or MAX_PACKAGE_BYTES compressed. */ + | 'too-large' + /** The stream ended before the trailer. */ + | 'truncated' + /** Bytes after the two-block tar trailer, or between the deflate stream and the gzip trailer (a second member, say). */ + | 'trailing-data' + /** VERSION is not the expected version. */ + | 'version-mismatch' + /** Staging could not be created or written. */ + | 'write-failed'; + +export class RunnerArchiveError extends Error { + readonly code: RunnerArchiveErrorCode; + + constructor(code: RunnerArchiveErrorCode, message: string) { + super(message); + this.name = 'RunnerArchiveError'; + this.code = code; + } +} + +export interface UnpackOptions { + /** An existing directory the private staging directory is made under. */ + into: string; + /** What VERSION must say. */ + version: string; + /** Test seam; MAX_UNPACKED_BYTES is the contract. */ + maxUnpackedBytes?: number; +} + +export interface UnpackedPackage { + /** The staging directory: mode 0700, holding exactly the layout. */ + dir: string; +} + +const fail = (code: RunnerArchiveErrorCode, message: string): RunnerArchiveError => new RunnerArchiveError(code, message); +const header = (message: string): RunnerArchiveError => fail('bad-header', message); + +/** A NUL-terminated text field; bytes after the terminator must be NUL. */ +function text(block: Buffer, at: number, length: number, what: string): string { + const field = block.subarray(at, at + length); + const end = field.indexOf(0); + if (end !== -1 && field.subarray(end).some((b) => b !== 0)) throw header(`The ${what} field has bytes after its terminator.`); + return (end === -1 ? field : field.subarray(0, end)).toString('latin1'); +} + +/** An octal field: digits, then NUL or space terminators. Base-256 (a GNU extension) is refused. */ +function octal(block: Buffer, at: number, length: number, what: string): number { + const field = block.subarray(at, at + length); + if (field[0] & 0x80) throw header(`The ${what} field is base-256, a GNU extension.`); + let end = length; + while (end > 0 && (field[end - 1] === 0 || field[end - 1] === 0x20)) end--; + const digits = field.subarray(0, end).toString('latin1'); + if (!/^[0-7]{1,11}$/.test(digits)) throw header(`The ${what} field is not octal.`); + return parseInt(digits, 8); +} + +const allZero = (bytes: Buffer): boolean => bytes.every((b) => b === 0); + +/** The fixed gzip header: deflate, and no optional field (zlib writes a package that way). */ +function checkGzipHeader(head: Buffer): void { + if (head[0] !== 0x1f || head[1] !== 0x8b) throw fail('corrupt', 'The package is not a gzip file.'); + if (head[2] !== 8) throw fail('corrupt', `The package's gzip member uses compression method ${head[2]}, not deflate.`); + if (head[3] !== 0) throw fail('corrupt', "The package's gzip header sets flags (an extra field, a name, a comment, a header CRC or a reserved bit); packages carry none."); +} + +/** Exactly `length` bytes at `position` of an open file. */ +function readAt(fd: number, position: number, length: number): Buffer { + const out = Buffer.alloc(length); + let got = 0; + while (got < length) { + const n = fs.readSync(fd, out, got, length - got, position + got); + if (n <= 0) throw fail('corrupt', 'The package file is shorter than its size says.'); + got += n; + } + return out; +} + +/** The typeflags this reader refuses, named for the error. */ +const REFUSED_TYPES: Record = { + '1': 'a hard link', + '2': 'a symbolic link', + '3': 'a character device', + '4': 'a block device', + '6': 'a FIFO', + '7': 'a contiguous file', + x: 'a PAX extended header', + g: 'a PAX global header', + L: 'a GNU long name', + K: 'a GNU long link', +}; + +interface Entry { + spec: RunnerPackageEntry; + size: number; +} + +/** Checks the path before anything is looked up: relative, forward slashes, no `.` or `..`, no empty segment but a directory's last. */ +function checkPath(name: string): void { + if (!name) throw fail('unsafe-path', 'An entry has an empty name.'); + if (name.startsWith('/')) throw fail('unsafe-path', `Entry ${JSON.stringify(name)} is an absolute path.`); + // eslint-disable-next-line no-control-regex + if (name.includes('\\') || /[\u0000-\u001f\u007f]/.test(name)) throw fail('unsafe-path', `Entry ${JSON.stringify(name)} has a backslash or a control character.`); + const segments = name.endsWith('/') ? name.slice(0, -1).split('/') : name.split('/'); + if (segments.some((s) => s === '' || s === '.' || s === '..')) throw fail('unsafe-path', `Entry ${JSON.stringify(name)} has an empty, "." or ".." segment.`); +} + +/** Reads and validates one non-zero header block against the layout. */ +function readHeader(block: Buffer, seen: Set): Entry { + let sum = 0; + for (let i = 0; i < BLOCK; i++) sum += i >= 148 && i < 156 ? 0x20 : block[i]; + if (sum !== octal(block, 148, 8, 'checksum')) throw header('An entry header has a wrong checksum.'); + if (text(block, 257, 6, 'magic') !== 'ustar' || block[262] !== 0 || block.subarray(263, 265).toString('latin1') !== '00') { + throw header('An entry header is not ustar (magic "ustar\\0", version "00").'); + } + const type = String.fromCharCode(block[156]); + const name = text(block, 0, 100, 'name'); + const refused = REFUSED_TYPES[type]; + if (refused) throw fail('unexpected-entry', `Entry ${JSON.stringify(name)} is ${refused}; a package holds regular files and bin/ only.`); + if (type !== '0' && type !== '\0' && type !== '5') throw fail('unexpected-entry', `Entry ${JSON.stringify(name)} has type flag ${JSON.stringify(type)}.`); + if (!allZero(block.subarray(157, 257))) throw header(`Entry ${JSON.stringify(name)} has a link name.`); + // The owner names are read like every text field; the device numbers are NUL, as the packager writes them. + text(block, 265, 32, 'uname'); + text(block, 297, 32, 'gname'); + if (!allZero(block.subarray(329, 345))) throw header(`Entry ${JSON.stringify(name)} has device numbers; a package's headers carry none.`); + if (!allZero(block.subarray(345, 500))) throw header(`Entry ${JSON.stringify(name)} uses the prefix field; package names are short.`); + if (!allZero(block.subarray(500, BLOCK))) throw header(`Entry ${JSON.stringify(name)} has bytes in the header's padding.`); + const mode = octal(block, 100, 8, 'mode'); + octal(block, 108, 8, 'uid'); + octal(block, 116, 8, 'gid'); + const size = octal(block, 124, 12, 'size'); + octal(block, 136, 12, 'mtime'); + checkPath(name); + const spec = RUNNER_PACKAGE_ENTRIES.find((e) => e.name === name); + const isDir = type === '5'; + if (!spec || (spec.type === 'dir') !== isDir) throw fail('unexpected-entry', `Entry ${JSON.stringify(name)}${isDir ? ' (a directory)' : ''} is not part of a runner package.`); + if (seen.has(name)) throw fail('duplicate-entry', `Entry ${JSON.stringify(name)} appears twice.`); + if (mode & 0o7000) throw fail('bad-mode', `Entry ${JSON.stringify(name)} has mode ${mode.toString(8)}: setuid, setgid and sticky bits are refused.`); + if (mode !== spec.mode) throw fail('bad-mode', `Entry ${JSON.stringify(name)} has mode ${mode.toString(8)}, not ${spec.mode.toString(8)}.`); + if (isDir && size !== 0) throw header(`Directory ${JSON.stringify(name)} has a size.`); + if (name === 'VERSION' && size > MAX_VERSION_BYTES) throw fail('version-mismatch', `VERSION is ${size} bytes; a version line is under ${MAX_VERSION_BYTES}.`); + seen.add(name); + return { spec, size }; +} + +/** + * The block reader: a Writable fed the inflated archive. It validates each + * header, streams each file's bytes into staging, and insists on the + * trailer and on nothing after it. + */ +class PackageSink extends Writable { + private pending: Buffer = Buffer.alloc(0); + private inflated = 0; + private crc = 0; + private zeroBlocks = 0; + private current: { entry: Entry; remaining: number; padding: number; file: fs.promises.FileHandle | null; version: Buffer[] } | null = null; + /** The chunk being consumed, settled either way; destruction waits for it so no file lands in staging after 'close'. */ + private inflight: Promise = Promise.resolve(); + readonly seen = new Set(); + private readonly dir: string; + private readonly version: string; + private readonly maxUnpackedBytes: number; + + constructor(dir: string, version: string, maxUnpackedBytes: number) { + super(); + this.dir = dir; + this.version = version; + this.maxUnpackedBytes = maxUnpackedBytes; + } + + /** What the gzip trailer must name: the inflated byte count and its CRC-32. */ + get inflatedBytes(): number { + return this.inflated; + } + + get inflatedCrc32(): number { + return this.crc; + } + + _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { + const work = this.consume(chunk); + this.inflight = work.then( + () => undefined, + () => undefined, + ); + work.then(() => callback(), callback); + } + + _final(callback: (error?: Error | null) => void): void { + if (this.current || this.pending.length || this.zeroBlocks !== 2) { + callback(fail('truncated', 'The archive ended before its trailer.')); + return; + } + const missing = RUNNER_PACKAGE_ENTRIES.filter((e) => !this.seen.has(e.name)).map((e) => e.name); + callback(missing.length ? fail('missing-entry', `The package lacks ${missing.join(', ')}.`) : null); + } + + /** + * After a failure: waits for the chunk in hand (its open or write may + * still be in the threadpool), then closes the file it left open. Only + * then does 'close' fire, so the caller can remove staging with nothing + * about to be created in it. + */ + _destroy(error: Error | null, callback: (error: Error | null) => void): void { + this.inflight + .then(async () => { + const file = this.current?.file ?? null; + this.current = null; + if (file) await file.close().catch(() => undefined); + }) + .then( + () => callback(error), + () => callback(error), + ); + } + + private async consume(chunk: Buffer): Promise { + this.inflated += chunk.length; + if (this.inflated > this.maxUnpackedBytes) throw fail('too-large', `The package inflates to over ${this.maxUnpackedBytes} bytes.`); + this.crc = crc32(chunk, this.crc); + let buf = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk; + this.pending = Buffer.alloc(0); + let at = 0; + while (at < buf.length) { + if (this.current) { + at = await this.body(buf, at); + continue; + } + if (this.zeroBlocks === 2) throw fail('trailing-data', 'The archive has bytes after its trailer.'); + if (buf.length - at < BLOCK) { + this.pending = Buffer.from(buf.subarray(at)); + return; + } + const block = buf.subarray(at, at + BLOCK); + at += BLOCK; + if (allZero(block)) { + this.zeroBlocks++; + continue; + } + if (this.zeroBlocks) throw fail('bad-header', 'An entry follows a zero block; the trailer is two zero blocks and nothing else.'); + await this.open(readHeader(block, this.seen)); + buf = buf.subarray(at); + at = 0; + } + } + + private async open(entry: Entry): Promise { + const { spec, size } = entry; + let file: fs.promises.FileHandle | null = null; + if (spec.type === 'file') { + try { + // bin/ is made with the staging directory; every file is this read's own. + file = await fs.promises.open(path.join(this.dir, spec.name), 'wx', spec.mode); + await file.chmod(spec.mode); + } catch (err) { + throw fail('write-failed', `Cannot create ${spec.name} in staging: ${err instanceof Error ? err.message : String(err)}`); + } + if (this.destroyed) { + // The read failed while this open was in flight: nothing more may stay open in staging. + await file.close().catch(() => undefined); + throw fail('write-failed', `The read ended while ${spec.name} was being created.`); + } + } + this.current = { entry, remaining: size, padding: (BLOCK - (size % BLOCK)) % BLOCK, file, version: [] }; + if (size === 0) await this.finish(); + } + + /** Consumes an entry's bytes and padding from `buf` at `at`; returns where it stopped. */ + private async body(buf: Buffer, at: number): Promise { + const current = this.current; + if (!current) return at; + if (current.remaining > 0) { + const take = Math.min(current.remaining, buf.length - at); + const bytes = buf.subarray(at, at + take); + if (current.entry.spec.name === 'VERSION') current.version.push(Buffer.from(bytes)); + if (current.file) { + try { + await writeAll(current.file, bytes); + } catch (err) { + throw fail('write-failed', `Cannot write ${current.entry.spec.name} in staging: ${err instanceof Error ? err.message : String(err)}`); + } + } + current.remaining -= take; + at += take; + } + if (current.remaining === 0 && current.padding > 0) { + const take = Math.min(current.padding, buf.length - at); + if (!allZero(buf.subarray(at, at + take))) throw header(`Entry ${JSON.stringify(current.entry.spec.name)} is padded with bytes other than NUL.`); + current.padding -= take; + at += take; + } + if (current.remaining === 0 && current.padding === 0) await this.finish(); + return at; + } + + private async finish(): Promise { + const current = this.current; + if (!current) return; + this.current = null; + if (current.file) await current.file.close(); + if (current.entry.spec.name === 'VERSION' && Buffer.concat(current.version).toString('utf8') !== `${this.version}\n`) { + throw fail('version-mismatch', `VERSION says ${JSON.stringify(Buffer.concat(current.version).toString('utf8').trim().slice(0, 40))}, not ${this.version}.`); + } + } +} + +/** Maps a stream failure to a RunnerArchiveError. */ +function archiveError(err: unknown): RunnerArchiveError { + if (err instanceof RunnerArchiveError) return err; + const code = err && typeof err === 'object' ? (err as { code?: unknown }).code : undefined; + if (code === 'Z_BUF_ERROR') return fail('truncated', 'The gzip stream ended early.'); + if (typeof code === 'string' && code.startsWith('Z_')) return fail('corrupt', `The package is not a readable gzip stream (${code}).`); + return fail('corrupt', `Reading the package failed: ${err instanceof Error ? err.message : String(err)}`); +} + +/** + * Reads the package at `archive` into a new private directory under + * `opts.into` (see the header). Resolves with that directory; throws + * RunnerArchiveError with the directory removed. + */ +export async function unpackRunnerPackage(archive: string, opts: UnpackOptions): Promise { + let size: number; + try { + size = fs.statSync(archive).size; + } catch (err) { + throw fail('corrupt', `Cannot read ${archive}: ${err instanceof Error ? err.message : String(err)}`); + } + if (size > MAX_PACKAGE_BYTES) throw fail('too-large', `The package is ${size} bytes; at most ${MAX_PACKAGE_BYTES} are read.`); + if (size <= GZIP_HEADER_BYTES + GZIP_TRAILER_BYTES) throw fail('corrupt', 'The package is too short to be a gzip file.'); + let head: Buffer; + let trailer: Buffer; + let fd: number; + try { + fd = fs.openSync(archive, 'r'); + } catch (err) { + throw fail('corrupt', `Cannot read ${archive}: ${err instanceof Error ? err.message : String(err)}`); + } + try { + head = readAt(fd, 0, GZIP_HEADER_BYTES); + trailer = readAt(fd, size - GZIP_TRAILER_BYTES, GZIP_TRAILER_BYTES); + } finally { + fs.closeSync(fd); + } + checkGzipHeader(head); + const deflateBytes = size - GZIP_HEADER_BYTES - GZIP_TRAILER_BYTES; + let dir: string; + try { + dir = fs.mkdtempSync(path.join(opts.into, 'puck-runner-')); + } catch (err) { + throw fail('write-failed', `Cannot make a staging directory under ${opts.into}: ${err instanceof Error ? err.message : String(err)}`); + } + try { + fs.chmodSync(dir, 0o700); + const bin = RUNNER_PACKAGE_ENTRIES.find((e) => e.type === 'dir') as RunnerPackageEntry; + fs.mkdirSync(path.join(dir, bin.name), { mode: bin.mode }); + fs.chmodSync(path.join(dir, bin.name), bin.mode); + } catch (err) { + fs.rmSync(dir, { recursive: true, force: true }); + throw fail('write-failed', `Cannot make a staging directory under ${opts.into}: ${err instanceof Error ? err.message : String(err)}`); + } + const sink = new PackageSink(dir, opts.version, opts.maxUnpackedBytes ?? MAX_UNPACKED_BYTES); + try { + await new Promise((resolve, reject) => { + // Only the bytes between the gzip header and trailer reach the inflater. + const source = fs.createReadStream(archive, { start: GZIP_HEADER_BYTES, end: size - GZIP_TRAILER_BYTES - 1 }); + const inflate = createInflateRaw(); + let settled = false; + // The first failure anywhere ends all three streams. The sink's 'close' means its in-flight + // work has settled and its file is closed, so the rejection (and the removal of staging) waits for it. + const end = (err: unknown): void => { + if (settled) return; + settled = true; + source.destroy(); + inflate.destroy(); + if (sink.closed) reject(err); + else sink.once('close', () => reject(err)); + sink.destroy(); + }; + source.on('error', end); + inflate.on('error', end); + sink.on('error', end); + // The deflate stream ended: it must have consumed every byte before the trailer. Registered + // before the pipe's own 'end' listener, so a leftover is reported before the sink judges the tar. + inflate.on('end', () => { + const leftover = deflateBytes - inflate.bytesWritten; + if (leftover !== 0) { + end(fail('trailing-data', `The package has ${leftover} bytes between its deflate stream and its gzip trailer (a second gzip member, or other data); a package is one gzip member and nothing else.`)); + } + }); + sink.on('finish', () => { + if (settled) return; + settled = true; + source.destroy(); + resolve(); + }); + source.pipe(inflate).pipe(sink); + }); + if (trailer.readUInt32LE(0) !== sink.inflatedCrc32 || trailer.readUInt32LE(4) !== sink.inflatedBytes % 0x1_0000_0000) { + throw fail('corrupt', "The package's gzip trailer does not match its contents."); + } + } catch (err) { + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); + throw archiveError(err); + } + return { dir }; +} diff --git a/src/runner-release/download.ts b/src/runner-release/download.ts new file mode 100644 index 0000000..9526480 --- /dev/null +++ b/src/runner-release/download.ts @@ -0,0 +1,456 @@ +/** + * The bounded transport for runner releases. The app and the runner bound + * a release listing with `readBoundedBody`; `downloadRunnerPackage` has no + * production caller yet. Node built-ins and the global fetch only; the + * renderer and the daemon never import this directory (.eslintrc.json), + * and nothing here runs a package. + * + * Metadata. `readBoundedBody` returns a response's bytes: the bound is + * checked against Content-Length before the read and against the count + * during it. A listing passes `refuseContentEncoding: false`, because + * fetch has already decoded the body and signed bytes travel base64 + * inside the JSON. A package download refuses a content encoding, because + * those bytes are hashed as sent. `verifySignedRunnerReleases` decodes a + * listing's records (at most MAX_RELEASE_RECORDS, each manifest at most + * MAX_MANIFEST_BYTES once decoded) and hands every manifest's exact bytes + * to the verifier. Nothing re-serialised is ever verified, and the bytes + * come back with the manifest so a server can pass them on unchanged. + * + * Packages. `downloadRunnerPackage` streams one package into a private + * file. Under the official policy the URL is built here from the + * repository's release prefix, the canonical version and the signed file + * name; nothing a server sends chooses it. Every redirect is checked by + * hand: at most MAX_REDIRECTS, https only, exactly the hosts in + * DOWNLOAD_HOSTS, no userinfo, fragment, port or control character, and a + * github.com path compared segment by segment with the one expected. A + * redirect's body is cancelled. The request carries only the headers + * written here, so no Puck, runner or GitHub token ever leaves with it, + * and a URL with credentials is refused before any request. The signed + * byte count is enforced while streaming, under MAX_PACKAGE_BYTES, with + * HEADER_TIMEOUT_MS to each response's headers and BODY_TIMEOUT_MS for the + * whole download. A short body, a long one, a wrong digest, a timeout or a + * cancellation fails the download, cancels the response, closes the file + * and removes it. + * + * The development policy is the caller's, never a server's: it allows http + * on the configured server's own origin and no redirect at all. + * + * Only erasable TypeScript here (see verify.ts). + */ + +import { createHash } from 'node:crypto'; +import { promises as fs, rmSync } from 'node:fs'; +import { + isReleaseVersion, + MAX_RELEASE_METADATA_BYTES, + readSignedRunnerReleases, + RUNNER_RELEASE_PUBLISHER, + RUNNER_TARGETS, + runnerPackageFile, + runnerReleaseTag, + RunnerReleaseError, +} from '../harness/runner-releases'; +import { RELEASE_KEYS } from './trust'; +import { verifyRunnerRelease, type VerifiedRunnerRelease } from './verify'; + +/** Where official packages hang: the repository's GitHub Release assets. */ +export const OFFICIAL_RELEASE_PREFIX = `https://github.com/${RUNNER_RELEASE_PUBLISHER}/releases/download`; +/** The only hosts an official download may touch: the release page and the asset store it redirects to (observed, not promised). */ +export const DOWNLOAD_HOSTS: readonly string[] = ['github.com', 'release-assets.githubusercontent.com']; +export const MAX_REDIRECTS = 5; +/** Most bytes a package may have on the wire, whatever a manifest says. */ +export const MAX_PACKAGE_BYTES = 256 * 1024 * 1024; +/** From a request to its response headers, per hop. */ +export const HEADER_TIMEOUT_MS = 30_000; +/** From the first request to the last byte. */ +export const BODY_TIMEOUT_MS = 15 * 60_000; + +export type RunnerDownloadErrorCode = + /** The URL, or a redirect's target, is outside the policy. */ + | 'bad-url' + | 'too-many-redirects' + /** The server answered with a status other than 200 or a redirect. */ + | 'http-status' + /** The response carries a content encoding, so its bytes are not the ones sent. */ + | 'content-encoding' + /** The response is over the bound. */ + | 'too-large' + /** Content-Length, or the body, is not the signed size. */ + | 'size-mismatch' + /** The body ended before the signed size. */ + | 'truncated' + | 'digest-mismatch' + | 'timeout' + /** The caller's signal aborted it. */ + | 'cancelled' + /** The request failed before any response. */ + | 'unreachable' + /** The file at `dest` could not be created or written. */ + | 'write-failed'; + +export class RunnerDownloadError extends Error { + readonly code: RunnerDownloadErrorCode; + + constructor(code: RunnerDownloadErrorCode, message: string) { + super(message); + this.name = 'RunnerDownloadError'; + this.code = code; + } +} + +export type DownloadPolicy = + /** Official releases: the URL is built here, and redirects stay on DOWNLOAD_HOSTS over https. */ + | { kind: 'official' } + /** A development server: `url` must be on `origin` (http allowed), and it must not redirect. */ + | { kind: 'development'; origin: string }; + +export interface PackageDownload { + /** The canonical release version. */ + version: string; + /** The selected package, from a verified manifest (or a development server's listing). */ + file: string; + sha256: string; + size: number; + /** Development only: the server's URL for the file. Refused under the official policy. */ + url?: string; + /** Where the bytes go: created with mode 0600, never an existing file. */ + dest: string; +} + +export interface DownloadOptions { + fetch?: typeof fetch; + signal?: AbortSignal; + /** Test seams; the exported constants are the contract. */ + headerTimeoutMs?: number; + bodyTimeoutMs?: number; +} + +export interface DownloadedPackage { + /** The URL that served the bytes. */ + url: string; + size: number; + sha256: string; +} + +// eslint-disable-next-line no-control-regex +const CONTROL_OR_SPACE_RE = /[\x00-\x20\x7f]/; +const REDIRECT_STATUSES = [301, 302, 303, 307, 308]; +const HEADERS: Record = { accept: 'application/octet-stream', 'accept-encoding': 'identity' }; + +const bad = (message: string): RunnerDownloadError => new RunnerDownloadError('bad-url', message); + +function contentEncoding(res: Response): void { + const encoding = res.headers.get('content-encoding')?.trim().toLowerCase(); + if (encoding && encoding !== 'identity') throw new RunnerDownloadError('content-encoding', `The response is ${encoding}-encoded; release bytes are read as sent.`); +} + +/** Content-Length as a number, or null when absent; anything unreadable is a size mismatch. */ +function contentLength(res: Response): number | null { + const raw = res.headers.get('content-length'); + if (raw === null) return null; + if (!/^\d{1,15}$/.test(raw.trim())) throw new RunnerDownloadError('size-mismatch', `The response's Content-Length ${JSON.stringify(raw)} is unreadable.`); + return Number(raw.trim()); +} + +const cancel = (res: Response): Promise => (res.body ? res.body.cancel().catch(() => undefined) : Promise.resolve()); + +/** As much of a file handle as writing needs; node:fs's FileHandle is one. */ +export interface ByteSink { + write(data: Uint8Array): Promise<{ bytesWritten: number }>; +} + +/** + * Writes all of `bytes`. A file handle's write may persist only a prefix + * (a full disk, RLIMIT_FSIZE, a short POSIX write) without throwing, so + * this loops until every byte is written and treats a write of nothing + * as a failure; a package is only as verified as the bytes on disk. + */ +export async function writeAll(file: ByteSink, bytes: Uint8Array): Promise { + let at = 0; + while (at < bytes.length) { + const { bytesWritten } = await file.write(bytes.subarray(at)); + if (!Number.isInteger(bytesWritten) || bytesWritten <= 0) throw new Error(`wrote none of the remaining ${bytes.length - at} bytes`); + at += bytesWritten; + } +} + +/** + * Reads a response's body, at most `maxBytes`. The bound is checked + * against Content-Length before the read and against the running count + * during it, and the body is cancelled the moment it is over. A content + * encoding is refused unless `opts.refuseContentEncoding` is false. + * Throws RunnerDownloadError. + */ +export async function readBoundedBody(res: Response, maxBytes: number, opts: { refuseContentEncoding?: boolean } = {}): Promise { + try { + if (opts.refuseContentEncoding !== false) contentEncoding(res); + const length = contentLength(res); + if (length !== null && length > maxBytes) throw new RunnerDownloadError('too-large', `The response announces ${length} bytes; at most ${maxBytes} are read.`); + } catch (err) { + await cancel(res); + throw err; + } + if (!res.body) return new Uint8Array(0); + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.length; + if (total > maxBytes) throw new RunnerDownloadError('too-large', `The response is over ${maxBytes} bytes.`); + chunks.push(value); + } + } catch (err) { + await reader.cancel().catch(() => undefined); + throw err; + } + const out = new Uint8Array(total); + let at = 0; + for (const chunk of chunks) { + out.set(chunk, at); + at += chunk.length; + } + return out; +} + +/** A release listing's body: at most MAX_RELEASE_METADATA_BYTES of UTF-8 JSON, returned parsed and as its exact bytes. */ +export async function readReleaseListing(res: Response): Promise<{ bytes: Uint8Array; value: unknown }> { + const bytes = await readBoundedBody(res, MAX_RELEASE_METADATA_BYTES, { refuseContentEncoding: false }); + let value: unknown; + try { + value = JSON.parse(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + throw new RunnerReleaseError('malformed', 'The release listing is not UTF-8 JSON.'); + } + return { bytes, value }; +} + +export interface VerifiedSignedRelease extends VerifiedRunnerRelease { + /** runner-release.json exactly as signed. */ + bytes: Uint8Array; + signature: Uint8Array; +} + +/** + * Reads a listing's signed release records and verifies each one's exact + * bytes against the trust roots (verify.ts). `keys` defaults to the + * compiled RELEASE_KEYS; only tests pass others. Throws RunnerReleaseError. + */ +export function verifySignedRunnerReleases(records: unknown, keys: readonly string[] = RELEASE_KEYS): VerifiedSignedRelease[] { + return readSignedRunnerReleases(records).map(({ manifest, signature }) => ({ ...verifyRunnerRelease(manifest, signature, keys), bytes: manifest, signature })); +} + +/** The official URL of a release's package: the release prefix, the version's tag and the package's exact file name. */ +export function officialPackageUrl(version: string, file: string): string { + if (!isReleaseVersion(version)) throw bad(`${JSON.stringify(version)} is not a release version.`); + if (!RUNNER_TARGETS.some((t) => runnerPackageFile(t, version) === file)) throw bad(`${JSON.stringify(file)} is not a package of runner ${version}.`); + return `${OFFICIAL_RELEASE_PREFIX}/${runnerReleaseTag(version)}/${file}`; +} + +/** Parses `text` (a URL or a Location, resolved against `base`) or throws; control characters and spaces never parse. */ +function parseUrl(text: string, base: URL | null, what: string): URL { + if (CONTROL_OR_SPACE_RE.test(text)) throw bad(`${what} contains a control character or a space.`); + if (text.includes('#')) throw bad(`${what} has a fragment.`); + try { + return base ? new URL(text, base) : new URL(text); + } catch { + throw bad(`${what} is not a URL: ${JSON.stringify(text.slice(0, 200))}`); + } +} + +/** Path segments after the leading slash; refuses empty, `.` and `..` segments. */ +function pathSegments(url: URL, what: string): string[] { + const segments = url.pathname.split('/').slice(1); + if (segments.some((s) => s === '' || s === '.' || s === '..')) throw bad(`${what} has an empty or relative path segment.`); + return segments; +} + +/** + * Throws unless `url` is one the policy allows: for the first request, or + * for a redirect (`hop` > 0) that the first request's response chain led + * to. `expected` is the package the official URL must name. + */ +export function checkDownloadUrl(url: URL, policy: DownloadPolicy, expected: { version: string; file: string }, hop: number): void { + const what = hop ? `Redirect ${hop}` : 'The download URL'; + if (url.username || url.password) throw bad(`${what} carries credentials.`); + if (url.hash) throw bad(`${what} has a fragment.`); + if (policy.kind === 'development') { + if (hop) throw bad(`${what} is refused: a development server's download must not redirect.`); + if (url.protocol !== 'http:' && url.protocol !== 'https:') throw bad(`${what} is not http or https.`); + let origin: string; + try { + origin = new URL(policy.origin).origin; + } catch { + throw bad(`The development server origin ${JSON.stringify(policy.origin)} is not a URL.`); + } + if (origin === 'null' || url.origin !== origin) throw bad(`${what} is not on the development server ${origin}.`); + pathSegments(url, what); + return; + } + if (url.protocol !== 'https:') throw bad(`${what} is not https.`); + if (url.port) throw bad(`${what} names a port.`); + if (!DOWNLOAD_HOSTS.includes(url.hostname)) throw bad(`${what} is on ${url.hostname}, not a release host.`); + const segments = pathSegments(url, what); + if (url.hostname === 'github.com') { + const want = [...RUNNER_RELEASE_PUBLISHER.split('/'), 'releases', 'download', runnerReleaseTag(expected.version), expected.file]; + const same = segments.length === want.length && segments.every((s, i) => s === want[i]); + if (!same || url.search) throw bad(`${what} is not the release page of ${expected.file}.`); + } +} + +/** The first URL of a download under the policy, checked. */ +function initialUrl(req: PackageDownload, policy: DownloadPolicy): URL { + let url: URL; + if (policy.kind === 'official') { + if (req.url !== undefined) throw bad('An official download takes no URL; it is built from the release.'); + url = parseUrl(officialPackageUrl(req.version, req.file), null, 'The download URL'); + } else { + if (typeof req.url !== 'string') throw bad('A development download needs the server\'s URL for the package.'); + url = parseUrl(req.url, null, 'The download URL'); + } + checkDownloadUrl(url, policy, req, 0); + return url; +} + +/** + * Downloads one package to `req.dest` (see the header). Resolves with the + * URL that served it and the size and digest checked; throws + * RunnerDownloadError, with nothing left at `req.dest`. + */ +export async function downloadRunnerPackage(req: PackageDownload, policy: DownloadPolicy, opts: DownloadOptions = {}): Promise { + const fetchImpl = opts.fetch ?? fetch; + const headerTimeoutMs = opts.headerTimeoutMs ?? HEADER_TIMEOUT_MS; + const bodyTimeoutMs = opts.bodyTimeoutMs ?? BODY_TIMEOUT_MS; + if (!Number.isSafeInteger(req.size) || req.size <= 0) throw new RunnerDownloadError('size-mismatch', `A package's size must be a positive whole number of bytes, not ${String(req.size)}.`); + if (req.size > MAX_PACKAGE_BYTES) throw new RunnerDownloadError('too-large', `The package is ${req.size} bytes; at most ${MAX_PACKAGE_BYTES} are downloaded.`); + if (!/^[0-9a-f]{64}$/.test(req.sha256)) throw new RunnerDownloadError('digest-mismatch', 'The expected sha256 is not 64 lowercase hex digits.'); + let url = initialUrl(req, policy); + + // One controller ends everything: a caller's cancel, a header timeout, or the whole-body deadline. + const controller = new AbortController(); + let why: 'timeout' | 'cancelled' | null = null; + const abort = (reason: 'timeout' | 'cancelled'): void => { + if (why) return; + why = reason; + controller.abort(); + }; + const onCancel = (): void => abort('cancelled'); + if (opts.signal?.aborted) throw new RunnerDownloadError('cancelled', 'The download was cancelled.'); + opts.signal?.addEventListener('abort', onCancel, { once: true }); + const deadline = setTimeout(() => abort('timeout'), bodyTimeoutMs); + const failure = (err: unknown, fallback: RunnerDownloadErrorCode, message: string): RunnerDownloadError => { + // An abort explains whatever it cut short (a closed body reads as truncated). + if (why === 'cancelled') return new RunnerDownloadError('cancelled', 'The download was cancelled.'); + if (why === 'timeout') return new RunnerDownloadError('timeout', `The download of ${req.file} timed out.`); + if (err instanceof RunnerDownloadError) return err; + return new RunnerDownloadError(fallback, `${message}: ${err instanceof Error ? err.message : String(err)}`); + }; + + const request = async (at: URL): Promise => { + const timer = setTimeout(() => abort('timeout'), headerTimeoutMs); + try { + return await fetchImpl(at.href, { method: 'GET', headers: { ...HEADERS }, redirect: 'manual', credentials: 'omit', signal: controller.signal }); + } catch (err) { + throw failure(err, 'unreachable', `Cannot reach ${at.origin}`); + } finally { + clearTimeout(timer); + } + }; + + try { + let res: Response; + for (let hop = 0; ; hop++) { + res = await request(url); + if (REDIRECT_STATUSES.includes(res.status)) { + await cancel(res); + if (hop >= MAX_REDIRECTS) throw new RunnerDownloadError('too-many-redirects', `The download redirected more than ${MAX_REDIRECTS} times.`); + const location = res.headers.get('location'); + if (location === null) throw bad(`Redirect ${hop + 1} names no Location.`); + const next = parseUrl(location, url, `Redirect ${hop + 1}`); + checkDownloadUrl(next, policy, req, hop + 1); + url = next; + continue; + } + if (res.status !== 200) { + await cancel(res); + throw new RunnerDownloadError('http-status', `${url.host} answered ${res.status} for ${req.file}.`); + } + break; + } + try { + contentEncoding(res); + const length = contentLength(res); + if (length !== null && length !== req.size) throw new RunnerDownloadError('size-mismatch', `${url.host} announces ${length} bytes for ${req.file}; the release signed ${req.size}.`); + } catch (err) { + await cancel(res); + throw err; + } + const sha256 = await store(res, req, controller.signal, failure); + return { url: url.href, size: req.size, sha256 }; + } finally { + clearTimeout(deadline); + opts.signal?.removeEventListener('abort', onCancel); + } +} + +/** Streams the body to `req.dest`, counting and hashing; on any failure nothing is left there. */ +async function store( + res: Response, + req: PackageDownload, + signal: AbortSignal, + failure: (err: unknown, code: RunnerDownloadErrorCode, message: string) => RunnerDownloadError, +): Promise { + const hash = createHash('sha256'); + let received = 0; + const reader = res.body?.getReader() ?? null; + // The abort ends the body too, whether or not the fetch tied the stream to its signal. + const onAbort = (): void => { + if (reader) reader.cancel().catch(() => undefined); + }; + signal.addEventListener('abort', onAbort, { once: true }); + let file: fs.FileHandle | null; + try { + // 'wx': the file is this download's own; an existing one is never touched. + file = await fs.open(req.dest, 'wx', 0o600); + } catch (err) { + signal.removeEventListener('abort', onAbort); + if (reader) await reader.cancel().catch(() => undefined); + throw failure(err, 'write-failed', `Cannot create ${req.dest}`); + } + try { + if (reader) { + for (;;) { + let chunk: ReadableStreamReadResult; + try { + chunk = await reader.read(); + } catch (err) { + throw failure(err, 'truncated', `The download of ${req.file} failed`); + } + if (chunk.done) break; + received += chunk.value.length; + if (received > req.size) throw new RunnerDownloadError('size-mismatch', `${req.file} is longer than the ${req.size} bytes the release signed.`); + hash.update(chunk.value); + try { + await writeAll(file, chunk.value); + } catch (err) { + throw failure(err, 'write-failed', `Cannot write ${req.dest}`); + } + } + } + if (received !== req.size) throw new RunnerDownloadError('truncated', `${req.file} ended after ${received} of ${req.size} bytes.`); + const sha256 = hash.digest('hex'); + if (sha256 !== req.sha256) throw new RunnerDownloadError('digest-mismatch', `${req.file} does not match the sha256 the release signed.`); + await file.close(); + file = null; + return sha256; + } catch (err) { + if (reader) await reader.cancel().catch(() => undefined); + if (file) await file.close().catch(() => undefined); + rmSync(req.dest, { force: true }); + throw failure(err, 'truncated', `The download of ${req.file} failed`); + } finally { + signal.removeEventListener('abort', onAbort); + } +} diff --git a/src/runner-release/verify.ts b/src/runner-release/verify.ts index f8dd897..a485db7 100644 --- a/src/runner-release/verify.ts +++ b/src/runner-release/verify.ts @@ -1,9 +1,9 @@ /** * Verifies a signed runner release (src/harness/runner-releases.ts) * against trust roots: the one check the Puck server, the app's main - * process and the runner run before they trust a runner package. Node's - * crypto only; the renderer and the daemon never import this directory - * (.eslintrc.json). + * process and the runner run before they trust a runner package. This + * file uses Node's crypto only. The renderer and the daemon never import + * this directory (.eslintrc.json). * * The order is the contract. Load the roots: Ed25519 SPKI public keys * only, at most MAX_RELEASE_KEYS, none twice. No roots means no trusted diff --git a/test/unit/lint-boundaries.test.ts b/test/unit/lint-boundaries.test.ts index d23aa7f..d22c771 100644 --- a/test/unit/lint-boundaries.test.ts +++ b/test/unit/lint-boundaries.test.ts @@ -2,9 +2,10 @@ import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { ESLint } from 'eslint'; -// The import boundaries .eslintrc.json enforces around the runner-release -// verifier: shared by the server, the app's main process and the runner; -// never in the renderer, the preload, the daemon or the pure harness. +// The import boundaries .eslintrc.json enforces on src/runner-release +// (the verifier, the transport and the archive reader): the server, the +// app's main process and the runner may import it; the renderer, the +// preload, the daemon and the pure harness never do. const root = join(__dirname, '..', '..'); const eslint = new ESLint({ cwd: root }); @@ -39,10 +40,14 @@ describe('runner-release import boundary', () => { } }); - it('lets the verifier import only the harness, itself and node:crypto', async () => { + it('lets runner-release import only the harness, itself and the built-ins the transport and the archive reader need', async () => { const file = 'src/runner-release/x.ts'; expect(await boundaryErrors(file, "import { createHash } from 'node:crypto';\nimport { readRunnerRelease } from '../harness/runner-releases';\nimport { RELEASE_KEYS } from './trust';\nexport const v = [createHash, readRunnerRelease, RELEASE_KEYS];\n")).toEqual([]); - for (const from of ['node:fs', 'fs', 'electron', '../channel/wire', '../main/log', '../server/http', '../puck-runner/tar', '../daemon/main', '../renderer/format']) { + for (const from of ['node:fs', 'node:path', 'node:stream', 'node:zlib']) { + expect(await boundaryErrors(file, `import * as m from '${from}';\nexport const v = m;\n`), from).toEqual([]); + } + // No unprefixed built-in, nothing that runs a process (a package is never probed here), no other directory. + for (const from of ['fs', 'crypto', 'node:child_process', 'node:http', 'node:https', 'node:os', 'electron', '../channel/wire', '../main/log', '../server/http', '../puck-runner/tar', '../daemon/main', '../renderer/format']) { expect(await boundaryErrors(file, `import * as m from '${from}';\nexport const v = m;\n`), from).not.toEqual([]); } }); diff --git a/test/unit/runner-release-archive.test.ts b/test/unit/runner-release-archive.test.ts new file mode 100644 index 0000000..ccc6dd2 --- /dev/null +++ b/test/unit/runner-release-archive.test.ts @@ -0,0 +1,599 @@ +import * as fs from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { packageRunner } from '../../scripts/package-runner.mjs'; +import { RUNNER_PACKAGE_ENTRIES, RUNNER_TARGETS, runnerPackageFile, targetName } from '../../src/harness/runner-releases'; +import { tar, tarGz, type TarEntry } from '../../src/puck-runner/tar'; +import { MAX_UNPACKED_BYTES, RunnerArchiveError, unpackRunnerPackage, type RunnerArchiveErrorCode } from '../../src/runner-release/archive'; +import { MAX_PACKAGE_BYTES } from '../../src/runner-release/download'; + +/** When set, the next staging directory is given a file named `bin`, so `bin/` cannot be created. */ +const plantBin = vi.hoisted(() => ({ on: false })); +/** + * When set, every file handle opened persists only part of what it is given: + * `half` writes half of the first write and nothing after, `zero` writes + * nothing at all, as a full disk or a short POSIX write would. + */ +const shortWrite = vi.hoisted(() => ({ mode: null as null | 'half' | 'zero', opened: 0 })); +/** + * When `ms` is set, every open for writing takes that long, and each open and + * close is recorded in order. `failSourceOnOpen` destroys the archive's read + * stream the moment the first open begins, so the stream fails while that + * open is in flight. + */ +const slowOpen = vi.hoisted(() => ({ ms: 0, events: [] as string[], failSourceOnOpen: false, source: null as null | { destroy(err?: Error): unknown } })); + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + mkdtempSync(prefix: string): string { + const dir = actual.mkdtempSync(prefix); + if (plantBin.on) actual.writeFileSync(join(dir, 'bin'), 'not a directory'); + return dir; + }, + createReadStream(...args: Parameters): ReturnType { + const stream = actual.createReadStream(...args); + slowOpen.source = stream; + return stream; + }, + promises: { + ...actual.promises, + async open(...args: Parameters): ReturnType { + if (slowOpen.failSourceOnOpen) { + slowOpen.failSourceOnOpen = false; + slowOpen.source?.destroy(new Error('the disk read failed')); + } + if (slowOpen.ms) await new Promise((resolve) => setTimeout(resolve, slowOpen.ms)); + const handle = await actual.promises.open(...args); + if (slowOpen.ms) { + slowOpen.events.push(`opened ${String(args[0]).split('/').pop()}`); + const close = handle.close.bind(handle); + handle.close = async () => { + slowOpen.events.push(`closed ${String(args[0]).split('/').pop()}`); + return close(); + }; + } + if (!shortWrite.mode) return handle; + shortWrite.opened++; + let calls = 0; + const patched = Object.create(handle) as typeof handle; + patched.write = (async (data: Uint8Array) => { + calls++; + if (shortWrite.mode === 'zero' || calls > 1) return { bytesWritten: 0, buffer: data }; + return handle.write(data.subarray(0, Math.max(1, data.length >> 1))); + }) as typeof handle.write; + return patched; + }, + }, + }; +}); + +const { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, truncateSync, writeFileSync } = fs; + +// The strict package reader: the packager's fixed layout and nothing else, +// read header by header into private staging, with every failure leaving +// no staging behind. Packages come from the runner's own tar writer (as +// the packager's do), from the packager itself with a stand-in bundle and +// runtime, and, where a packaging run left them under out/, from the real +// artifacts. + +const VERSION = '1.2.3'; +const root = join(__dirname, '..', '..'); + +const dirs: string[] = []; +let into: string; +const tmp = (): string => { + const dir = mkdtempSync(join(tmpdir(), 'puck-archive-')); + dirs.push(dir); + return dir; +}; +beforeEach(() => { + into = tmp(); + vi.spyOn(console, 'log').mockImplementation(() => undefined); +}); +afterEach(() => { + plantBin.on = false; + shortWrite.mode = null; + shortWrite.opened = 0; + slowOpen.ms = 0; + slowOpen.events = []; + slowOpen.failSourceOnOpen = false; + slowOpen.source = null; + vi.restoreAllMocks(); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +/** 300 KiB of fixed pseudo-random bytes (xorshift32 from a fixed seed): incompressible like a runtime, and the same in every run. */ +const NODE = (() => { + const out = Buffer.alloc(300 * 1024); + let x = 0x9e3779b9; + for (let i = 0; i < out.length; i++) { + x = (x ^ (x << 13)) >>> 0; + x = (x ^ (x >>> 17)) >>> 0; + x = (x ^ (x << 5)) >>> 0; + out[i] = x & 0xff; + } + return out; +})(); +const BUNDLE = Buffer.from(`// runner ${VERSION}\n`.repeat(2000)); +const BODIES: Record = { + 'config.sh': Buffer.from('#!/bin/sh\n# config\n'), + 'run.sh': Buffer.from('#!/bin/sh\n# run\n'), + 'svc.sh': Buffer.from('#!/bin/sh\n# svc\n'), + VERSION: Buffer.from(`${VERSION}\n`), + 'README.md': Buffer.from('# runner\n'), + LICENSE: Buffer.from('MIT\n'), + 'bin/node': NODE, + 'bin/node.LICENSE': Buffer.from('Node.js license\n'), + 'bin/puck-runner.cjs': BUNDLE, +}; + +/** The layout as entries for the writer, with `overrides` applied by name. */ +function entries(overrides: Record | null> = {}): TarEntry[] { + return RUNNER_PACKAGE_ENTRIES.flatMap((e) => { + const change = overrides[e.name]; + if (change === null) return []; + const entry: TarEntry = e.type === 'dir' ? { name: e.name, type: 'dir', mode: e.mode } : { name: e.name, type: 'file', mode: e.mode, body: BODIES[e.name] }; + return [{ ...entry, ...change }]; + }); +} + +const write = (bytes: Buffer): string => { + const file = join(tmp(), 'package.tar.gz'); + writeFileSync(file, bytes); + return file; +}; + +const unpack = (bytes: Buffer, version = VERSION, maxUnpackedBytes?: number) => unpackRunnerPackage(write(bytes), { into, version, maxUnpackedBytes }); + +async function failure(promise: Promise): Promise<{ code: RunnerArchiveErrorCode; message: string }> { + try { + await promise; + } catch (err) { + if (err instanceof RunnerArchiveError) return { code: err.code, message: err.message }; + throw err; + } + throw new Error('expected a RunnerArchiveError'); +} + +/** Unpacking fails with `code` and a message matching `message`, and leaves no staging behind. */ +async function refused(bytes: Buffer, code: RunnerArchiveErrorCode, message: RegExp | string, version = VERSION): Promise { + const f = await failure(unpack(bytes, version)); + expect(f.code, f.message).toBe(code); + expect(f.message).toMatch(message); + expect(readdirSync(into)).toEqual([]); +} + +/** Everything under `dir`, relative, sorted, with directories marked by a slash. */ +function listing(dir: string): string[] { + const out: string[] = []; + const walk = (rel: string): void => { + for (const name of readdirSync(join(dir, rel)).sort()) { + const path = rel ? `${rel}/${name}` : name; + const st = statSync(join(dir, path)); + if (st.isDirectory()) { + out.push(`${path}/`); + walk(path); + } else out.push(path); + } + }; + walk(''); + return out; +} + +function expectLayout(dir: string, bodies: Record = BODIES): void { + expect(statSync(dir).mode & 0o777).toBe(0o700); + expect(listing(dir)).toEqual([...RUNNER_PACKAGE_ENTRIES.map((e) => e.name)].sort()); + for (const e of RUNNER_PACKAGE_ENTRIES) { + const st = statSync(join(dir, e.name)); + expect(st.mode & 0o7777, e.name).toBe(e.mode); + expect(st.isDirectory(), e.name).toBe(e.type === 'dir'); + if (e.type === 'file') expect(readFileSync(join(dir, e.name)).equals(bodies[e.name]), e.name).toBe(true); + } +} + +/** Where `name`'s header and body sit in a raw archive. */ +function locate(raw: Buffer, name: string): { header: number; body: number; size: number } { + const field = (h: Buffer, at: number, len: number): string => h.subarray(at, at + len).toString('latin1').replace(/\0.*$/s, ''); + for (let at = 0; at + 512 <= raw.length; ) { + const header = raw.subarray(at, at + 512); + if (header.every((b) => b === 0)) break; + const size = parseInt(field(header, 124, 12), 8); + if (field(header, 0, 100) === name) return { header: at, body: at + 512, size }; + at += 512 + Math.ceil(size / 512) * 512; + } + throw new Error(`no entry ${name}`); +} + +function checksum(header: Buffer): void { + header.fill(0x20, 148, 156); + let sum = 0; + for (const b of header) sum += b; + header.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 'latin1'); +} + +/** The archive with `name`'s header changed by `edit` (and its checksum fixed, unless `keepChecksum`). */ +function mutated(name: string, edit: (header: Buffer) => void, opts: { keepChecksum?: boolean; entries?: TarEntry[] } = {}): Buffer { + const raw = Buffer.from(tar(opts.entries ?? entries())); + const { header } = locate(raw, name); + const block = raw.subarray(header, header + 512); + edit(block); + if (!opts.keepChecksum) checksum(block); + return gzipSync(raw); +} + +const octal = (value: number, width: number): string => value.toString(8).padStart(width - 1, '0') + '\0'; + +describe('a valid package', () => { + it('unpacks into a fresh 0700 directory under the parent, with the layout and its modes, and nothing else', async () => { + const { dir } = await unpack(tarGz(entries())); + expect(dir.startsWith(into + '/')).toBe(true); + expect(readdirSync(into)).toEqual([dir.slice(into.length + 1)]); + expectLayout(dir); + }); + + it('accepts the entries in any order', async () => { + const reversed = [...entries()].reverse(); + expectLayout((await unpack(tarGz(reversed))).dir); + const binFirst = entries().sort((a) => (a.name === 'bin/node' ? -1 : 0)); + expectLayout((await unpack(tarGz(binFirst))).dir); + }); + + it('restores the layout modes whatever the umask, and refuses nothing on an empty file', async () => { + const empty = { ...BODIES, LICENSE: Buffer.alloc(0) }; + const { dir } = await unpack(tarGz(entries({ LICENSE: { body: empty.LICENSE } }))); + expectLayout(dir, empty); + }); + + it('is what scripts/package-runner.mjs writes, for every target', async () => { + const version = '9.8.7'; + const outDir = tmp(); + const build = async ({ mode }: { mode: string }) => { + const bundlePath = join(tmp(), 'puck-runner.cjs'); + const probe = JSON.stringify({ version, trustMode: mode, runnerProtocol: 7 }); + writeFileSync(bundlePath, `// stand-in runner bundle\nprocess.stdout.write(${JSON.stringify(probe + '\n')});\n`); + return { bundlePath, version }; + }; + const runtime = async (target: { node: string }) => ({ node: Buffer.from(`#!/bin/sh\n# node for ${target.node}\n`), license: Buffer.from('Node.js license\n') }); + const result = await packageRunner({ mode: 'development', outDir, build, runtime, commit: () => { throw new Error('no commit'); } }); + expect(result.files).toHaveLength(RUNNER_TARGETS.length); + for (const target of RUNNER_TARGETS) { + const file = join(outDir, version, runnerPackageFile(target, version)); + const { dir } = await unpackRunnerPackage(file, { into, version }); + expect(listing(dir), targetName(target)).toEqual([...RUNNER_PACKAGE_ENTRIES.map((e) => e.name)].sort()); + for (const e of RUNNER_PACKAGE_ENTRIES) expect(statSync(join(dir, e.name)).mode & 0o7777, `${targetName(target)} ${e.name}`).toBe(e.mode); + expect(readFileSync(join(dir, 'VERSION'), 'utf8')).toBe(`${version}\n`); + expect(readFileSync(join(dir, 'bin', 'node'), 'utf8')).toContain(`node for`); + expect(readFileSync(join(dir, 'config.sh'))).toEqual(readFileSync(join(root, 'src', 'puck-runner', 'sh', 'config.sh'))); + expect(readFileSync(join(dir, 'LICENSE'))).toEqual(readFileSync(join(root, 'LICENSE'))); + await expect(failure(unpackRunnerPackage(file, { into, version: '9.8.8' }))).resolves.toMatchObject({ code: 'version-mismatch' }); + } + }); + + it('reads the real packaged artifacts under out/, when a packaging run left them there', async (ctx) => { + const version = (JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')) as { version: string }).version; + const files = ['puck-runner-suite', 'puck-runner'] + .flatMap((out) => RUNNER_TARGETS.map((t) => join(root, 'out', out, version, runnerPackageFile(t, version)))) + .filter((file) => existsSync(file)); + if (!files.length) ctx.skip(`no packaged runner under out/puck-runner-suite/${version} or out/puck-runner/${version}`); + for (const file of files) { + const { dir } = await unpackRunnerPackage(file, { into, version }); + expect(listing(dir), file).toEqual([...RUNNER_PACKAGE_ENTRIES.map((e) => e.name)].sort()); + for (const e of RUNNER_PACKAGE_ENTRIES) expect(statSync(join(dir, e.name)).mode & 0o7777, `${file} ${e.name}`).toBe(e.mode); + expect(readFileSync(join(dir, 'VERSION'), 'utf8')).toBe(`${version}\n`); + expect(statSync(join(dir, 'bin', 'node')).size).toBeGreaterThan(10 * 1024 * 1024); + rmSync(dir, { recursive: true, force: true }); + } + }, 300_000); +}); + +describe('entries outside the layout', () => { + it('refuses any other name, a state file, another directory, or bin as a file', async () => { + for (const [name, entry] of [ + ['extra.txt', { name: 'extra.txt', type: 'file', mode: 0o644, body: 'x' }], + ['bin/extra', { name: 'bin/extra', type: 'file', mode: 0o644, body: 'x' }], + ['.runner', { name: '.runner', type: 'file', mode: 0o644, body: '{}' }], + ['.runner_key', { name: '.runner_key', type: 'file', mode: 0o600, body: 'key' }], + ['_update/', { name: '_update/', type: 'dir', mode: 0o700 }], + ['lib/', { name: 'lib/', type: 'dir', mode: 0o755 }], + ['config.sh/', { name: 'config.sh/', type: 'dir', mode: 0o755 }], + ['Bin/node', { name: 'Bin/node', type: 'file', mode: 0o755, body: 'x' }], + ] as [string, TarEntry][]) { + await refused(tarGz([...entries(), entry]), 'unexpected-entry', `Entry ${JSON.stringify(name)}`); + } + await refused(tarGz([...entries({ 'bin/': null }), { name: 'bin', type: 'file', mode: 0o755, body: 'x' }]), 'unexpected-entry', 'Entry "bin"'); + await refused(tarGz(entries({ 'bin/node': { name: 'bin/node', type: 'dir' } })), 'unexpected-entry', 'Entry "bin/node/" (a directory)'); + }); + + it('refuses a missing member, including bin/ itself, and a duplicate', async () => { + await refused(tarGz(entries({ LICENSE: null })), 'missing-entry', 'The package lacks LICENSE.'); + await refused(tarGz(entries({ 'bin/': null })), 'missing-entry', 'The package lacks bin/.'); + await refused(tarGz(entries({ 'bin/node': null, 'bin/puck-runner.cjs': null })), 'missing-entry', 'lacks bin/node, bin/puck-runner.cjs'); + await refused(tarGz([]), 'missing-entry', 'The package lacks config.sh, run.sh'); + const all = entries(); + await refused(tarGz([...all, all[3]]), 'duplicate-entry', 'Entry "VERSION" appears twice.'); + await refused(tarGz([...all, all[6]]), 'duplicate-entry', 'Entry "bin/" appears twice.'); + }); + + it('refuses absolute, traversing and otherwise unsafe paths before looking anything up', async () => { + const rename = (to: string): Buffer => + mutated('LICENSE', (h) => { + h.fill(0, 0, 100); + h.write(to, 0, 'latin1'); + }); + await refused(rename('/etc/passwd'), 'unsafe-path', 'is an absolute path'); + await refused(rename('/LICENSE'), 'unsafe-path', 'is an absolute path'); + await refused(rename('../LICENSE'), 'unsafe-path', 'has an empty, "." or ".." segment'); + await refused(rename('bin/../LICENSE'), 'unsafe-path', '".." segment'); + await refused(rename('./LICENSE'), 'unsafe-path', '"." or ".." segment'); + await refused(rename('bin//LICENSE'), 'unsafe-path', 'empty'); + await refused(rename('bin\\LICENSE'), 'unsafe-path', 'backslash or a control character'); + await refused(rename('LICENSE\x01'), 'unsafe-path', 'backslash or a control character'); + await refused(rename(''), 'unsafe-path', 'empty name'); + }); +}); + +describe('entry types and modes', () => { + it('refuses links, devices, FIFOs, contiguous files and unknown type flags', async () => { + const typed = (flag: string, extra?: (h: Buffer) => void): Buffer => + mutated('LICENSE', (h) => { + h.write(flag, 156, 'latin1'); + extra?.(h); + }); + await refused(typed('1'), 'unexpected-entry', 'Entry "LICENSE" is a hard link'); + await refused(typed('2'), 'unexpected-entry', 'Entry "LICENSE" is a symbolic link'); + await refused(typed('3'), 'unexpected-entry', 'is a character device'); + await refused(typed('4'), 'unexpected-entry', 'is a block device'); + await refused(typed('6'), 'unexpected-entry', 'is a FIFO'); + await refused(typed('7'), 'unexpected-entry', 'is a contiguous file'); + await refused(typed('A'), 'unexpected-entry', 'has type flag "A"'); + await refused(typed('S'), 'unexpected-entry', 'has type flag "S"'); + // A symlink shaped like the real thing: a link name and no body. + await refused(typed('2', (h) => h.write('/etc/passwd', 157, 'latin1')), 'unexpected-entry', 'symbolic link'); + }); + + it('refuses PAX and GNU extension entries, including the ones the shared writer makes for long names', async () => { + await refused(mutated('LICENSE', (h) => h.write('x', 156, 'latin1')), 'unexpected-entry', 'is a PAX extended header'); + await refused(mutated('LICENSE', (h) => h.write('g', 156, 'latin1')), 'unexpected-entry', 'is a PAX global header'); + await refused(mutated('LICENSE', (h) => h.write('L', 156, 'latin1')), 'unexpected-entry', 'is a GNU long name'); + await refused(mutated('LICENSE', (h) => h.write('K', 156, 'latin1')), 'unexpected-entry', 'is a GNU long link'); + const long = 'x'.repeat(300); + await refused(tarGz([...entries(), { name: long, type: 'file', mode: 0o644, body: 'x' }]), 'unexpected-entry', 'Entry "PaxHeader" is a PAX extended header'); + }); + + it('refuses setuid, setgid and sticky bits, and any mode but the layout\'s', async () => { + await refused(tarGz(entries({ 'config.sh': { mode: 0o4755 } })), 'bad-mode', 'Entry "config.sh" has mode 4755: setuid, setgid and sticky bits are refused.'); + await refused(tarGz(entries({ 'bin/node': { mode: 0o2755 } })), 'bad-mode', 'has mode 2755: setuid'); + await refused(tarGz(entries({ 'bin/': { mode: 0o1755 } })), 'bad-mode', 'has mode 1755: setuid'); + await refused(tarGz(entries({ 'bin/node': { mode: 0o6755 } })), 'bad-mode', 'has mode 6755: setuid'); + await refused(tarGz(entries({ 'run.sh': { mode: 0o644 } })), 'bad-mode', 'Entry "run.sh" has mode 644, not 755.'); + await refused(tarGz(entries({ VERSION: { mode: 0o755 } })), 'bad-mode', 'Entry "VERSION" has mode 755, not 644.'); + await refused(tarGz(entries({ 'bin/': { mode: 0o700 } })), 'bad-mode', 'Entry "bin/" has mode 700, not 755.'); + await refused(tarGz(entries({ LICENSE: { mode: 0o664 } })), 'bad-mode', 'has mode 664, not 644'); + await refused(tarGz(entries({ 'bin/puck-runner.cjs': { mode: 0o755 } })), 'bad-mode', 'has mode 755, not 644'); + }); +}); + +describe('header fields', () => { + it('verifies the checksum', async () => { + // The same bytes, one field changed: the checksum no longer matches, and that is what is reported. + await refused(mutated('LICENSE', (h) => h.write(octal(0o600, 8), 100, 'latin1'), { keepChecksum: true }), 'bad-header', 'wrong checksum'); + await refused(mutated('run.sh', (h) => h.write('X', 0, 'latin1'), { keepChecksum: true }), 'bad-header', 'wrong checksum'); + await refused(mutated('LICENSE', (h) => h.write('0000000\0', 148, 'latin1'), { keepChecksum: true }), 'bad-header', 'wrong checksum'); + }); + + it('requires ustar magic and version 00', async () => { + await refused(mutated('LICENSE', (h) => h.write('ustar \0', 257, 'latin1')), 'bad-header', 'not ustar'); + await refused(mutated('LICENSE', (h) => h.write('01', 263, 'latin1')), 'bad-header', 'not ustar'); + await refused(mutated('LICENSE', (h) => h.fill(0, 257, 265)), 'bad-header', 'not ustar'); + }); + + it('reads octal fields only: no base-256 or other GNU numeric encodings', async () => { + await refused(mutated('LICENSE', (h) => (h[124] = 0x80)), 'bad-header', 'size field is base-256, a GNU extension'); + await refused(mutated('LICENSE', (h) => h.write('00000000089\0', 124, 'latin1')), 'bad-header', 'size field is not octal'); + await refused(mutated('LICENSE', (h) => h.write(' \0', 124, 'latin1')), 'bad-header', 'size field is not octal'); + await refused(mutated('LICENSE', (h) => h.write('0000x44\0', 100, 'latin1')), 'bad-header', 'mode field is not octal'); + await refused(mutated('LICENSE', (h) => (h[136] = 0xff)), 'bad-header', 'mtime field is base-256'); + await refused(mutated('LICENSE', (h) => h.write('9', 108, 'latin1')), 'bad-header', 'uid field is not octal'); + }); + + it('refuses a prefix, a link name on a regular file, a directory with a size, and stray header bytes', async () => { + await refused(mutated('LICENSE', (h) => h.write('bin', 345, 'latin1')), 'bad-header', 'uses the prefix field'); + await refused(mutated('LICENSE', (h) => h.write('target', 157, 'latin1')), 'bad-header', 'has a link name'); + await refused(mutated('bin/', (h) => h.write(octal(512, 12), 124, 'latin1')), 'bad-header', 'Directory "bin/" has a size'); + await refused(mutated('LICENSE', (h) => (h[500] = 1)), 'bad-header', "bytes in the header's padding"); + await refused(mutated('LICENSE', (h) => (h[8] = 0x41)), 'bad-header', 'name field has bytes after its terminator'); + }); + + it('reads the owner names like every text field and requires NUL device numbers, whatever the checksum says', async () => { + // Each mutation carries a corrected checksum: only the field itself is what refuses the header. + await refused(mutated('LICENSE', (h) => h.write('root\0x', 265, 'latin1')), 'bad-header', 'uname field has bytes after its terminator'); + await refused(mutated('LICENSE', (h) => (h[296] = 0xff)), 'bad-header', 'uname field has bytes after its terminator'); + await refused(mutated('LICENSE', (h) => h.write('root\0x', 297, 'latin1')), 'bad-header', 'gname field has bytes after its terminator'); + await refused(mutated('LICENSE', (h) => (h[328] = 0xff)), 'bad-header', 'gname field has bytes after its terminator'); + await refused(mutated('LICENSE', (h) => (h[329] = 0xff)), 'bad-header', 'has device numbers'); + await refused(mutated('LICENSE', (h) => h.write('0000001\0', 329, 'latin1')), 'bad-header', 'has device numbers'); + await refused(mutated('LICENSE', (h) => (h[337] = 0xff)), 'bad-header', 'has device numbers'); + await refused(mutated('LICENSE', (h) => h.write('0000001\0', 337, 'latin1')), 'bad-header', 'has device numbers'); + await refused(mutated('bin/', (h) => (h[344] = 0x01)), 'bad-header', 'has device numbers'); + // Owner names that fill their field, and the packager's own, are fine. + expectLayout((await unpack(mutated('LICENSE', (h) => h.write('x'.repeat(32), 265, 'latin1')))).dir); + expectLayout((await unpack(mutated('LICENSE', (h) => h.write('wheel', 297, 'latin1')))).dir); + }); + + it('requires NUL padding after a body', async () => { + const raw = Buffer.from(tar(entries())); + const { body, size } = locate(raw, 'LICENSE'); + raw[body + size] = 0x41; + await refused(gzipSync(raw), 'bad-header', 'padded with bytes other than NUL'); + }); +}); + +describe('the stream and the trailer', () => { + it('requires the two-block trailer and nothing after it', async () => { + const raw = Buffer.from(tar(entries())); + await refused(gzipSync(raw.subarray(0, raw.length - 512)), 'truncated', 'ended before its trailer'); + await refused(gzipSync(raw.subarray(0, raw.length - 1024)), 'truncated', 'ended before its trailer'); + await refused(gzipSync(raw.subarray(0, raw.length - 1024 - 100)), 'truncated', 'ended before its trailer'); + await refused(gzipSync(Buffer.concat([raw, Buffer.alloc(512)])), 'trailing-data', 'bytes after its trailer'); + await refused(gzipSync(Buffer.concat([raw, Buffer.alloc(1)])), 'trailing-data', 'bytes after its trailer'); + await refused(gzipSync(Buffer.concat([raw, raw])), 'trailing-data', 'bytes after its trailer'); + const withoutTrailer = raw.subarray(0, raw.length - 1024); + await refused(gzipSync(Buffer.concat([withoutTrailer, Buffer.alloc(512), withoutTrailer, Buffer.alloc(1024)])), 'bad-header', 'An entry follows a zero block'); + }); + + it('reads exactly one gzip member: nothing before it, between its deflate stream and its trailer, or after', async () => { + const raw = Buffer.from(tar(entries())); + const gz = gzipSync(raw, { level: 9 }); + const empty = gzipSync(Buffer.alloc(0)); + const ONE_MEMBER = 'a package is one gzip member and nothing else'; + // gunzip would join or ignore every one of these; the reader refuses them all. + await refused(Buffer.concat([gz, empty]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([empty, gz]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([gzipSync(raw.subarray(0, 3000)), gzipSync(raw.subarray(3000))]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([gz, gz]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([gz, Buffer.from([0]), Buffer.from('trailing garbage')]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([gz, Buffer.alloc(8)]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([gz, Buffer.from('garbage')]), 'trailing-data', ONE_MEMBER); + await refused(Buffer.concat([gz, Buffer.from([0])]), 'trailing-data', ONE_MEMBER); + // A member is intact only with its own trailer: the CRC-32 and the inflated size. + for (const at of [gz.length - 8, gz.length - 5, gz.length - 4, gz.length - 1]) { + const bad = Buffer.from(gz); + bad[at] ^= 0x01; + await refused(bad, 'corrupt', 'gzip trailer does not match its contents'); + } + // One byte short: the trailer is then the deflate stream's last bytes, so the stream itself ends early. + await refused(gz.subarray(0, gz.length - 1), 'truncated', 'gzip stream ended early'); + }); + + it('takes the fixed gzip header only: deflate, no optional fields', async () => { + const gz = tarGz(entries()); + expect(gz.subarray(0, 4)).toEqual(Buffer.from([0x1f, 0x8b, 0x08, 0x00])); + const withFlag = (flag: number): Buffer => { + const out = Buffer.from(gz); + out[3] = flag; + return out; + }; + for (const [flag, what] of [ + [0x01, 'text'], + [0x02, 'header CRC'], + [0x04, 'extra field'], + [0x08, 'name'], + [0x10, 'comment'], + [0x20, 'reserved'], + ] as [number, string][]) { + await refused(withFlag(flag), 'corrupt', new RegExp(`gzip header sets flags.*${what === 'text' || what === 'reserved' ? '' : what}`)); + } + const method = Buffer.from(gz); + method[2] = 0x09; + await refused(method, 'corrupt', 'compression method 9, not deflate'); + const magic = Buffer.from(gz); + magic[1] = 0x8c; + await refused(magic, 'corrupt', 'not a gzip file'); + await refused(Buffer.from('not a gzip stream at all'), 'corrupt', 'not a gzip file'); + for (const short of [gz.subarray(0, 10), gz.subarray(0, 18), Buffer.alloc(0)]) await refused(short, 'corrupt', 'too short to be a gzip file'); + }); + + it('refuses a truncated or corrupt deflate stream', async () => { + const gz = tarGz(entries()); + await refused(gz.subarray(0, gz.length - 100), 'truncated', 'gzip stream ended early'); + await refused(gz.subarray(0, 40), 'truncated', 'gzip stream ended early'); + // A flipped byte near the end can corrupt the stream, cut it short, or end it early with bytes left over + // before the trailer; each is a correct rejection, and the fixed payload makes this one reproducible. + const flipped = Buffer.from(gz); + flipped[gz.length - 30] ^= 0xff; + expect((await failure(unpack(flipped))).code).toMatch(/^(corrupt|truncated|trailing-data)$/); + expect(readdirSync(into)).toEqual([]); + const garbageBody = Buffer.concat([gz.subarray(0, 10), Buffer.from('this is not a deflate stream, whatever the framing says'), gz.subarray(gz.length - 8)]); + await refused(garbageBody, 'corrupt', 'not a readable gzip stream'); + }); + + it('caps what the package may inflate to, and how large the file may be', async () => { + expect(MAX_UNPACKED_BYTES).toBe(512 * 1024 * 1024); + const gz = tarGz(entries()); + const f = await failure(unpack(gz, VERSION, 64 * 1024)); + expect(f).toEqual({ code: 'too-large', message: 'The package inflates to over 65536 bytes.' }); + expect(readdirSync(into)).toEqual([]); + const fits = await unpack(gz, VERSION, 2 * 1024 * 1024); + expectLayout(fits.dir); + rmSync(fits.dir, { recursive: true, force: true }); + const sparse = write(gz); + truncateSync(sparse, MAX_PACKAGE_BYTES + 1); + const large = await failure(unpackRunnerPackage(sparse, { into, version: VERSION })); + expect(large).toEqual({ code: 'too-large', message: `The package is ${MAX_PACKAGE_BYTES + 1} bytes; at most ${MAX_PACKAGE_BYTES} are read.` }); + expect(readdirSync(into)).toEqual([]); + }); +}); + +describe('VERSION', () => { + it('must be exactly the expected version and a newline', async () => { + await refused(tarGz(entries()), 'version-mismatch', 'VERSION says "1.2.3", not 1.2.4.', '1.2.4'); + await refused(tarGz(entries({ VERSION: { body: '1.2.3' } })), 'version-mismatch', 'not 1.2.3'); + await refused(tarGz(entries({ VERSION: { body: '1.2.3\n\n' } })), 'version-mismatch', 'not 1.2.3'); + await refused(tarGz(entries({ VERSION: { body: ' 1.2.3\n' } })), 'version-mismatch', 'not 1.2.3'); + await refused(tarGz(entries({ VERSION: { body: '' } })), 'version-mismatch', 'VERSION says ""'); + await refused(tarGz(entries({ VERSION: { body: `${'9'.repeat(70)}\n` } })), 'version-mismatch', 'VERSION is 71 bytes'); + }); +}); + +describe('staging', () => { + it('needs an existing parent, and removes its directory on every failure', async () => { + const missing = join(into, 'missing'); + const f = await failure(unpackRunnerPackage(write(tarGz(entries())), { into: missing, version: VERSION })); + expect(f.code).toBe('write-failed'); + expect(existsSync(missing)).toBe(false); + // A failure late in the stream, after files were written, still leaves nothing. + const late = tarGz([...entries(), { name: 'zzz-extra', type: 'file', mode: 0o644, body: 'x' }]); + expect((await failure(unpack(late))).code).toBe('unexpected-entry'); + expect(readdirSync(into)).toEqual([]); + // Each read gets its own directory; a second read beside a first leaves the first alone. + const first = await unpack(tarGz(entries())); + const second = await unpack(tarGz(entries())); + expect(first.dir).not.toBe(second.dir); + expect(readdirSync(into).sort()).toEqual([first.dir, second.dir].map((d) => d.slice(into.length + 1)).sort()); + mkdirSync(join(into, 'other')); + expect((await failure(unpack(tarGz(entries({ LICENSE: null }))))).code).toBe('missing-entry'); + expect(readdirSync(into).sort()).toEqual([first.dir, second.dir].map((d) => d.slice(into.length + 1)).concat('other').sort()); + }); + + it('removes the staging directory when bin/ cannot be created', async () => { + plantBin.on = true; + const f = await failure(unpack(tarGz(entries()))); + expect(f.code).toBe('write-failed'); + expect(readdirSync(into)).toEqual([]); + }); + + it('closes a file whose open was still in flight when the stream failed, before it rejects and removes staging', async () => { + // The read stream fails the moment the first entry's open begins, and that open takes a while. + slowOpen.ms = 150; + slowOpen.failSourceOnOpen = true; + const f = await failure(unpack(tarGz(entries()))); + expect(f).toEqual({ code: 'corrupt', message: 'Reading the package failed: the disk read failed' }); + // By the time the caller hears of the failure, the late open has completed and its handle is closed. + expect(slowOpen.events).toEqual(['opened config.sh', 'closed config.sh']); + expect(readdirSync(into)).toEqual([]); + // The same ordering when the sink itself is what fails, with an open in flight for the next entry. + slowOpen.events = []; + const g = await failure(unpack(tarGz(entries({ 'run.sh': { mode: 0o4755 } })))); + expect(g.code).toBe('bad-mode'); + expect(slowOpen.events).toEqual(['opened config.sh', 'closed config.sh']); + expect(readdirSync(into)).toEqual([]); + }); + + it('fails with write-failed and removes staging when a file write persists only a prefix, or nothing', async () => { + for (const mode of ['half', 'zero'] as const) { + shortWrite.mode = mode; + shortWrite.opened = 0; + const f = await failure(unpack(tarGz(entries()))); + expect(f.code, mode).toBe('write-failed'); + expect(f.message, mode).toMatch(/Cannot write config\.sh in staging: wrote none of the remaining \d+ bytes/); + expect(shortWrite.opened, mode).toBe(1); + expect(readdirSync(into), mode).toEqual([]); + } + shortWrite.mode = null; + expectLayout((await unpack(tarGz(entries()))).dir); + }); +}); diff --git a/test/unit/runner-release-download.test.ts b/test/unit/runner-release-download.test.ts new file mode 100644 index 0000000..e537ab8 --- /dev/null +++ b/test/unit/runner-release-download.test.ts @@ -0,0 +1,735 @@ +import { createHash, generateKeyPairSync, randomBytes, sign, type KeyObject } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + formatRunnerRelease, + formatSignedRunnerRelease, + MAX_MANIFEST_BYTES, + MAX_RELEASE_METADATA_BYTES, + MAX_RELEASE_RECORDS, + RUNNER_TARGETS, + runnerPackageFile, + RunnerReleaseError, + type RunnerReleaseManifest, +} from '../../src/harness/runner-releases'; +import * as mainApi from '../../src/main/server/api'; +import { serverRequest, useServerDeps } from '../../src/main/server/http'; +import { ServerApi } from '../../src/puck-runner/api'; +import { + BODY_TIMEOUT_MS, + checkDownloadUrl, + DOWNLOAD_HOSTS, + downloadRunnerPackage, + HEADER_TIMEOUT_MS, + MAX_PACKAGE_BYTES, + MAX_REDIRECTS, + OFFICIAL_RELEASE_PREFIX, + officialPackageUrl, + readBoundedBody, + readReleaseListing, + RunnerDownloadError, + verifySignedRunnerReleases, + writeAll, + type DownloadPolicy, + type PackageDownload, + type RunnerDownloadErrorCode, +} from '../../src/runner-release/download'; +import { releaseKeyId } from '../../src/runner-release/verify'; +import { startServer, type Harness } from './server-fakes'; + +// The bounded release transport: the metadata reader's bounds and +// encodings, signed records verified on their exact bytes, and the package +// downloader's URL construction, redirect policy, credentials, byte +// accounting, timeouts and cleanup. Every response here is a fake fetch's +// except the development branch, which runs against the real server. + +const VERSION = '1.2.3'; +const FILE = runnerPackageFile({ os: 'linux', arch: 'x64' }, VERSION); +const PAGE = `${OFFICIAL_RELEASE_PREFIX}/v${VERSION}/${FILE}`; +const CDN = 'https://release-assets.githubusercontent.com/github-production-release-asset/1/2?sp=r&sig=abc'; +const OFFICIAL: DownloadPolicy = { kind: 'official' }; + +const sha256 = (bytes: Uint8Array | string): string => createHash('sha256').update(bytes).digest('hex'); +/** A package's bytes: several stream chunks' worth of random data. */ +const PACKAGE = randomBytes(70_000); + +const dirs: string[] = []; +let h: Harness | null = null; +afterEach(async () => { + await h?.close(); + h = null; + useServerDeps(null); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const tmp = (): string => { + const dir = mkdtempSync(join(tmpdir(), 'puck-download-')); + dirs.push(dir); + return dir; +}; + +function request(overrides: Partial = {}): PackageDownload { + return { version: VERSION, file: FILE, sha256: sha256(PACKAGE), size: PACKAGE.length, dest: join(tmp(), FILE), ...overrides }; +} + +interface Call { + url: string; + init: RequestInit; +} + +interface BodyControl { + /** Set once the reader cancelled the stream. */ + cancelled: boolean; + /** Set once the reader pulled from the stream. */ + pulled: boolean; +} + +/** A response whose body streams `chunks`; `stall` keeps it open after them, so only cancellation ends it. */ +function streamed(chunks: Uint8Array[], init: ResponseInit & { stall?: boolean } = {}): { res: Response; body: BodyControl } { + const body: BodyControl = { cancelled: false, pulled: false }; + let at = 0; + const stream = new ReadableStream({ + pull(controller) { + body.pulled = true; + if (at < chunks.length) controller.enqueue(chunks[at++]); + else if (!init.stall) controller.close(); + else return new Promise(() => undefined); + }, + cancel() { + body.cancelled = true; + }, + }, { highWaterMark: 0 }); + const rest: ResponseInit = { status: init.status, headers: init.headers }; + return { res: new Response(stream, { status: 200, ...rest }), body }; +} + +const chunked = (bytes: Uint8Array, size = 16 * 1024): Uint8Array[] => { + const out: Uint8Array[] = []; + for (let at = 0; at < bytes.length; at += size) out.push(bytes.subarray(at, at + size)); + return out; +}; + +const redirect = (location: string | null, status = 302): { res: Response; body: BodyControl } => + streamed([new TextEncoder().encode('moved')], { status, headers: location === null ? {} : { location } }); + +type Route = (url: string, init: RequestInit) => Promise | Response; + +/** A fetch answering from `routes` in order of the URLs it sees; records every call. */ +function fakeFetch(routes: Record): { fetch: typeof fetch; calls: Call[] } { + const calls: Call[] = []; + const fetchImpl = (async (input: string | URL | Request, init: RequestInit = {}) => { + const url = String(input); + calls.push({ url, init }); + const route = routes[url]; + if (!route) throw new Error(`unexpected fetch ${url}`); + const next = Array.isArray(route) ? route.shift() : route; + if (!next) throw new Error(`no more answers for ${url}`); + return next(url, init); + }) as typeof fetch; + return { fetch: fetchImpl, calls }; +} + +async function failure(promise: Promise): Promise<{ code: RunnerDownloadErrorCode; message: string }> { + try { + await promise; + } catch (err) { + if (err instanceof RunnerDownloadError) return { code: err.code, message: err.message }; + throw err; + } + throw new Error('expected a RunnerDownloadError'); +} + +/** The one shape of every request: GET, manual redirects, no credentials, only the two headers. */ +function expectPlainRequest(call: Call): void { + expect(call.init.method).toBe('GET'); + expect(call.init.redirect).toBe('manual'); + expect(call.init.credentials).toBe('omit'); + expect(call.init.headers).toEqual({ accept: 'application/octet-stream', 'accept-encoding': 'identity' }); + expect(new URL(call.url).username).toBe(''); + expect(new URL(call.url).password).toBe(''); +} + +describe('official package URLs', () => { + it('are built from the release prefix, the tag and the exact file name, never from a server', () => { + expect(OFFICIAL_RELEASE_PREFIX).toBe('https://github.com/namikmesic/puck/releases/download'); + expect(officialPackageUrl(VERSION, FILE)).toBe(PAGE); + for (const t of RUNNER_TARGETS) expect(officialPackageUrl('10.0.0', runnerPackageFile(t, '10.0.0'))).toBe(`${OFFICIAL_RELEASE_PREFIX}/v10.0.0/${runnerPackageFile(t, '10.0.0')}`); + }); + + it('refuse a non-canonical version or a file that is not a package of it', () => { + for (const version of ['v1.2.3', '01.2.3', '1.2', '1.2.3-rc1', '1.2.3\n', '', '../1.2.3']) { + expect(() => officialPackageUrl(version, FILE), version).toThrow(/not a release version/); + } + for (const file of ['puck-runner-linux-x64-1.2.4.tar.gz', 'puck-runner-windows-x64-1.2.3.tar.gz', 'puck-runner-linux-x64-1.2.3.tar.gz/../x', '../SHA256SUMS', 'runner-release.json', '']) { + expect(() => officialPackageUrl(VERSION, file), file).toThrow(/not a package of runner/); + } + }); +}); + +describe('the redirect policy', () => { + const expected = { version: VERSION, file: FILE }; + const refused = (url: string, hop = 1, policy: DownloadPolicy = OFFICIAL): string => { + try { + checkDownloadUrl(new URL(url), policy, expected, hop); + } catch (err) { + if (err instanceof RunnerDownloadError && err.code === 'bad-url') return err.message; + throw err; + } + throw new Error(`expected ${url} to be refused`); + }; + + it('allows exactly the two release hosts over https', () => { + expect(DOWNLOAD_HOSTS).toEqual(['github.com', 'release-assets.githubusercontent.com']); + expect(() => checkDownloadUrl(new URL(PAGE), OFFICIAL, expected, 0)).not.toThrow(); + expect(() => checkDownloadUrl(new URL(CDN), OFFICIAL, expected, 1)).not.toThrow(); + expect(() => checkDownloadUrl(new URL('https://GITHUB.COM/namikmesic/puck/releases/download/v1.2.3/' + FILE), OFFICIAL, expected, 1)).not.toThrow(); + for (const url of [ + 'https://evil.example/x', + 'https://github.com.evil.example/x', + 'https://evilgithub.com/x', + 'https://release-assets.githubusercontent.com.evil.example/x', + 'https://objects.githubusercontent.com/x', + 'https://api.github.com/x', + 'https://gіthub.com/x', + 'https://github.com./namikmesic/puck/releases/download/v1.2.3/' + FILE, + ]) { + expect(refused(url), url).toMatch(/not a release host/); + } + }); + + it('refuses an https downgrade, credentials, a port, a fragment and other schemes', () => { + expect(refused(`http://github.com/namikmesic/puck/releases/download/v${VERSION}/${FILE}`)).toMatch(/not https/); + expect(refused('ftp://github.com/x')).toMatch(/not https/); + expect(refused('https://user:pw@release-assets.githubusercontent.com/x')).toMatch(/carries credentials/); + expect(refused('https://user@release-assets.githubusercontent.com/x')).toMatch(/carries credentials/); + expect(refused('https://release-assets.githubusercontent.com:8443/x')).toMatch(/names a port/); + expect(refused(`https://github.com:444/namikmesic/puck/releases/download/v${VERSION}/${FILE}`)).toMatch(/names a port/); + expect(refused('https://release-assets.githubusercontent.com/x#fragment')).toMatch(/fragment/); + // The default port is no port. + expect(() => checkDownloadUrl(new URL('https://release-assets.githubusercontent.com:443/x'), OFFICIAL, expected, 1)).not.toThrow(); + }); + + it('compares the github.com path segment by segment with the expected release page', () => { + const prefix = 'https://github.com/namikmesic/puck/releases/download'; + for (const url of [ + `${prefix}/v${VERSION}/${FILE}?token=x`, + `${prefix}/v${VERSION}/${FILE}/`, + `${prefix}/v${VERSION}/${FILE}/extra`, + `${prefix}/v${VERSION}/puck-runner-linux-arm64-${VERSION}.tar.gz`, + `${prefix}/v9.9.9/${FILE}`, + `${prefix}-x/v${VERSION}/${FILE}`, + `https://github.com/namikmesic/puckx/releases/download/v${VERSION}/${FILE}`, + `https://github.com/namikmesic/puck/releases/downloads/v${VERSION}/${FILE}`, + `https://github.com/namikmesicx/puck/releases/download/v${VERSION}/${FILE}`, + `https://github.com/other/puck/releases/download/v${VERSION}/${FILE}`, + `https://github.com/namikmesic/puck/releases/download/v${VERSION}`, + `https://github.com/namikmesic/puck/releases/download/v${VERSION}/../v9.9.9/${FILE}`, + `https://github.com/namikmesic/puck/releases/download/v${VERSION}/%2e%2e/v9.9.9/${FILE}`, + `https://github.com/namikmesic/puck/releases/download//v${VERSION}/${FILE}`, + 'https://github.com/', + ]) { + expect(refused(url), url).toMatch(/not the release page|empty or relative path segment/); + } + // The comparison is on the parsed path, where the URL parser has already resolved dot segments. + expect(() => checkDownloadUrl(new URL(`https://github.com/namikmesic/puck/releases/download/./v${VERSION}/${FILE}`), OFFICIAL, expected, 1)).not.toThrow(); + expect(() => checkDownloadUrl(new URL(`https://github.com/namikmesic/puck/x/../releases/download/v${VERSION}/${FILE}`), OFFICIAL, expected, 1)).not.toThrow(); + // A string prefix would pass these; the segment comparison does not. + expect(`${prefix}-x/v${VERSION}/${FILE}`.startsWith(prefix)).toBe(true); + expect(`${prefix}/v${VERSION}/${FILE}/extra`.startsWith(`${prefix}/v${VERSION}/${FILE}`)).toBe(true); + }); + + it('refuses empty and relative segments on the asset host too', () => { + for (const url of ['https://release-assets.githubusercontent.com', 'https://release-assets.githubusercontent.com//x', 'https://release-assets.githubusercontent.com/a//b']) { + expect(refused(url), url).toMatch(/empty or relative path segment/); + } + }); + + it('under the development policy allows http on the configured origin only, and no redirect at all', () => { + const policy: DownloadPolicy = { kind: 'development', origin: 'http://puck.test:8765' }; + expect(() => checkDownloadUrl(new URL(`http://puck.test:8765/runner/${VERSION}/${FILE}`), policy, expected, 0)).not.toThrow(); + expect(refused(`http://puck.test:8765/runner/${VERSION}/${FILE}`, 1, policy)).toMatch(/must not redirect/); + for (const url of [`http://puck.test/runner/${VERSION}/${FILE}`, `https://puck.test:8765/runner/${VERSION}/${FILE}`, `http://127.0.0.1:8765/runner/${VERSION}/${FILE}`, PAGE, CDN]) { + expect(refused(url, 0, policy), url).toMatch(/not on the development server http:\/\/puck.test:8765/); + } + expect(refused('http://user:pw@puck.test:8765/x', 0, policy)).toMatch(/carries credentials/); + expect(refused('http://puck.test:8765/x#y', 0, policy)).toMatch(/fragment/); + expect(refused('http://puck.test:8765//b', 0, policy)).toMatch(/empty or relative path segment/); + expect(refused('http://puck.test:8765/x', 0, { kind: 'development', origin: 'not a url' })).toMatch(/origin "not a url" is not a URL/); + expect(refused('http://puck.test:8765/x', 0, { kind: 'development', origin: 'file:///tmp' })).toMatch(/not on the development server/); + }); +}); + +describe('downloadRunnerPackage', () => { + it('follows the release page to the asset host, sends no credentials, cancels the redirect and stores the exact bytes', async () => { + const page = redirect(CDN); + const { fetch, calls } = fakeFetch({ [PAGE]: () => page.res, [CDN]: () => streamed(chunked(PACKAGE), { headers: { 'content-length': String(PACKAGE.length) } }).res }); + const req = request(); + const result = await downloadRunnerPackage(req, OFFICIAL, { fetch }); + expect(result).toEqual({ url: CDN, size: PACKAGE.length, sha256: sha256(PACKAGE) }); + expect(calls.map((c) => c.url)).toEqual([PAGE, CDN]); + for (const call of calls) expectPlainRequest(call); + expect(page.body.cancelled).toBe(true); + expect(readFileSync(req.dest).equals(PACKAGE)).toBe(true); + expect(statSync(req.dest).mode & 0o777).toBe(0o600); + }); + + it('never carries a token, whatever the caller has around', async () => { + const { fetch, calls } = fakeFetch({ [PAGE]: () => streamed(chunked(PACKAGE)).res }); + process.env.GITHUB_TOKEN = 'ghp_never'; + try { + await downloadRunnerPackage(request(), OFFICIAL, { fetch }); + } finally { + delete process.env.GITHUB_TOKEN; + } + const headers = Object.keys(calls[0].init.headers as Record).map((k) => k.toLowerCase()); + expect(headers).not.toContain('authorization'); + expect(headers).not.toContain('cookie'); + expect(headers).not.toContain('x-runner-token'); + expect(JSON.stringify(calls)).not.toContain('ghp_never'); + }); + + it('takes every redirect status, absolute or relative Locations, and at most MAX_REDIRECTS hops', async () => { + expect(MAX_REDIRECTS).toBe(5); + const hop = (n: number): string => `https://release-assets.githubusercontent.com/hop/${n}`; + for (const status of [301, 302, 303, 307, 308]) { + const { fetch, calls } = fakeFetch({ + [PAGE]: () => redirect('https://release-assets.githubusercontent.com/hop/1', status).res, + [hop(1)]: () => redirect('/hop/2', status).res, + [hop(2)]: () => redirect('../hop/3', status).res, + [hop(3)]: () => redirect('4', status).res, + [hop(4)]: () => redirect(`https://github.com/namikmesic/puck/releases/download/v${VERSION}/${FILE}`, status).res, + [PAGE + '']: [() => redirect('https://release-assets.githubusercontent.com/hop/1', status).res, () => streamed(chunked(PACKAGE)).res], + }); + const req = request(); + await downloadRunnerPackage(req, OFFICIAL, { fetch }); + expect(calls.map((c) => c.url)).toEqual([PAGE, hop(1), hop(2), hop(3), hop(4), PAGE]); + expect(readFileSync(req.dest).equals(PACKAGE)).toBe(true); + } + const six = fakeFetch({ [PAGE]: () => redirect(hop(1)).res, ...Object.fromEntries([1, 2, 3, 4, 5, 6].map((n) => [hop(n), () => redirect(hop(n + 1)).res])) }); + const req = request(); + expect(await failure(downloadRunnerPackage(req, OFFICIAL, { fetch: six.fetch }))).toEqual({ code: 'too-many-redirects', message: 'The download redirected more than 5 times.' }); + expect(six.calls).toHaveLength(6); + expect(existsSync(req.dest)).toBe(false); + }); + + it('refuses a redirect outside the policy before requesting it', async () => { + for (const [location, message] of [ + ['https://evil.example/x', /not a release host/], + [`http://github.com/namikmesic/puck/releases/download/v${VERSION}/${FILE}`, /not https/], + ['https://user:pw@release-assets.githubusercontent.com/x', /carries credentials/], + ['https://release-assets.githubusercontent.com:8443/x', /names a port/], + ['https://release-assets.githubusercontent.com/x#f', /fragment/], + ['https://release-assets.githubusercontent.com/x\tSet-Cookie: a', /control character or a space/], + ['https://release-assets.githubusercontent.com/x\u0001y', /control character or a space/], + ['https://release-assets.githubusercontent.com/a b', /control character or a space/], + ['http://[', /not a URL/], + [null, /names no Location/], + ] as [string | null, RegExp][]) { + const { fetch, calls } = fakeFetch({ [PAGE]: () => redirect(location).res }); + const req = request(); + const f = await failure(downloadRunnerPackage(req, OFFICIAL, { fetch })); + expect(f.code, String(location)).toBe('bad-url'); + expect(f.message, String(location)).toMatch(message); + expect(calls, String(location)).toHaveLength(1); + expect(existsSync(req.dest)).toBe(false); + } + // An empty Location is the page itself; the hop limit ends that loop. + const self = fakeFetch({ [PAGE]: () => redirect('').res }); + expect((await failure(downloadRunnerPackage(request(), OFFICIAL, { fetch: self.fetch }))).code).toBe('too-many-redirects'); + expect(self.calls).toHaveLength(MAX_REDIRECTS + 1); + }); + + it('refuses a URL under the official policy, and a package over the absolute cap, without any request', async () => { + const { fetch, calls } = fakeFetch({}); + expect((await failure(downloadRunnerPackage(request({ url: CDN }), OFFICIAL, { fetch }))).message).toMatch(/takes no URL/); + expect(await failure(downloadRunnerPackage(request({ size: MAX_PACKAGE_BYTES + 1 }), OFFICIAL, { fetch }))).toEqual({ + code: 'too-large', + message: `The package is ${MAX_PACKAGE_BYTES + 1} bytes; at most ${MAX_PACKAGE_BYTES} are downloaded.`, + }); + expect(MAX_PACKAGE_BYTES).toBe(256 * 1024 * 1024); + for (const size of [0, -1, 1.5, Number.NaN]) expect((await failure(downloadRunnerPackage(request({ size }), OFFICIAL, { fetch }))).code).toBe('size-mismatch'); + expect((await failure(downloadRunnerPackage(request({ sha256: 'ABC' }), OFFICIAL, { fetch }))).code).toBe('digest-mismatch'); + expect((await failure(downloadRunnerPackage(request({ version: 'v1' }), OFFICIAL, { fetch }))).code).toBe('bad-url'); + expect(calls).toHaveLength(0); + }); + + it('rejects any status but 200 after the redirects, and a content encoding', async () => { + for (const status of [404, 403, 500, 206, 204]) { + const { fetch } = fakeFetch({ [PAGE]: () => (status === 204 ? new Response(null, { status }) : streamed(chunked(PACKAGE), { status }).res) }); + const req = request(); + expect(await failure(downloadRunnerPackage(req, OFFICIAL, { fetch })), String(status)).toEqual({ code: 'http-status', message: `github.com answered ${status} for ${FILE}.` }); + expect(existsSync(req.dest)).toBe(false); + } + for (const encoding of ['gzip', 'br', 'GZIP', ' deflate ']) { + const body = streamed(chunked(PACKAGE), { headers: { 'content-encoding': encoding } }); + const { fetch } = fakeFetch({ [PAGE]: () => body.res }); + const req = request(); + expect((await failure(downloadRunnerPackage(req, OFFICIAL, { fetch }))).code, encoding).toBe('content-encoding'); + expect(body.body.pulled).toBe(false); + expect(body.body.cancelled).toBe(true); + expect(existsSync(req.dest)).toBe(false); + } + const identity = streamed(chunked(PACKAGE), { headers: { 'content-encoding': 'identity' } }); + await downloadRunnerPackage(request(), OFFICIAL, { fetch: fakeFetch({ [PAGE]: () => identity.res }).fetch }); + }); + + it('checks Content-Length against the signed size before reading a byte', async () => { + for (const length of [String(PACKAGE.length + 1), String(PACKAGE.length - 1), 'abc', '-1', '1e3']) { + const body = streamed(chunked(PACKAGE), { headers: { 'content-length': length } }); + const { fetch } = fakeFetch({ [PAGE]: () => body.res }); + const req = request(); + expect((await failure(downloadRunnerPackage(req, OFFICIAL, { fetch }))).code, length).toBe('size-mismatch'); + expect(body.body.pulled, length).toBe(false); + expect(body.body.cancelled, length).toBe(true); + expect(existsSync(req.dest)).toBe(false); + } + }); + + it('rejects a short body, a long one and a wrong digest, cancelling the response and removing the file', async () => { + const cases: [string, Uint8Array[], Partial, RunnerDownloadErrorCode, RegExp][] = [ + ['short', chunked(PACKAGE.subarray(0, PACKAGE.length - 1)), {}, 'truncated', /ended after 69999 of 70000 bytes/], + ['empty', [], {}, 'truncated', /ended after 0 of 70000 bytes/], + ['long', chunked(Buffer.concat([PACKAGE, Buffer.from([1])])), {}, 'size-mismatch', /longer than the 70000 bytes/], + ['digest', chunked(PACKAGE), { sha256: 'a'.repeat(64) }, 'digest-mismatch', /does not match the sha256/], + ['same length, other bytes', chunked(randomBytes(PACKAGE.length)), {}, 'digest-mismatch', /does not match the sha256/], + ]; + for (const [name, chunks, overrides, code, message] of cases) { + const body = streamed(chunks, { stall: name === 'long' }); + const { fetch } = fakeFetch({ [PAGE]: () => body.res }); + const req = request(overrides); + const f = await failure(downloadRunnerPackage(req, OFFICIAL, { fetch })); + expect(f.code, name).toBe(code); + expect(f.message, name).toMatch(message); + expect(existsSync(req.dest), name).toBe(false); + if (name === 'long') expect(body.body.cancelled).toBe(true); + } + }); + + it('stops on the caller\'s signal mid-stream, cancelling the response and removing the file', async () => { + const body = streamed(chunked(PACKAGE).slice(0, 1), { stall: true }); + const { fetch } = fakeFetch({ [PAGE]: () => body.res }); + const controller = new AbortController(); + const req = request(); + const pending = failure(downloadRunnerPackage(req, OFFICIAL, { fetch, signal: controller.signal })); + await new Promise((r) => setTimeout(r, 20)); + controller.abort(); + expect(await pending).toEqual({ code: 'cancelled', message: 'The download was cancelled.' }); + expect(body.body.cancelled).toBe(true); + expect(existsSync(req.dest)).toBe(false); + const already = new AbortController(); + already.abort(); + expect((await failure(downloadRunnerPackage(request(), OFFICIAL, { fetch, signal: already.signal }))).code).toBe('cancelled'); + }); + + it('times out waiting for headers and for the whole body', async () => { + expect(HEADER_TIMEOUT_MS).toBe(30_000); + expect(BODY_TIMEOUT_MS).toBe(15 * 60_000); + const hanging = (async (_url: string, init: RequestInit) => + new Promise((_resolve, reject) => init.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError'))))) as unknown as typeof fetch; + const req = request(); + expect(await failure(downloadRunnerPackage(req, OFFICIAL, { fetch: hanging, headerTimeoutMs: 20 }))).toEqual({ code: 'timeout', message: `The download of ${FILE} timed out.` }); + expect(existsSync(req.dest)).toBe(false); + + const body = streamed(chunked(PACKAGE).slice(0, 2), { stall: true }); + const { fetch } = fakeFetch({ [PAGE]: () => body.res }); + const slow = request(); + expect((await failure(downloadRunnerPackage(slow, OFFICIAL, { fetch, bodyTimeoutMs: 30 }))).code).toBe('timeout'); + expect(body.body.cancelled).toBe(true); + expect(existsSync(slow.dest)).toBe(false); + }); + + it('reports an unreachable host and never touches an existing file at dest', async () => { + const down = (async () => { + throw new TypeError('fetch failed'); + }) as unknown as typeof fetch; + const req = request(); + expect(await failure(downloadRunnerPackage(req, OFFICIAL, { fetch: down }))).toEqual({ code: 'unreachable', message: 'Cannot reach https://github.com: fetch failed' }); + const { fetch } = fakeFetch({ [PAGE]: () => streamed(chunked(PACKAGE)).res }); + const taken = request(); + writeFileSync(taken.dest, 'keep me'); + expect((await failure(downloadRunnerPackage(taken, OFFICIAL, { fetch }))).code).toBe('write-failed'); + expect(readFileSync(taken.dest, 'utf8')).toBe('keep me'); + }); + + it('downloads from a development server over http on its own origin, with no redirect', async () => { + const downloads = tmp(); + mkdirSync(join(downloads, VERSION)); + writeFileSync(join(downloads, VERSION, FILE), PACKAGE); + h = await startServer({ PUCK_DEVELOPMENT: 'true', PUCK_RUNNER_DOWNLOADS: downloads }); + const policy: DownloadPolicy = { kind: 'development', origin: h.base }; + const listed = await new ServerApi(h.base).releases(); + const asset = listed.assets.find((a) => a.file === FILE); + expect(asset).toBeDefined(); + // The server names its public URL; a client points that at the origin it configured. + const url = new URL(asset?.url ?? ''); + const local = `${h.base}${url.pathname}`; + const req = request({ url: local }); + expect(await downloadRunnerPackage(req, policy)).toEqual({ url: local, size: PACKAGE.length, sha256: sha256(PACKAGE) }); + expect(readFileSync(req.dest).equals(PACKAGE)).toBe(true); + + const { fetch, calls } = fakeFetch({}); + for (const other of [asset?.url ?? '', PAGE, `${h.base.replace('127.0.0.1', 'localhost')}${url.pathname}`, `http://user:pw@${new URL(h.base).host}${url.pathname}`]) { + expect((await failure(downloadRunnerPackage(request({ url: other }), policy, { fetch }))).code, other).toBe('bad-url'); + } + expect((await failure(downloadRunnerPackage(request(), policy, { fetch }))).message).toMatch(/needs the server's URL/); + expect(calls).toHaveLength(0); + const redirecting = fakeFetch({ [local]: () => redirect(`${h.base}/elsewhere`).res }); + expect((await failure(downloadRunnerPackage(request({ url: local }), policy, { fetch: redirecting.fetch }))).message).toMatch(/must not redirect/); + expect(redirecting.calls).toHaveLength(1); + }); +}); + +describe('readBoundedBody', () => { + it('returns the exact bytes within the bound', async () => { + const bytes = randomBytes(40_000); + expect(Buffer.from(await readBoundedBody(streamed(chunked(bytes, 7_000)).res, 40_000)).equals(bytes)).toBe(true); + expect(await readBoundedBody(new Response(null), 10)).toEqual(new Uint8Array(0)); + }); + + it('refuses an announced or a streamed overrun, cancelling the body', async () => { + const announced = streamed(chunked(randomBytes(10)), { headers: { 'content-length': '11' } }); + expect(await failure(readBoundedBody(announced.res, 10))).toEqual({ code: 'too-large', message: 'The response announces 11 bytes; at most 10 are read.' }); + expect(announced.body.pulled).toBe(false); + expect(announced.body.cancelled).toBe(true); + const streamedOver = streamed(chunked(randomBytes(30), 7), { stall: true }); + expect(await failure(readBoundedBody(streamedOver.res, 20))).toEqual({ code: 'too-large', message: 'The response is over 20 bytes.' }); + expect(streamedOver.body.cancelled).toBe(true); + expect(Buffer.from(await readBoundedBody(streamed(chunked(randomBytes(20), 7)).res, 20))).toHaveLength(20); + }); + + it('refuses a content encoding or an unreadable Content-Length, cancelling the body unread', async () => { + const encoded = streamed(chunked(randomBytes(10)), { headers: { 'content-encoding': 'gzip' } }); + expect((await failure(readBoundedBody(encoded.res, 100))).code).toBe('content-encoding'); + expect(encoded.body.pulled).toBe(false); + expect(encoded.body.cancelled).toBe(true); + const odd = streamed(chunked(randomBytes(10)), { headers: { 'content-length': 'ten' } }); + expect((await failure(readBoundedBody(odd.res, 100))).code).toBe('size-mismatch'); + expect(odd.body.pulled).toBe(false); + expect(odd.body.cancelled).toBe(true); + }); +}); + +describe('release listings', () => { + it('parses bounded UTF-8 JSON and hands back the exact bytes', async () => { + const text = '{"latest":"1.2.3","releases":[]}'; + const { bytes, value } = await readReleaseListing(new Response(text)); + expect(Buffer.from(bytes).toString('utf8')).toBe(text); + expect(value).toEqual({ latest: '1.2.3', releases: [] }); + const encoded = await readReleaseListing(new Response(text, { headers: { 'content-encoding': 'gzip' } })); + expect(Buffer.from(encoded.bytes).toString('utf8')).toBe(text); + expect(encoded.value).toEqual({ latest: '1.2.3', releases: [] }); + for (const body of [new Uint8Array([0x7b, 0xff, 0x7d]), new TextEncoder().encode('{"a":'), new TextEncoder().encode('{}')]) { + await expect(readReleaseListing(new Response(body))).rejects.toMatchObject({ name: 'RunnerReleaseError', code: 'malformed' }); + } + const big = streamed(chunked(Buffer.alloc(MAX_RELEASE_METADATA_BYTES + 1, 0x20), 65_536)); + expect((await failure(readReleaseListing(big.res))).code).toBe('too-large'); + expect(big.body.cancelled).toBe(true); + }); + + it('bounds the app\'s and the runner\'s release readers, and only those', async () => { + const huge = Buffer.alloc(MAX_RELEASE_METADATA_BYTES + 1, 0x20); + const seen: Call[] = []; + const fetchImpl = (async (input: string | URL | Request, init: RequestInit = {}) => { + seen.push({ url: String(input), init }); + return streamed(chunked(huge, 65_536), { headers: { 'content-type': 'application/json' } }).res; + }) as typeof fetch; + useServerDeps({ fetch: fetchImpl as never }, 'http://puck.test'); + expect((await failure(mainApi.releases())).code).toBe('too-large'); + expect(seen[0].url).toBe('http://puck.test/v1/runner/releases'); + expect((seen[0].init.headers as Record)['Accept-Encoding']).toBeUndefined(); + // Other requests read as before: the whole (blank) body, parsed leniently. + expect(await serverRequest('GET', '/v1/me')).toBeNull(); + expect((seen[1].init.headers as Record)['Accept-Encoding']).toBeUndefined(); + + const runner = new ServerApi('http://puck.test', fetchImpl); + expect((await failure(runner.releases())).code).toBe('too-large'); + expect((seen[2].init.headers as Record)['Accept-Encoding']).toBeUndefined(); + expect(seen[2].init.headers).not.toHaveProperty('Authorization'); + }); + + it('reads a content-encoded listing within the byte bound', async () => { + const listing = JSON.stringify({ latest: VERSION, minVersion: null, assets: [] }); + const fetchImpl = (async () => + streamed(chunked(Buffer.from(listing)), { headers: { 'content-type': 'application/json', 'content-encoding': 'gzip' } }).res) as typeof fetch; + useServerDeps({ fetch: fetchImpl as never }, 'http://puck.test'); + expect(await mainApi.releases()).toEqual({ latest: VERSION, minVersion: null, assets: [] }); + expect(await new ServerApi('http://puck.test', fetchImpl).releases()).toEqual({ latest: VERSION, minVersion: null, assets: [] }); + }); + + it('reads a development server listing through both bounded readers', async () => { + const downloads = tmp(); + mkdirSync(join(downloads, VERSION)); + writeFileSync(join(downloads, VERSION, FILE), PACKAGE); + h = await startServer({ PUCK_DEVELOPMENT: 'true', PUCK_RUNNER_DOWNLOADS: downloads }); + const probe = await fetch(`${h.base}/v1/runner/releases`); + expect(probe.headers.get('content-encoding') ?? 'identity').toBe('identity'); + await probe.body?.cancel(); + const expected = { + latest: VERSION, + minVersion: null, + assets: [ + { + os: 'linux', + arch: 'x64', + version: VERSION, + file: FILE, + url: `http://puck.test/runner/${VERSION}/${FILE}`, + sha256: sha256(PACKAGE), + size: PACKAGE.length, + }, + ], + }; + useServerDeps({}, h.base); + expect(await mainApi.releases()).toEqual(expected); + expect(await new ServerApi(h.base).releases()).toEqual(expected); + }); +}); + +describe('verifySignedRunnerReleases', () => { + interface TestKey { + privateKey: KeyObject; + pem: string; + id: string; + } + const ed25519 = (): TestKey => { + const { privateKey, publicKey } = generateKeyPairSync('ed25519'); + return { privateKey, pem: publicKey.export({ type: 'spki', format: 'pem' }) as string, id: releaseKeyId(publicKey) }; + }; + const manifest = (key: TestKey, version = '2.0.0'): RunnerReleaseManifest => ({ + schemaVersion: 1, + product: 'puck-runner', + publisher: 'namikmesic/puck', + version, + sourceCommit: 'f'.repeat(40), + runnerProtocol: 1, + signingKeyId: key.id, + sha256sumsSha256: 'a'.repeat(64), + assets: RUNNER_TARGETS.map((t, i) => ({ os: t.os, arch: t.arch, file: runnerPackageFile(t, version), sha256: sha256(`p${i}`), size: 100 + i })), + }); + const record = (key: TestKey, text: string) => { + const bytes = new TextEncoder().encode(text); + return formatSignedRunnerRelease(bytes, sign(null, bytes, key.privateKey)); + }; + const releaseFailure = (fn: () => unknown): string => { + try { + fn(); + } catch (err) { + if (err instanceof RunnerReleaseError) return err.code; + throw err; + } + throw new Error('expected a RunnerReleaseError'); + }; + + it('verifies each record on its exact bytes and returns them with the manifest', () => { + const key = ed25519(); + const texts = [formatRunnerRelease(manifest(key, '2.0.0')), formatRunnerRelease(manifest(key, '2.1.0'))]; + const verified = verifySignedRunnerReleases(texts.map((t) => record(key, t)), [key.pem]); + expect(verified.map((v) => v.manifest.version)).toEqual(['2.0.0', '2.1.0']); + expect(verified.map((v) => v.keyId)).toEqual([key.id, key.id]); + expect(verified.map((v) => Buffer.from(v.bytes).toString('utf8'))).toEqual(texts); + expect(verified[0].signature).toHaveLength(64); + expect(verifySignedRunnerReleases([], [key.pem])).toEqual([]); + }); + + it('fails closed on the compiled roots, and refuses another key, altered bytes and a re-serialised manifest', () => { + const key = ed25519(); + const other = ed25519(); + const text = formatRunnerRelease(manifest(key)); + const good = record(key, text); + expect(releaseFailure(() => verifySignedRunnerReleases([good]))).toBe('no-trusted-key'); + expect(releaseFailure(() => verifySignedRunnerReleases([good], [other.pem]))).toBe('bad-signature'); + const reserialised = { ...good, manifest: Buffer.from(JSON.stringify(JSON.parse(text))).toString('base64') }; + expect(releaseFailure(() => verifySignedRunnerReleases([reserialised], [key.pem]))).toBe('bad-signature'); + const signedByOther = record(other, text); + expect(releaseFailure(() => verifySignedRunnerReleases([signedByOther], [key.pem, other.pem]))).toBe('key-id-mismatch'); + // Signed garbage verifies, then fails to read; unsigned garbage never gets that far. + expect(releaseFailure(() => verifySignedRunnerReleases([good, record(key, 'not json')], [key.pem]))).toBe('malformed'); + expect(releaseFailure(() => verifySignedRunnerReleases([{ ...good, signature: Buffer.alloc(64).toString('base64') }], [key.pem]))).toBe('bad-signature'); + }); + + it('bounds the records before any signature check', () => { + const key = ed25519(); + const good = record(key, formatRunnerRelease(manifest(key))); + expect(releaseFailure(() => verifySignedRunnerReleases(Array.from({ length: MAX_RELEASE_RECORDS + 1 }, () => good), [key.pem]))).toBe('malformed'); + expect(releaseFailure(() => verifySignedRunnerReleases([{ ...good, manifest: Buffer.alloc(MAX_MANIFEST_BYTES + 1, 0x20).toString('base64') }], [key.pem]))).toBe('malformed'); + expect(releaseFailure(() => verifySignedRunnerReleases([{ ...good, manifest: `${good.manifest}\n` }], [key.pem]))).toBe('malformed'); + expect(releaseFailure(() => verifySignedRunnerReleases({ records: [good] }, [key.pem]))).toBe('malformed'); + }); +}); + + +describe('writeAll', () => { + /** A handle that persists at most `most` bytes per write and records what it got. */ + const sink = (most: number) => { + const written: number[] = []; + let calls = 0; + return { + written, + calls: () => calls, + write: async (data: Uint8Array) => { + calls++; + const take = Math.min(most, data.length); + written.push(...data.subarray(0, take)); + return { bytesWritten: take }; + }, + }; + }; + + it('keeps writing until every byte is persisted, in order', async () => { + const bytes = randomBytes(10); + const short = sink(3); + await writeAll(short, bytes); + expect(Buffer.from(short.written).equals(bytes)).toBe(true); + expect(short.calls()).toBe(4); + const whole = sink(1024); + await writeAll(whole, bytes); + expect(whole.calls()).toBe(1); + const empty = sink(3); + await writeAll(empty, new Uint8Array(0)); + expect(empty.calls()).toBe(0); + }); + + it('fails on a write that persists nothing rather than looping or reporting success', async () => { + const stuck = sink(0); + await expect(writeAll(stuck, randomBytes(4))).rejects.toThrow('wrote none of the remaining 4 bytes'); + expect(stuck.calls()).toBe(1); + const stalls = { write: async () => ({ bytesWritten: Number.NaN }) }; + await expect(writeAll(stalls, randomBytes(4))).rejects.toThrow(/wrote none/); + }); + + it('makes a short write fail a download with the file removed (the unpack case is in the archive suite)', async () => { + // A handle that writes half of what it is given, then nothing. + const half = (real: { write(data: Uint8Array): Promise<{ bytesWritten: number }> }) => { + let calls = 0; + return async (data: Uint8Array) => { + calls++; + if (calls > 1) return { bytesWritten: 0 }; + return real.write(data.subarray(0, Math.max(1, data.length >> 1))); + }; + }; + const fsPromises = (await import('node:fs')).promises; + const realOpen = fsPromises.open; + const opened: { write: (data: Uint8Array) => Promise<{ bytesWritten: number }> }[] = []; + (fsPromises as { open: unknown }).open = async (...args: unknown[]) => { + const handle = await (realOpen as (...a: unknown[]) => Promise<{ write(data: Uint8Array): Promise<{ bytesWritten: number }> }>)(...args); + const write = half(handle); + const patched = Object.create(handle) as typeof handle; + patched.write = write; + opened.push(patched); + return patched; + }; + try { + const { fetch } = fakeFetch({ [PAGE]: () => streamed(chunked(PACKAGE)).res }); + const req = request(); + const f = await failure(downloadRunnerPackage(req, OFFICIAL, { fetch })); + expect(f.code).toBe('write-failed'); + expect(f.message).toMatch(/wrote none of the remaining/); + expect(existsSync(req.dest)).toBe(false); + expect(opened).toHaveLength(1); + } finally { + (fsPromises as { open: unknown }).open = realOpen; + } + }); +}); diff --git a/test/unit/runner-releases.test.ts b/test/unit/runner-releases.test.ts index e22b436..5e63b65 100644 --- a/test/unit/runner-releases.test.ts +++ b/test/unit/runner-releases.test.ts @@ -1,17 +1,28 @@ import { describe, expect, it } from 'vitest'; import { checkSha256Sums, + ED25519_SIGNATURE_BYTES, formatRunnerRelease, formatSha256Sums, + formatSignedRunnerRelease, formatVersionProbe, isReleaseVersion, isRunnerTrustMode, + MAX_MANIFEST_BASE64_CHARS, + MAX_MANIFEST_BYTES, + MAX_RELEASE_METADATA_BYTES, + MAX_RELEASE_RECORDS, readRunnerRelease, readSha256Sums, + readSignedRunnerRelease, + readSignedRunnerReleases, readVersionProbe, + RUNNER_PACKAGE_ENTRIES, RUNNER_TARGETS, runnerPackageFile, + runnerReleaseTag, RunnerReleaseError, + runnerTargetFor, selectRunnerAsset, type RunnerReleaseErrorCode, type RunnerReleaseManifest, @@ -236,3 +247,95 @@ describe('version probe', () => { for (const bad of ['Production', 'dev', '', undefined]) expect(isRunnerTrustMode(bad)).toBe(false); }); }); + +describe('release tags and package targets', () => { + it('names the tag a release hangs off', () => { + expect(runnerReleaseTag('1.2.3')).toBe('v1.2.3'); + }); + + it("maps Node's platform names to the manifest's once: darwin is macos", () => { + expect(runnerTargetFor('darwin', 'arm64')).toEqual({ os: 'macos', arch: 'arm64' }); + expect(runnerTargetFor('linux', 'x64')).toEqual({ os: 'linux', arch: 'x64' }); + expect(runnerTargetFor('linux', 'arm64')).toEqual({ os: 'linux', arch: 'arm64' }); + for (const [platform, arch] of [ + ['win32', 'x64'], + ['freebsd', 'x64'], + ['linux', 'ia32'], + ['darwin', 'ppc64'], + ['', ''], + ]) { + expect(runnerTargetFor(platform, arch), `${platform}-${arch}`).toBeNull(); + } + }); + + it('lists the fixed package layout: nine regular files and bin/, with plain modes', () => { + expect(RUNNER_PACKAGE_ENTRIES.map((e) => e.name)).toEqual(['config.sh', 'run.sh', 'svc.sh', 'VERSION', 'README.md', 'LICENSE', 'bin/', 'bin/node', 'bin/node.LICENSE', 'bin/puck-runner.cjs']); + expect(RUNNER_PACKAGE_ENTRIES.filter((e) => e.type === 'file')).toHaveLength(9); + expect(RUNNER_PACKAGE_ENTRIES.filter((e) => e.type === 'dir').map((e) => e.name)).toEqual(['bin/']); + for (const e of RUNNER_PACKAGE_ENTRIES) { + expect([0o755, 0o644], e.name).toContain(e.mode); + expect(e.mode & 0o7000, e.name).toBe(0); + } + expect(RUNNER_PACKAGE_ENTRIES.filter((e) => e.mode === 0o755).map((e) => e.name)).toEqual(['config.sh', 'run.sh', 'svc.sh', 'bin/', 'bin/node']); + }); +}); + +describe('signed release records', () => { + const bytes = (n: number, fill = 0x7b): Uint8Array => new Uint8Array(n).fill(fill); + const sig = bytes(ED25519_SIGNATURE_BYTES, 0xab); + + it('carries the exact bytes through standard base64, and reads them back unchanged', () => { + const manifest = new TextEncoder().encode('{"exact": "bytes"}\n'); + const record = formatSignedRunnerRelease(manifest, sig); + expect(record).toEqual({ manifest: Buffer.from(manifest).toString('base64'), signature: Buffer.from(sig).toString('base64') }); + expect(readSignedRunnerRelease(record)).toEqual({ manifest, signature: sig }); + expect(readSignedRunnerRelease(JSON.parse(JSON.stringify(record)))).toEqual({ manifest, signature: sig }); + // Every byte value survives, including the ones base64 encodes with + and /. + const all = new Uint8Array(256).map((_, i) => i); + expect(readSignedRunnerRelease(formatSignedRunnerRelease(all, sig)).manifest).toEqual(all); + expect(formatSignedRunnerRelease(bytes(MAX_MANIFEST_BYTES), sig).manifest).toHaveLength(MAX_MANIFEST_BASE64_CHARS); + }); + + it('is the record shape exactly', () => { + const record = formatSignedRunnerRelease(bytes(10), sig); + for (const value of [null, 'text', [], 42, { ...record, extra: 1 }, { manifest: record.manifest }, { signature: record.signature }, { ...record, manifest: 7 }, { ...record, signature: null }]) { + expect(failure(() => readSignedRunnerRelease(value)).code).toBe('malformed'); + } + }); + + it('takes canonical base64 only: no whitespace, no URL alphabet, full padding, zero trailing bits', () => { + const good = formatSignedRunnerRelease(bytes(10), sig); + for (const manifest of [`${good.manifest}\n`, good.manifest.replace(/^(..)/, '$1 '), 'e3t7e3t7e3t7e3s', 'e3t7e3t7e3t7e3s==', 'QR==', 'e3t7-_t7', '====', 'e3t7e3t7e3t7e3s=x']) { + expect(failure(() => readSignedRunnerRelease({ ...good, manifest })).message, manifest).toMatch(/base64/); + } + expect(readSignedRunnerRelease({ ...good, manifest: '' }).manifest).toEqual(new Uint8Array(0)); + expect(readSignedRunnerRelease({ ...good, manifest: 'QQ==' }).manifest).toEqual(new Uint8Array([0x41])); + }); + + it('bounds the manifest by its text before decoding and by its bytes after', () => { + const over = formatSignedRunnerRelease(bytes(MAX_MANIFEST_BYTES + 3), sig); + expect(over.manifest.length).toBeGreaterThan(MAX_MANIFEST_BASE64_CHARS); + expect(failure(() => readSignedRunnerRelease(over)).message).toMatch(`over ${MAX_MANIFEST_BASE64_CHARS} characters`); + // One byte over fits the text bound (the last base64 group is padded) and fails the byte bound. + const byOne = formatSignedRunnerRelease(bytes(MAX_MANIFEST_BYTES + 1), sig); + expect(byOne.manifest.length).toBe(MAX_MANIFEST_BASE64_CHARS); + expect(failure(() => readSignedRunnerRelease(byOne)).message).toMatch(`over ${MAX_MANIFEST_BYTES} bytes`); + expect(readSignedRunnerRelease(formatSignedRunnerRelease(bytes(MAX_MANIFEST_BYTES), sig)).manifest).toHaveLength(MAX_MANIFEST_BYTES); + }); + + it('requires a 64-byte signature, bounding its text first', () => { + for (const n of [0, 63, 65]) { + expect(failure(() => readSignedRunnerRelease(formatSignedRunnerRelease(bytes(10), bytes(n)))).message, String(n)).toMatch(`is ${n} bytes, not 64`); + } + expect(failure(() => readSignedRunnerRelease(formatSignedRunnerRelease(bytes(10), bytes(128)))).message).toMatch('over 88 characters'); + }); + + it('reads a listing of at most MAX_RELEASE_RECORDS records', () => { + const record = formatSignedRunnerRelease(bytes(10), sig); + expect(readSignedRunnerReleases([])).toEqual([]); + expect(readSignedRunnerReleases(Array.from({ length: MAX_RELEASE_RECORDS }, () => record))).toHaveLength(MAX_RELEASE_RECORDS); + expect(failure(() => readSignedRunnerReleases(Array.from({ length: MAX_RELEASE_RECORDS + 1 }, () => record))).message).toMatch(`at most ${MAX_RELEASE_RECORDS}`); + for (const value of [null, {}, 'x', [record, 1]]) expect(failure(() => readSignedRunnerReleases(value)).code).toBe('malformed'); + expect(MAX_RELEASE_METADATA_BYTES).toBe(4 * 1024 * 1024); + }); +});