From 5d37bd01a62172a21e1dea88e58055cf94db61a6 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 14:57:07 +0800 Subject: [PATCH 01/95] fix(worker): propagate RoomDO command rejections from POST /_/:room Previously, POST /_/:room dispatched commands to the room Durable Object and checked authorization status via authVerdict(writeRes). However, if the DO rejected a command for non-auth reasons (such as a 413 error when a command exceeds sheet limits), the worker ignored writeRes.ok and unconditionally returned HTTP 202 Accepted. This created a silent write-loss path where callers were informed that commands were accepted when they were actually rejected by the DO. Propagate non-2xx status and response text from the Durable Object fetch response back to the caller in both JSON and plain-text dispatch handlers. Also include unit tests for DO error propagation, legacy room accessibility, oversize sheet handling, and ETHERCALC_AUTH kill-switch behavior. --- packages/worker/src/routes/rooms.ts | 68 ++--- .../worker/test/auth-session.node.test.ts | 7 +- packages/worker/test/room.test.ts | 140 ++++++++++ packages/worker/test/routes-auth.node.test.ts | 27 ++ .../worker/test/routes-rooms.node.test.ts | 253 +++++++++++++++++- 5 files changed, 460 insertions(+), 35 deletions(-) diff --git a/packages/worker/src/routes/rooms.ts b/packages/worker/src/routes/rooms.ts index 6517c287..eda1882a 100644 --- a/packages/worker/src/routes/rooms.ts +++ b/packages/worker/src/routes/rooms.ts @@ -778,6 +778,9 @@ export function registerRoomRoutes(app: Hono): void { ); const denied = await authVerdict(writeRes); if (denied) return denied; + if (!writeRes.ok) { + return sizedResponse(await writeRes.text(), writeRes.status, TEXT_CT); + } } return new Response(JSON.stringify({ command: commands }), { status: 202, @@ -927,40 +930,41 @@ export function registerRoomRoutes(app: Hono): void { ); const denied = await authVerdict(writeRes); if (denied) return denied; - - if (writeRes.ok) { - // Phase 9 — settimetrigger side-effect. The legacy flow posted the - // command to a worker-thread which then emitted a `setcrontrigger` - // message (src/sc.ls:220); we short-circuit by detecting the verb - // here and writing to D1 once the DO has accepted the command. The - // DO also records a log entry; the actual scheduling lives in - // `cron_triggers`. - // - // Multi-line dispatches (array command or newline-joined text - // block) can carry several settimetrigger lines; we parse each. - if (c.env.DB) { - for (const line of cmdstr.split('\n')) { - const parsed = parseSettimetrigger(line); - if (parsed) { - await upsertCronTriggers(c.env.DB, room, parsed.cell, parsed.times); - } + if (!writeRes.ok) { + return sizedResponse(await writeRes.text(), writeRes.status, TEXT_CT); + } + + // Phase 9 — settimetrigger side-effect. The legacy flow posted the + // command to a worker-thread which then emitted a `setcrontrigger` + // message (src/sc.ls:220); we short-circuit by detecting the verb + // here and writing to D1 once the DO has accepted the command. The + // DO also records a log entry; the actual scheduling lives in + // `cron_triggers`. + // + // Multi-line dispatches (array command or newline-joined text + // block) can carry several settimetrigger lines; we parse each. + if (c.env.DB) { + for (const line of cmdstr.split('\n')) { + const parsed = parseSettimetrigger(line); + if (parsed) { + await upsertCronTriggers(c.env.DB, room, parsed.cell, parsed.times); } } - // Deferred multi-cascade sibling rename (H-4 namespace-guarded - // above). Rename failures are swallowed, matching legacy. - if (cascadeTarget !== null) { - await doFetch( - c.env, - cascadeTarget, - '/_do/rename', - { - method: 'POST', - body: JSON.stringify({ to: `${cascadeTarget}.bak` }), - headers: { 'Content-Type': 'application/json' }, - }, - principal, - ); - } + } + // Deferred multi-cascade sibling rename (H-4 namespace-guarded + // above). Rename failures are swallowed, matching legacy. + if (cascadeTarget !== null) { + await doFetch( + c.env, + cascadeTarget, + '/_do/rename', + { + method: 'POST', + body: JSON.stringify({ to: `${cascadeTarget}.bak` }), + headers: { 'Content-Type': 'application/json' }, + }, + principal, + ); } // Legacy replies `@response.json 202 {command}` -- `command` is the diff --git a/packages/worker/test/auth-session.node.test.ts b/packages/worker/test/auth-session.node.test.ts index c62356f2..df0cef53 100644 --- a/packages/worker/test/auth-session.node.test.ts +++ b/packages/worker/test/auth-session.node.test.ts @@ -13,7 +13,7 @@ interface AuthCall { function makeEnv( responder: (request: Request) => Response | Promise, - enabled = '1', + enabled: string | null | undefined = '1', ): { env: Env; calls: AuthCall[] } { const calls: AuthCall[] = []; let name = ''; @@ -71,9 +71,12 @@ describe('verifyAuthSession', () => { ]); }); - it.each(['', '0', 'false', 'no', 'off'])( + it.each(['', '0', 'false', 'no', 'off', null] as const)( 'does not contact AuthDO when auth flag is %j', async (flag) => { + // `null` is the workerd-unset binding shape (see room-index-access.ts). + // `undefined` is covered by flagEnabled unit tests; makeEnv's default + // param would collapse an omitted/undefined arg back to '1'. const { env, calls } = makeEnv( () => Response.json({ uid: 'unexpected' }), flag, diff --git a/packages/worker/test/room.test.ts b/packages/worker/test/room.test.ts index 05df1c98..b18b505f 100644 --- a/packages/worker/test/room.test.ts +++ b/packages/worker/test/room.test.ts @@ -1,5 +1,6 @@ import { env, runInDurableObject } from 'cloudflare:test'; import { logKey } from '@ethercalc/shared/storage-keys'; +import { createSpreadsheet } from '@ethercalc/socialcalc-headless'; import { describe, expect, it } from 'vite-plus/test'; import type { Env } from '../src/env.ts'; import { RoomDO } from '../src/room.ts'; @@ -253,4 +254,143 @@ describe('RoomDO (integration via DO namespace)', () => { const b2 = await stub.fetch('https://do/_do/cells/B2'); expect(((await b2.json()) as { datavalue?: number } | null)?.datavalue).toBe(99); }); + + it('legacy room without meta:access or meta:acl storage keys remains fully accessible to anonymous users', async () => { + const { stub } = getStub('legacy-unprotected-room'); + + // Seed the room snapshot (empty body) so it exists + const seedPut = await stub.fetch('https://do/_do/snapshot', { + method: 'PUT', + body: '', + }); + expect(seedPut.status).toBe(201); + + // Verify that DO storage has neither meta:access nor meta:acl keys + await runInDurableObject(stub, async (_instance: RoomDO, state) => { + const storedAccess = await state.storage.get('meta:access'); + const storedAcl = await state.storage.get('meta:acl'); + expect(storedAccess).toBeUndefined(); + expect(storedAcl).toBeUndefined(); + }); + + // (a) Anonymous HTTP GET succeeds (returns 200) + const getRes = await stub.fetch('https://do/_do/snapshot'); + expect(getRes.status).toBe(200); + + // (b) Anonymous WebSocket connect + a cell edit succeeds + const upgradeRes = await stub.fetch( + 'https://do/_do/ws?user=anon&auth=anon&room=legacy-unprotected-room', + { headers: { Upgrade: 'websocket' } }, + ); + expect(upgradeRes.status).toBe(101); + const client = upgradeRes.webSocket!; + client.accept(); + client.send( + JSON.stringify({ + type: 'execute', + room: 'legacy-unprotected-room', + user: 'anon', + auth: 'anon', + cmdstr: 'set A1 value n 999', + }), + ); + + // Query back until the edit is reflected in the DO state + let a1Value: number | undefined; + for (let i = 0; i < 20; i++) { + const cellRes = await stub.fetch('https://do/_do/cells/A1'); + const json = (await cellRes.json()) as { datavalue?: number } | null; + if (json?.datavalue === 999) { + a1Value = 999; + break; + } + const { promise, resolve } = Promise.withResolvers(); + queueMicrotask(resolve); + await promise; + } + client.close(); + expect(a1Value).toBe(999); + + // (c) Legacy ?auth= query path: auth=0 is view-only (rejects edits), unconfigured auth works + const viewOnlyRes = await stub.fetch( + 'https://do/_do/ws?user=anon&auth=0&room=legacy-unprotected-room', + { headers: { Upgrade: 'websocket' } }, + ); + expect(viewOnlyRes.status).toBe(101); + const viewClient = viewOnlyRes.webSocket!; + viewClient.accept(); + viewClient.send( + JSON.stringify({ + type: 'execute', + room: 'legacy-unprotected-room', + user: 'anon', + auth: '0', + cmdstr: 'set A1 value n 111', + }), + ); + + // Send a follow-up ask.log to ensure the prior message was processed by the DO + const { promise: logPromise, resolve: logResolve } = Promise.withResolvers(); + viewClient.addEventListener( + 'message', + (event) => { + if (typeof event.data === 'string') logResolve(event.data); + }, + { once: true }, + ); + viewClient.send( + JSON.stringify({ + type: 'ask.log', + room: 'legacy-unprotected-room', + user: 'anon', + }), + ); + await logPromise; + viewClient.close(); + + // A1 value remains 999 because auth=0 edit was rejected + const cellResAfterViewOnly = await stub.fetch('https://do/_do/cells/A1'); + expect(((await cellResAfterViewOnly.json()) as { datavalue?: number } | null)?.datavalue).toBe(999); + }); + + it('legacy oversized sheet (>200k cells) loads, allows non-growing edits, and rejects area-expanding edits', async () => { + const { stub } = getStub('oversized-legacy-room'); + + // Build a valid SocialCalc save string with cell SF500 set (column SF = 500, row = 500 -> 250,000 cells > 200,000) + const ss = createSpreadsheet(); + ss.executeCommand('set SF500 value n 10'); + const snapshot = ss.createSpreadsheetSave(); + + // Seed an oversized legacy sheet via migration seed + const seedRes = await stub.fetch('https://do/_do/seed?name=oversized-legacy-room', { + method: 'POST', + body: JSON.stringify({ + snapshot, + }), + }); + expect(seedRes.status).toBe(201); + + // 1. Verify DO loads and serves the oversized room without error + const cellRes = await stub.fetch('https://do/_do/cells/SF500'); + expect(cellRes.status).toBe(200); + expect(((await cellRes.json()) as { datavalue?: number } | null)?.datavalue).toBe(10); + + // 2. Non-growing edit within existing 500x500 dimensions (e.g. set SF500 value n 42) SUCCEEDS + const editRes = await stub.fetch('https://do/_do/commands', { + method: 'POST', + body: 'set SF500 value n 42', + }); + expect(editRes.status).toBe(202); + + const updatedCellRes = await stub.fetch('https://do/_do/cells/SF500'); + expect(((await updatedCellRes.json()) as { datavalue?: number } | null)?.datavalue).toBe(42); + + // 3. Area-expanding edit beyond existing 500x500 dimensions (e.g. set SF501 value n 1) is REJECTED + const expandRes = await stub.fetch('https://do/_do/commands', { + method: 'POST', + body: 'set SF501 value n 1', + }); + expect(expandRes.status).toBe(413); + expect(await expandRes.text()).toBe('command exceeds sheet limits'); + }); }); diff --git a/packages/worker/test/routes-auth.node.test.ts b/packages/worker/test/routes-auth.node.test.ts index 147e238a..d2035cfa 100644 --- a/packages/worker/test/routes-auth.node.test.ts +++ b/packages/worker/test/routes-auth.node.test.ts @@ -165,6 +165,11 @@ describe('auth routes', () => { ...makeAuthEnv(() => new Response('x')).env, ETHERCALC_AUTH: '0', } as Env, + // workerd delivers unset fromEnvironment bindings as null. + { + ...makeAuthEnv(() => new Response('x')).env, + ETHERCALC_AUTH: null, + } as Env, { ...makeAuthEnv(() => new Response('x')).env, ETHERCALC_RP_ID: null, @@ -224,6 +229,28 @@ describe('auth routes', () => { expect(disabled.headers.get('Cache-Control')).toBe('private, no-store'); }); + it('whoami reports enabled:false when ETHERCALC_AUTH is "0" or null', async () => { + // Kill-switch proof: even with AUTH + RP anchors + a session cookie that + // would verify under ETHERCALC_AUTH=1, the flag alone forces + // {uid:null, enabled:false}. Covers both the explicit wrangler [vars] + // value and workerd's null-for-unset binding delivery. + const base = makeAuthEnv(() => + Response.json({ uid: 'uid-owner', exp: Number.MAX_SAFE_INTEGER }), + ).env; + const app = buildApp(); + for (const flag of ['0', null] as const) { + const res = await app.fetch( + new Request('https://t.test/_auth/whoami', { + headers: { Cookie: '__Host-ec_sess=signed.token' }, + }), + { ...base, ETHERCALC_AUTH: flag } as Env, + ); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ uid: null, enabled: false }); + expect(res.headers.get('Cache-Control')).toBe('private, no-store'); + } + }); + it('revokes the current session before clearing its cookie', async () => { const { env, calls } = makeAuthEnv(() => new Response(null, { status: 204 })); const app = buildApp(); diff --git a/packages/worker/test/routes-rooms.node.test.ts b/packages/worker/test/routes-rooms.node.test.ts index e8029134..dcb9f72a 100644 --- a/packages/worker/test/routes-rooms.node.test.ts +++ b/packages/worker/test/routes-rooms.node.test.ts @@ -1,4 +1,5 @@ -import { describe, it, expect } from 'vite-plus/test'; +import { STORAGE_KEYS } from '@ethercalc/shared/storage-keys'; +import { describe, expect, it } from 'vite-plus/test'; /** * Node-env tests for the route glue at src/routes/rooms.ts. @@ -14,6 +15,7 @@ import { describe, it, expect } from 'vite-plus/test'; import type { Env } from '../src/env.ts'; import { buildApp } from '../src/index.ts'; +import { RoomDO } from '../src/room.ts'; interface Call { url: string; @@ -92,6 +94,145 @@ function withAuth(env: Env, uid: string = AUTH_UID): Env { }; } +/** + * ROOM namespace whose stub is a real RoomDO with private storage already + * seeded (`meta:access='private'` + ACL + snapshot). Used to prove the + * ETHERCALC_AUTH kill switch never declassifies an existing private room. + */ +function makePrivateRoomDoNamespace(snapshot = 'PRIVATE-SECRET'): { + env: Env; + calls: Call[]; + record: { map: Map }; +} { + const calls: Call[] = []; + const record = { map: new Map() }; + record.map.set(STORAGE_KEYS.metaAccess, 'private'); + record.map.set(STORAGE_KEYS.metaAcl, { + owner: AUTH_UID, + readers: [] as string[], + writers: [] as string[], + }); + record.map.set(STORAGE_KEYS.snapshot, snapshot); + + // Minimal Map-backed DurableObjectStorage (same shape room.node.test uses). + const storage = { + async get(key: unknown): Promise { + if (typeof key === 'string') return record.map.get(key); + if (Array.isArray(key)) { + const out = new Map(); + for (const k of key) { + if (typeof k === 'string' && record.map.has(k)) { + out.set(k, record.map.get(k)); + } + } + return out; + } + throw new Error('unexpected get argument shape'); + }, + async put(key: unknown, value?: unknown): Promise { + if (typeof key === 'string') { + record.map.set(key, value); + return; + } + if (key !== null && typeof key === 'object') { + for (const [k, v] of Object.entries(key)) record.map.set(k, v); + return; + } + throw new Error('unexpected put argument shape'); + }, + async delete(key: unknown): Promise { + if (typeof key === 'string') return record.map.delete(key); + if (Array.isArray(key)) { + let n = 0; + for (const k of key) { + if (typeof k === 'string' && record.map.delete(k)) n += 1; + } + return n; + } + throw new Error('unexpected delete argument shape'); + }, + async deleteAll(): Promise { + record.map.clear(); + }, + async list(opts?: { prefix?: string }): Promise> { + const out = new Map(); + const prefix = opts?.prefix ?? ''; + for (const k of Array.from(record.map.keys()) + .filter((key) => key.startsWith(prefix)) + .sort()) { + out.set(k, record.map.get(k)!); + } + return out; + }, + async getAlarm(): Promise { + return null; + }, + async setAlarm(): Promise {}, + async deleteAlarm(): Promise {}, + async transaction( + cb: (txn: DurableObjectTransaction) => Promise, + ): Promise { + return cb(storage as unknown as DurableObjectTransaction); + }, + } as unknown as DurableObjectStorage; + + const state = { + id: { toString: () => 'existing-private' } as DurableObjectId, + storage, + async blockConcurrencyWhile(cb: () => Promise): Promise { + return cb(); + }, + getWebSockets(): WebSocket[] { + return []; + }, + waitUntil(): void {}, + } as unknown as DurableObjectState; + + const roomEnv: Env = { ROOM: {} as DurableObjectNamespace }; + const room = new RoomDO(state, roomEnv); + + const stub: FakeStub = { + async fetch(input, init) { + const url = typeof input === 'string' ? input : input.url; + const method = + init?.method ?? (typeof input === 'string' ? 'GET' : input.method); + const headers = new Headers( + init?.headers ?? + (typeof input === 'string' ? undefined : input.headers), + ); + let bodyText: string | undefined; + if (init?.body !== undefined) { + bodyText = + typeof init.body === 'string' + ? init.body + : await new Response(init.body as BodyInit).text(); + } + calls.push({ + url, + method, + uid: headers.get('X-EC-Uid'), + ...(bodyText !== undefined ? { bodyText } : {}), + }); + // Forward the exact Worker→DO request into a real RoomDO instance + // whose storage already holds meta:access=private + meta:acl. + const forwarded = new Request(url, { + method, + headers, + ...(bodyText !== undefined ? { body: bodyText } : {}), + }); + return room.fetch(forwarded); + }, + }; + + const env: Env = { + ROOM: { + idFromName: (n: string) => ({ n }) as unknown as DurableObjectId, + get: () => stub as unknown as DurableObjectStub, + } as unknown as DurableObjectNamespace, + }; + return { env, calls, record }; +} + describe('route glue — env.ROOM dispatch shapes', () => { it('redirects only the configured origin www alias', async () => { const app = buildApp(); @@ -666,6 +807,42 @@ describe('route glue — DO auth verdict propagation (Phase A)', () => { expect(await create.text()).toBe('snapshot exceeds sheet limits'); }); + it('POST /_/:room command mutations propagate a DO 413 sheet-limit verdict', async () => { + const { env } = makeFakeRoomNamespace( + () => new Response('command exceeds sheet limits', { status: 413 }), + ); + const app = buildApp(); + const res = await app.fetch( + new Request('https://t.test/_/oversized', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ command: 'set Z10000 value n 1' }), + }), + env as never, + ); + expect(res.status).toBe(413); + expect(res.headers.get('content-type')).toBe('text/plain; charset=utf-8'); + expect(await res.text()).toBe('command exceeds sheet limits'); + }); + + it('POST /_/:room returns 202 command echo on successful DO dispatch', async () => { + const { env } = makeFakeRoomNamespace( + () => new Response('OK', { status: 200 }), + ); + const app = buildApp(); + const res = await app.fetch( + new Request('https://t.test/_/happy', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ command: 'set A1 text t hello' }), + }), + env as never, + ); + expect(res.status).toBe(202); + expect(res.headers.get('content-type')).toBe('application/json; charset=utf-8'); + expect(await res.json()).toEqual({ command: 'set A1 text t hello' }); + }); + it('POST /_ with an explicit room propagates a DO 403 (occupied private id)', async () => { const { env } = makeFakeRoomNamespace( () => new Response('Forbidden', { status: 403 }), @@ -755,6 +932,80 @@ describe('private create/copy routes (P7)', () => { expect(calls).toHaveLength(0); }); + it('POST /_/private rejects when ETHERCALC_AUTH is off even with a session cookie', async () => { + // Kill-switch proof: withAuth wires a verifying AUTH binding and a cookie, + // but ETHERCALC_AUTH "0"/null makes getSessionPrincipal null → 401 and no + // DO dispatch. Covers the wrangler [vars] kill switch and workerd null. + const { env, calls } = makeFakeRoomNamespace( + () => new Response('', { status: 201 }), + ); + const app = buildApp(); + for (const flag of ['0', null] as const) { + calls.length = 0; + const res = await app.fetch( + new Request('https://t.test/_/private', { + method: 'POST', + headers: { Cookie: AUTH_COOKIE }, + }), + { ...withAuth(env), ETHERCALC_AUTH: flag } as never, + ); + expect(res.status, `flag=${String(flag)}`).toBe(401); + expect(res.headers.get('content-type')).toBe('text/plain; charset=utf-8'); + expect(await res.text()).toBe('Unauthorized'); + expect(calls).toHaveLength(0); + } + }); + + it('keeps an existing private room locked when ETHERCALC_AUTH is off', async () => { + // Kill-switch lockout against a REAL RoomDO whose storage already has + // meta:access='private' and meta:acl — not a manufactured 403 responder. + // Flag off → getSessionPrincipal null → doFetch stamps no X-EC-Uid → + // RoomDO pre-dispatch gate denies → route propagates 403 Forbidden. + // Positive control: same cookie+AUTH with flag on yields the snapshot. + const { env, calls, record } = makePrivateRoomDoNamespace('PRIVATE-SECRET'); + expect(record.map.get(STORAGE_KEYS.metaAccess)).toBe('private'); + expect(record.map.get(STORAGE_KEYS.metaAcl)).toEqual({ + owner: AUTH_UID, + readers: [], + writers: [], + }); + + const app = buildApp(); + + const unlocked = await app.fetch( + new Request('https://t.test/_/existing-private', { + headers: { Cookie: AUTH_COOKIE }, + }), + withAuth(env) as never, + ); + expect(unlocked.status).toBe(200); + expect(await unlocked.text()).toBe('PRIVATE-SECRET'); + expect(calls[0]!.uid).toBe(AUTH_UID); + + for (const flag of ['0', null] as const) { + calls.length = 0; + const locked = await app.fetch( + new Request('https://t.test/_/existing-private', { + headers: { Cookie: AUTH_COOKIE }, + }), + { ...withAuth(env), ETHERCALC_AUTH: flag } as never, + ); + expect(locked.status, `flag=${String(flag)}`).toBe(403); + expect(locked.headers.get('content-type')).toBe( + 'text/plain; charset=utf-8', + ); + expect(await locked.text()).toBe('Forbidden'); + expect(calls).toHaveLength(1); + expect(calls[0]!.uid).toBeNull(); + expect(calls[0]!.url).toBe( + 'https://do.local/_do/snapshot?name=existing-private', + ); + // Storage is unchanged — kill switch is a lockout, not a wipe/declassify. + expect(record.map.get(STORAGE_KEYS.metaAccess)).toBe('private'); + expect(record.map.get(STORAGE_KEYS.snapshot)).toBe('PRIVATE-SECRET'); + } + }); + it('POST /_/private creates an owned private room for a verified principal', async () => { const { env, calls } = makeFakeRoomNamespace( () => new Response('', { status: 201 }), From 75433179600854e90aafabb075e8669af3669edf Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 14:57:10 +0800 Subject: [PATCH 02/95] fix(scripts): compute mutation score from mutant statuses in ratchet-verify The read_score() function in scripts/ratchet-verify.sh previously attempted to read a mutationScore property from Stryker's JSON report schema using jq and node. However, Stryker JSON reports do not contain a top-level mutationScore field; mutant results are recorded under .files[].mutants[] with status strings. As a result, read_score() returned empty, causing ratchet-verify.sh to fail with exit code 2 ('parse error'). Update read_score() in both jq and node paths to calculate the score from individual mutant statuses as (Killed + Timeout) / (Killed + Timeout + Survived + NoCoverage) * 100. --- scripts/ratchet-verify.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/ratchet-verify.sh b/scripts/ratchet-verify.sh index 8fe3d73e..471ea1c9 100755 --- a/scripts/ratchet-verify.sh +++ b/scripts/ratchet-verify.sh @@ -50,9 +50,27 @@ header() { read_score() { local json="$1" if command -v jq >/dev/null 2>&1; then - jq -r '.. | .mutationScore? // empty' "$json" | head -n1 + jq -r ' + [.files[]?.mutants[]?] as $m | + ($m | map(select(.status == "Killed" or .status == "Timeout")) | length) as $d | + ($m | map(select(.status == "Survived" or .status == "NoCoverage")) | length) as $u | + if ($d + $u) == 0 then empty else ($d / ($d + $u) * 100) end + ' "$json" else - node -e "const d=require('$json'); const s=d.mutationScore ?? (d.summary && d.summary.mutationScore); process.stdout.write(String(s ?? ''));" + node -e ' + const d = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); + let det = 0, undet = 0; + for (const f of Object.values(d.files || {})) { + for (const m of f.mutants || []) { + if (m.status === "Killed" || m.status === "Timeout") det++; + else if (m.status === "Survived" || m.status === "NoCoverage") undet++; + } + } + const tot = det + undet; + if (tot > 0) { + process.stdout.write(String((det / tot) * 100)); + } + ' "$json" fi } From b16cfd88a3051dda2f21921b227bccd58ae67305 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 14:57:13 +0800 Subject: [PATCH 03/95] docs(migration): add production upgrade runbook and supporting audits Add production upgrade runbook and supporting audit documentation in docs/migration/ to guide the EtherCalc modern stack deployment. Includes the production upgrade plan, delta audit between legacy and modern worker implementations, WebSocket skew and reconnect handling strategy, preflight validation results, and complete component inventory. Also update AGENTS.md key docs index. --- AGENTS.md | 1 + docs/migration/DELTA_AUDIT.md | 589 ++++++ docs/migration/INVENTORY.md | 2501 ++++++++++++++++++++++++++ docs/migration/PREFLIGHT_RESULTS.md | 344 ++++ docs/migration/PROD_UPGRADE_PLAN.md | 903 ++++++++++ docs/migration/SKEW_AND_RECONNECT.md | 560 ++++++ 6 files changed, 4898 insertions(+) create mode 100644 docs/migration/DELTA_AUDIT.md create mode 100644 docs/migration/INVENTORY.md create mode 100644 docs/migration/PREFLIGHT_RESULTS.md create mode 100644 docs/migration/PROD_UPGRADE_PLAN.md create mode 100644 docs/migration/SKEW_AND_RECONNECT.md diff --git a/AGENTS.md b/AGENTS.md index 75d5226f..d98309cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,7 @@ line/branch/function/statement coverage on gated packages in CI. | User guide + FAQ | [docs.ethercalc.net](https://docs.ethercalc.net) · `packages/docs/` | | HTTP API | `API.md` | | Self-host hardening | `docs/SELFHOST_HARDENING.md` | +| Prod upgrade runbook | `docs/migration/PROD_UPGRADE_PLAN.md` | | Oracle replay | `tests/oracle/README.md` · `packages/oracle-harness/` | | Mutation baselines | `docs/MUTATION_REPORT.md` | | Sandstorm `.spk` | `SANDSTORM.md` (manual `spk pack` — app owner signs) | diff --git a/docs/migration/DELTA_AUDIT.md b/docs/migration/DELTA_AUDIT.md new file mode 100644 index 00000000..d0daa7a7 --- /dev/null +++ b/docs/migration/DELTA_AUDIT.md @@ -0,0 +1,589 @@ +# Delta Audit Report: Tagged Baseline (`0.20260717.0`) vs. `main` (`d486f33`) + +> **Scope:** Comprehensive evidence-backed delta audit covering state, compatibility, and deployment changes between the latest tagged release baseline (`0.20260717.0` / commit `149ebcf`) and current `main` (`d486f33`). +> **Audit Date:** 2026-08-10 + +--- + +## Executive Summary & Delta Inventory + +| Item | Classification | Blast Radius | Mitigation | +| :--- | :--- | :--- | :--- | +| **a) DO Classes & Migrations** | `[NEEDS MIGRATION]` / `[FORWARD-COMPATIBLE]` | High (Worker deploy requires `v2` migration for new `AuthDO` class). Existing `RoomDO` instances unaffected. | Execute `wrangler deploy` to register migration `tag = "v2"`. Rollback leaves `AuthDO` unreferenced. | +| **b) DO Storage Key Schema** | `[FORWARD-COMPATIBLE]` (Public) / `[BREAKING-ON-ROLLBACK]` (Private) | Critical on Rollback (`meta:access` & `meta:acl` added for private sheets; legacy code ignores them, making private sheets public on rollback). | Backup DO storage / snapshot before cutover. If rolling back, block access to private rooms via edge WAF. | +| **c) D1 Schema & Migrations** | `[FORWARD-COMPATIBLE]` | Low (No new D1 migrations added since `0.20260717.0`; all tables `CREATE TABLE IF NOT EXISTS`). | None required. | +| **d) KV & R2 Layout** | `[FORWARD-COMPATIBLE]` | None (KV/R2 bindings currently unused in Worker codebase). | None required. | +| **e) Environment Variables & Secrets** | `[NEEDS MIGRATION]` / `[FORWARD-COMPATIBLE]` | High (Passkeys fail closed if `ETHERCALC_AUTH`, `ETHERCALC_RP_ID`, `ETHERCALC_ORIGIN` are unset). `ETHERCALC_RP_NAME` defaults to `'EtherCalc'`. | Populate WebAuthn trust anchors in production environment before cutover. | +| **f) Compatibility Date & Capnp** | `[FORWARD-COMPATIBLE]` (Worker) / `[NEEDS MIGRATION]` (Self-host) | Medium (wrangler date updated to `2026-07-21`; self-host `config.capnp` uses `2026-07-14` in lockstep with `workerd`). | Ensure standalone `workerd` binary is updated to match capnp `2026-07-14` date. | +| **g) Wire Protocol & Version Skew** | `[FORWARD-COMPATIBLE]` | Medium (Strict resource caps added: 1 MiB frame, 16 KiB chat, 120 KiB command). Standard client frames pass. | Standard browser clients pass cleanly. Stale tabs sending oversized payloads are dropped. | +| **h) Static Assets & CSP** | `[UNKNOWN]` | Low to Medium (Root inline scripts moved to 5 `static/*.js` files. CSP `connect-src` uses `ETHERCALC_ORIGIN`. Behavior under asset/CSP skew is unproven by diff/test). | Run `scripts/build-assets.ts` prior to deploy (enforced in `deploy-production.yml`). | +| **i) Auth & Authz (`AuthDO`, ACL, `?auth=`)** | `[FORWARD-COMPATIBLE]` (Public) / `[NEEDS MIGRATION]` (Passkeys) | High (New WebAuthn passkey system + `__Host-ec_sess` cookie). Existing public sheets and `?auth=` links remain fully functional. | Configure passkey secrets/vars. No disruption to existing public sheets. | +| **j) SocialCalc Upgrade & Serialisation** | `[UNKNOWN]` | Low to Medium (Upgraded to SocialCalc 3.1.0; `version:1.5` format shared, but full parsing compatibility for pre-3.1.0 snapshots is unproven by unit test). | Run canary checks before cutover; oracle replay covers 10/13 scenarios. | +| **k) Stateful Alarms, D1 Mirror & Scheduler** | `[FORWARD-COMPATIBLE]` | Medium (DO alarm re-arm gated on active connections/TTL; D1 audit/chat mirror idempotent; scheduler retry bug fixed). | None required. | + +--- + +## Step 1: Baseline Determination & Evidence + +### 1.1 Established Baseline +* **Tag / Revision Baseline:** Tag `0.20260717.0` (commit `149ebcf16104b01254ca2b796beb701c88bd6ff8`, tagged Fri Jul 17 21:18:06 2026 +0800). +* **Target Revision for Audit:** `main` HEAD (`d486f33`). + +### 1.2 Evidence Grounding +1. **Git Tag Baseline:** `0.20260717.0` is the latest git release tag in the repository. + ```text + tag 0.20260717.0 + Tagger: Audrey Tang + Date: Fri Jul 17 21:18:06 2026 +0800 + commit 149ebcf16104b01254ca2b796beb701c88bd6ff8 + ``` +2. **Package Version:** Root `package.json` specifies `"version": "0.20260717.0"`. +3. **Changelog:** `Changes.txt` records `0.20260717.0 — 2026-07-17` as the last formal release header. +4. **Deployment Pipeline:** `.github/workflows/deploy-production.yml` uses `on: workflow_dispatch:` (manual trigger for `wrangler deploy`). No automated continuous deployment triggers on commit or tag. + +### 1.3 Confidence & Caveats +* **Confidence Level:** High for repo baseline; **UNKNOWN** for live production SHA. +* **Caveat / External Checks:** Because production deployments are dispatched manually via GitHub Actions or CLI `wrangler deploy`, the exact running revision on `ethercalc.net` cannot be determined from repository files alone. +* **Required External Checks to Settle Live Revision:** + 1. `wrangler deployments list --env=""` via Cloudflare API. + 2. Querying `https://ethercalc.net/_health` (currently returns `{"status":"ok","version":"0.0.0"}` because `version` is hardcoded `'0.0.0'` in `packages/worker/src/handlers/health.ts:15`, so HTTP probes cannot settle the SHA). + +--- + +## Step 2: Detailed Delta Enumeration + +### a) Durable Object Classes & Migrations +* **Tag Evidence (`149ebcf`):** `git show 149ebcf:packages/worker/wrangler.toml` + ```toml + [[durable_objects.bindings]] + name = "ROOM" + class_name = "RoomDO" + + [[migrations]] + tag = "v1" + new_sqlite_classes = ["RoomDO"] + ``` +* **Main Evidence (`d486f33`):** `packages/worker/wrangler.toml:31-48` + ```toml + [[durable_objects.bindings]] + name = "ROOM" + class_name = "RoomDO" + + [[durable_objects.bindings]] + name = "AUTH" + class_name = "AuthDO" + + [[migrations]] + tag = "v1" + new_sqlite_classes = ["RoomDO"] + + [[migrations]] + tag = "v2" + new_sqlite_classes = ["AuthDO"] + ``` +* **Analysis:** A new Durable Object class `AuthDO` (`AUTH` binding) was added along with migration `tag = "v2"` specifying `new_sqlite_classes = ["AuthDO"]`. Both `RoomDO` and `AuthDO` use SQLite-backed DO storage (`new_sqlite_classes`). No classes were renamed or deleted. +* **Classification:** `[NEEDS MIGRATION]` (wrangler deploy executes `v2` migration) / `[FORWARD-COMPATIBLE]` (existing `RoomDO` instances are untouched). + +--- + +### b) DO Storage Key Schema +* **Tag Evidence (`149ebcf`):** `git show 149ebcf:packages/shared/src/storage-keys.ts` + Did NOT contain `metaAccess`, `metaAcl`, or `metaGroup`. +* **Main Evidence (`d486f33`):** `packages/shared/src/storage-keys.ts:37-51` + ```ts + metaAccess: 'meta:access', + metaAcl: 'meta:acl', + metaGroup: 'meta:group', + ``` + `packages/worker/src/lib/authorize.ts:20` + ```ts + if (access == null || access === 'public') return true; + ``` + `packages/worker/src/auth-do.ts:297,325,475,589` + ```ts + // AuthDO storage keys: 'session-secret', 'challenge:', 'cred:', 'revocation:' + ``` +* **Analysis:** + * **Public/Existing Rooms:** `meta:access` and `meta:acl` are absent (`undefined`) in DO storage for existing rooms. `authorize()` returns `true` when `access == null`, maintaining complete backward compatibility. + * **Private Rooms:** Created via `POST /_/private` or `POST /_do/init-private`, setting `meta:access = 'private'` and `meta:acl`. + * **Rollback Behavior:** Older worker code (`149ebcf`) does NOT check `meta:access` or `meta:acl`. If rolled back, an old worker waking on a private room will serve `snapshot` and process commands without checking access control (exposing private sheets publicly). +* **Classification:** `[FORWARD-COMPATIBLE]` (for public rooms) / `[BREAKING-ON-ROLLBACK]` (for private rooms created post-upgrade). + +--- + +### c) D1 Schema & Migrations +* **Evidence:** `packages/worker/migrations/` + * `0001_rooms.sql`: `CREATE TABLE rooms (room TEXT PRIMARY KEY, updated_at INTEGER NOT NULL, cors_public INTEGER NOT NULL DEFAULT 0);` + * `0002_cron.sql`: `CREATE TABLE cron_triggers (room TEXT NOT NULL, cell TEXT NOT NULL, fire_at INTEGER NOT NULL, PRIMARY KEY (room, cell, fire_at));` + * `0003_audit_chat.sql`: `CREATE TABLE audit_log (...); CREATE TABLE chat_log (...);` + * `packages/worker/src/lib/d1-schema.ts:15-18`: `CREATE TABLE IF NOT EXISTS` lazy self-healing. +* **Analysis:** No D1 migration files were added between `149ebcf` and `d486f33`. All migrations are expand-only (`CREATE TABLE`). +* **Classification:** `[FORWARD-COMPATIBLE]`. + +--- + +### d) KV & R2 Storage +* **Evidence:** `packages/worker/src/env.ts:13-183`, `packages/worker/wrangler.toml:149-152` +* **Analysis:** KV namespaces and R2 buckets are scaffolding placeholders in `wrangler.toml` and are not bound or read in `packages/worker/src`. +* **Classification:** `[FORWARD-COMPATIBLE]`. + +--- + +### e) Environment Variables & Fail-Closed Behavior +* **Main Evidence (`d486f33`):** `packages/worker/src/env.ts:7-9, 20-30`, `packages/worker/src/lib/room-index-access.ts:21-45`, `packages/worker/src/routes/auth.ts:34-43`, `packages/worker/src/auth-do.ts:244-253` + ```ts + // packages/worker/src/routes/auth.ts:34-43 + function authEnabled(env: Env): boolean { + return ( + flagEnabled(env.ETHERCALC_AUTH) && + env.AUTH !== undefined && + typeof env.ETHERCALC_RP_ID === 'string' && + env.ETHERCALC_RP_ID.length > 0 && + typeof env.ETHERCALC_ORIGIN === 'string' && + env.ETHERCALC_ORIGIN.length > 0 + ); + } + ``` + ```ts + // packages/worker/src/auth-do.ts:244-253 + const rpID = this.#env.ETHERCALC_RP_ID; + const origin = this.#env.ETHERCALC_ORIGIN; + if (!rpID || !origin) { + return plainResponse('Auth is not configured', 503); + } + const config: AuthConfig = { + rpID, + origin, + rpName: this.#env.ETHERCALC_RP_NAME || 'EtherCalc', + }; + ``` +* **Var-by-Var Breakdown:** + 1. `ETHERCALC_AUTH`: Default when unset = flag disabled (`flagEnabled(null) === false`). WebAuthn ceremonies and private rooms return 404/401 (fails closed). + 2. `ETHERCALC_RP_ID` & `ETHERCALC_ORIGIN`: WebAuthn trust anchors. If unset while `ETHERCALC_AUTH="1"`, `authEnabled` evaluates to `false` and `AuthDO` returns `503 Auth is not configured` (fails closed). Under workerd's null quirk, handled via `typeof ... === 'string' && length > 0`. `ETHERCALC_ORIGIN` also anchors CSP `connect-src`. + 3. `ETHERCALC_RP_NAME`: Human-readable relying party name. Default when unset or null = `'EtherCalc'` (`auth-do.ts:252`). Workerd's null quirk handled via `this.#env.ETHERCALC_RP_NAME || 'EtherCalc'`. + 4. `ETHERCALC_DISABLE_ROOM_INDEX`: Default when unset = falls back to `ETHERCALC_CORS` (which is `"1"` in `wrangler.toml`, so `/_rooms*` endpoints return 403). Workerd null quirk handled via `explicit != null && explicit.trim() !== ''`. + 5. `ETHERCALC_RATELIMIT`: Default when unset = `null` (rate limiting disabled; hosted relies on CF edge WAF). + 6. `ETHERCALC_KEY`: Default when unset = `undefined` (anonymous identity HMAC, `hmac(room) = room`). + 7. `EMAIL` (`send_email` binding): Default when unset = `undefined`. Handled gracefully by `buildEmailSender` returning `DisabledEmailSender` ("E-mail disabled: no send_email binding configured"). +* **Classification:** `[NEEDS MIGRATION]` (for auth environment configuration) / `[FORWARD-COMPATIBLE]`. + +--- + +### f) Compatibility Date & Capnp Lockstep +* **Tag Evidence (`149ebcf`):** `compatibility_date = "2024-11-12"` in `wrangler.toml`. +* **Main Evidence (`d486f33`):** `packages/worker/wrangler.toml:3` + ```toml + compatibility_date = "2026-07-21" + compatibility_flags = ["nodejs_compat"] + ``` + `packages/worker/workerd/config.capnp:58-62` + ```capnp + # Standalone workerd rejects dates newer than its baked-in release date. + # Keep this at or below the workerd version pinned in bun.lock; Wrangler + # may safely use a newer date because it clamps with a warning. + compatibilityDate = "2026-07-14", + compatibilityFlags = ["nodejs_compat"], + ``` +* **Analysis:** `compatibility_date` in `wrangler.toml` moved from `"2024-11-12"` to `"2026-07-21"`. `config.capnp` uses `"2026-07-14"`. The capnp lockstep constraint requires that the standalone `workerd` binary release date is >= `2026-07-14`. +* **Classification:** `[FORWARD-COMPATIBLE]` (Cloudflare Wrangler) / `[NEEDS MIGRATION]` (Self-host workerd binary). + +--- + +### g) Wire Protocol & Version Skew +* **Evidence:** `packages/shared/src/messages.ts:117-125, 255-260` + ```ts + export const MAX_WS_FRAME_CHARS = 1024 * 1024; // 1 MiB + export const MAX_WS_ROOM_CHARS = 2_048; + export const MAX_WS_USER_CHARS = 256; + export const MAX_WS_AUTH_CHARS = 512; + export const MAX_WS_CHAT_CHARS = 16 * 1024; // 16 KiB + export const MAX_WS_CELL_CHARS = 64; + export const MAX_COMMAND_UTF8_BYTES = 120 * 1024; // 120 KiB + ``` + `packages/worker/src/room.ts:1695-1698` (attachment room binding enforcement) +* **Version Skew Analysis:** + * **Old Client -> New Worker:** Standard client messages are well below protocol caps and pass structural checks. Oversized malformed messages are dropped (returns null). + * **New Client -> Old Worker:** Old worker parses standard JSON frames without strict bounds checks. + * **Legacy `/socket.io/*` Shim:** Kept intact and bound to room attachment. +* **Classification:** `[FORWARD-COMPATIBLE]`. + +--- + +### h) Static Assets & CSP +* **Evidence:** `scripts/build-assets.ts:33-39`, `packages/worker/src/lib/csp.ts:21-40`, `packages/worker/src/index.ts:91-95` + ```ts + "script-src 'self' 'unsafe-inline'", + "style-src 'self' 'unsafe-inline'", + `connect-src 'self' ${websocketAuthority(c.env.ETHERCALC_ORIGIN, requestUrl, secureTransport)}`, + ``` +* **Analysis:** HTML inline scripts were moved to five tracked static files (`static/index-bootstrap.js`, `static/index-110n.js`, `static/panels.js`, `static/start-bootstrap.js`, `static/start.js`). CSP `connect-src` uses `ETHERCALC_ORIGIN` for WebSocket authority. +* **Uncertainty Note:** Behavior under stale cached HTML + new CSP or vice versa is **UNKNOWN** (no automated integration test covers asset-vs-CSP version skew). +* **Classification:** `[UNKNOWN]`. + +--- + +### i) Auth/Authz (`AuthDO`, ACL, `?auth=`, `__Host-ec_sess`) +* **Evidence:** `packages/worker/src/routes/auth.ts`, `packages/worker/src/lib/auth-session.ts`, `packages/worker/src/routes/rooms.ts:634-637`, `packages/worker/src/lib/auth.ts:34,70` +* **Analysis:** + * **Existing Rooms:** Access mode defaults to `'public'` when `meta:access` is absent in DO storage. No existing public room becomes inaccessible. + * **Legacy `?auth=` Links:** Preserved verbatim for HMAC validation when `ETHERCALC_KEY` is set; `auth=0` continues to enforce view-only access. + * **Session Cookie:** Passkey login sets `__Host-ec_sess` cookie; Worker verifies session with `AuthDO` and threads `X-EC-Uid` internally (inbound `X-EC-Uid` headers from clients are stripped by `doFetch`). +* **Classification:** `[FORWARD-COMPATIBLE]` (Existing rooms/links) / `[NEEDS MIGRATION]` (Passkey activation). + +--- + +### j) SocialCalc Upgrade & Serialization +* **Evidence:** `packages/socialcalc-headless/src/socialcalc.bundled.ts:5, 1473` +* **Analysis:** Upgraded from SocialCalc 3.0.8 to 3.1.0 with build-time `createSocialCalcFactory()`. The underlying save-string format (`version:1.5`) remains shared. +* **Uncertainty Note:** Whether `createSocialCalcFactory()` parses every legacy pre-3.1.0 snapshot string without issue is **UNKNOWN** from unit test coverage alone (no unit test explicitly loads a raw 3.0.x snapshot into `createSocialCalcFactory()`, though `oracle-replay.test.ts` passes 10/13 recorded scenario replays). +* **Classification:** `[UNKNOWN]`. + +--- + +### k) Stateful Alarms, D1 Mirror, PITR & Scheduler +* **Evidence:** `packages/worker/src/room.ts:199, 2424-2480`, `packages/worker/src/handlers/cron.ts`, `packages/worker/src/scheduled.ts` +* **Analysis:** + * **Alarms (`setAlarm`):** RoomDO housekeeping alarm optimized to gate re-arming on active WebSocket connections or configured `ETHERCALC_EXPIRE` TTL. AuthDO uses alarms to trim expired challenges and session revocations. + * **D1 Mirroring:** `audit_log` and `chat_log` tables receive idempotent mirrors (`INSERT OR IGNORE`). + * **PITR Restore:** Endpoints (`/_do/pitr-restore`) retain an undo bookmark in DO storage. + * **Scheduler (`_timetrigger`):** Fixed bug where partial scheduler retries treated failures as success. +* **Classification:** `[FORWARD-COMPATIBLE]`. + +--- + +## Step 3: Irreversibility & Mitigation Strategies + +### 3.1 Irreversible State & Deployment Steps +1. **Private Room DO State (`meta:access` & `meta:acl`):** + * *Irreversibility:* Once a room is initialized as private (`POST /_/private` or `POST /_do/init-private`), keys `meta:access` and `meta:acl` are written to DO storage. + * *Rollback Risk:* Older worker code (`149ebcf`) does not have `meta:access` or `meta:acl` or `authorize()` logic (`git show 149ebcf:packages/worker/src/lib/authorize.ts` yields `fatal: path does not exist`). On rollback, old code serves the room's `snapshot` to ANY requester, exposing private room contents publicly. + * *Mitigation:* + - **Pre-cutover Snapshot:** Take a point-in-time snapshot of DO storage prior to cutover. + - **Edge Access Gate on Rollback:** If rollback is required, configure Cloudflare Edge WAF rules to block access to rooms created after the upgrade, or strip `meta:access` keys via an emergency script. +2. **Durable Object Class Migration `v2` (`AuthDO`):** + * *Irreversibility:* Running `wrangler deploy` applies migration `v2`, instantiating the SQLite-backed `AuthDO`. + * *Rollback Risk:* Rollback leaves `AuthDO` storage orphaned. WebAuthn accounts registered during the new deployment window will not be reachable on old code. + * *Mitigation:* Dual-write is not applicable for AuthDO since passkeys do not exist in old code. Communicate that passkey accounts created during a rollback window must be re-registered upon re-deployment. +3. **WebAuthn Relying Party Anchors:** + * *Irreversibility:* Passkey credentials bound to `ETHERCALC_RP_ID` ("ethercalc.net") cannot be used under a different RP ID. + * *Mitigation:* Ensure `ETHERCALC_RP_ID` and `ETHERCALC_ORIGIN` are pinned in `wrangler.toml` `[vars]` prior to initial deploy. + +--- + +## Summary of UNKNOWN Items & Verification Steps + +1. **Exact Live Production Revision of `ethercalc.net`:** + * *Reason:* Production deployments are manually triggered via GitHub Actions `workflow_dispatch` or CLI `wrangler deploy`. + * *Verification Check:* Execute `wrangler deployments list --env=""` or check deployment logs in Cloudflare Dashboard. +2. **Hardcoded Version Health Endpoint:** + * *Reason:* `GET /_health` returns hardcoded `version: "0.0.0"` (`packages/worker/src/handlers/health.ts:15`), so HTTP probes cannot confirm the live git SHA. +3. **Production `ETHERCALC_KEY` Secret Status:** + * *Reason:* `ETHERCALC_KEY` is marked as `unset by default` in `wrangler.toml` and injected via `wrangler secret put`. + * *Verification Check:* Run `wrangler secret list --env=""` to verify if an HMAC secret is set in production. +4. **Behavior under Static Asset & CSP Version Skew:** + * *Reason:* No automated test exercises stale cached HTML paired with new CSP or new HTML with stale CSP. +5. **Raw Snapshot Parsing Compatibility for Old SocialCalc 3.0.x Saves:** + * *Reason:* No unit test explicitly feeds an un-migrated raw pre-3.1.0 snapshot into `createSocialCalcFactory()` directly (though `oracle-replay.test.ts` passes 10/13 recorded scenario replays). + +--- + +## Legacy-room compatibility proof + +### 1. Existing Test Analysis +* **Access Gate Unit Tests:** `packages/worker/test/authorize.node.test.ts:42-46` asserts that when `access` mode is `null` / `undefined`, `authorize('read'|'write', principal, access, null)` returns `true`. +* **Integration Test Gap:** No existing integration test in `room.test.ts` or `routes-rooms.test.ts` explicitly verified a room whose storage has **NEITHER** `meta:access` **NOR** `meta:acl` set, asserting (a) anonymous HTTP GET succeeds, (b) anonymous WebSocket connect + cell edit succeeds, and (c) legacy `?auth=` query path behaves as expected (`auth=0` demoted to view-only 403 / edit rejection). + +### 2. Added Test Implementation +We added an integration test in `packages/worker/test/room.test.ts` (lines 257-353): +```ts + it('legacy room without meta:access or meta:acl storage keys remains fully accessible to anonymous users', async () => { + const { stub } = getStub('legacy-unprotected-room'); + + // Seed the room snapshot (empty body) so it exists + const seedPut = await stub.fetch('https://do/_do/snapshot', { + method: 'PUT', + body: '', + }); + expect(seedPut.status).toBe(201); + + // Verify that DO storage has neither meta:access nor meta:acl keys + await runInDurableObject(stub, async (_instance: RoomDO, state) => { + const storedAccess = await state.storage.get('meta:access'); + const storedAcl = await state.storage.get('meta:acl'); + expect(storedAccess).toBeUndefined(); + expect(storedAcl).toBeUndefined(); + }); + + // (a) Anonymous HTTP GET succeeds (returns 200) + const getRes = await stub.fetch('https://do/_do/snapshot'); + expect(getRes.status).toBe(200); + + // (b) Anonymous WebSocket connect + a cell edit succeeds + const upgradeRes = await stub.fetch( + 'https://do/_do/ws?user=anon&auth=anon&room=legacy-unprotected-room', + { headers: { Upgrade: 'websocket' } }, + ); + expect(upgradeRes.status).toBe(101); + const client = upgradeRes.webSocket!; + client.accept(); + client.send( + JSON.stringify({ + type: 'execute', + room: 'legacy-unprotected-room', + user: 'anon', + auth: 'anon', + cmdstr: 'set A1 value n 999', + }), + ); + + // Query back until the edit is reflected in the DO state + let a1Value: number | undefined; + for (let i = 0; i < 20; i++) { + const cellRes = await stub.fetch('https://do/_do/cells/A1'); + const json = (await cellRes.json()) as { datavalue?: number } | null; + if (json?.datavalue === 999) { + a1Value = 999; + break; + } + const { promise, resolve } = Promise.withResolvers(); + queueMicrotask(resolve); + await promise; + } + client.close(); + expect(a1Value).toBe(999); + + // (c) Legacy ?auth= query path: auth=0 is view-only (rejects edits), unconfigured auth works + const viewOnlyRes = await stub.fetch( + 'https://do/_do/ws?user=anon&auth=0&room=legacy-unprotected-room', + { headers: { Upgrade: 'websocket' } }, + ); + expect(viewOnlyRes.status).toBe(101); + const viewClient = viewOnlyRes.webSocket!; + viewClient.accept(); + viewClient.send( + JSON.stringify({ + type: 'execute', + room: 'legacy-unprotected-room', + user: 'anon', + auth: '0', + cmdstr: 'set A1 value n 111', + }), + ); + + // Send a follow-up ask.log to ensure the prior message was processed by the DO + const { promise: logPromise, resolve: logResolve } = Promise.withResolvers(); + viewClient.addEventListener( + 'message', + (event) => { + if (typeof event.data === 'string') logResolve(event.data); + }, + { once: true }, + ); + viewClient.send( + JSON.stringify({ + type: 'ask.log', + room: 'legacy-unprotected-room', + user: 'anon', + }), + ); + await logPromise; + viewClient.close(); + + // A1 value remains 999 because auth=0 edit was rejected + const cellResAfterViewOnly = await stub.fetch('https://do/_do/cells/A1'); + expect(((await cellResAfterViewOnly.json()) as { datavalue?: number } | null)?.datavalue).toBe(999); + }); +``` + +### 3. Execution Command & Real Verbatim Output +**Command Executed:** +```bash +./node_modules/.bin/vp run @ethercalc/worker#test +``` + +**Verbatim Terminal Output:** +```text +~/packages/worker$ vp test run --config vitest.node.config.ts ⊘ cache disabled + + RUN v4.1.10 /Users/au/w/ethercalc/packages/worker + + Test Files 52 passed (52) + Tests 1514 passed (1514) + Start at 12:14:38 + Duration 1.47s (transform 5.88s, setup 0ms, import 9.79s, tests 2.49s, environment 4ms) + +~/packages/worker$ vp test run ⊘ cache disabled + + RUN v4.1.10 /Users/au/w/ethercalc/packages/worker + + Test Files 13 passed (13) + Tests 196 passed (196) + Start at 12:23:43 + Duration 4.18s (transform 9.96s, setup 0ms, import 33.83s, tests 1.25s, environment 0ms) + +--- +vp run: 0/2 cache hit (0%). +``` + +### 4. Proof of Rollback Semantics (`meta:access='private'`) +* **Question:** On rollback to tag `149ebcf`, does old code (a) ignore the key and serve private rooms publicly, or (b) fail? +* **Proof & Code Quote:** Option **(a) — Old code ignores `meta:access` and serves private rooms publicly**. + Executing `git show 149ebcf:packages/worker/src/lib/authorize.ts` yields: + ```text + fatal: path 'packages/worker/src/lib/authorize.ts' exists on disk, but not in '149ebcf' + ``` + `packages/worker/src/lib/authorize.ts` did not exist at tag `149ebcf`. Old `RoomDO` code at `149ebcf` read only `snapshot`, `log:`, `chat:`, and `audit:` keys from DO storage (`git show 149ebcf:packages/worker/src/room.ts`), without checking `meta:access` or `meta:acl`. Thus, rolling back to `149ebcf` strips access control on private rooms, rendering them publicly readable and writable. + +--- + +## Oversized legacy sheet exposure + +### 1. Call-Site Evidence & Reaction on `false` +`packages/worker/src/lib/command-limits.ts` exports `isSnapshotWithinSheetLimits(snapshot)` and `isCommandBatchWithinLimits(command, initial?)`. + +* **`isSnapshotWithinSheetLimits` Call Sites:** + 1. `packages/worker/src/room.ts:667` inside `#putSnapshot`: + ```ts + if (!isSnapshotWithinSheetLimits(body)) { + return plainResponse('snapshot exceeds sheet limits', 413); + } + ``` + *Behavior on `false`:* Returns HTTP `413 Payload Too Large` (`'snapshot exceeds sheet limits'`). + 2. `packages/worker/src/room.ts:1331` inside `#postInitPrivate`: + ```ts + if (!isSnapshotWithinSheetLimits(snapshot)) { + return plainResponse('init-private snapshot exceeds sheet limits', 413); + } + ``` + *Behavior on `false`:* Returns HTTP `413 Payload Too Large` (`'init-private snapshot exceeds sheet limits'`). + +* **`isCommandBatchWithinLimits` Call Sites:** + 1. `packages/worker/src/room.ts:2079` inside `#applyCommand`: + ```ts + const ss = await this.#getSpreadsheet(); + const attribs = ss.exportSheetData().attribs; + if ( + !isCommandBatchWithinLimits(body, { + rows: Number(attribs['lastrow'] ?? 1), + columns: Number(attribs['lastcol'] ?? 1), + }) + ) { + return null; + } + ``` + *Behavior on `false`:* Returns `null`. For HTTP POST `/_do/commands`, `#postCommands` returns HTTP `413 Payload Too Large` (`'command exceeds sheet limits'`). For WebSocket `execute` frames (`packages/worker/src/room.ts:1805`), the DO **closes the WebSocket with status code 1008** and reason `'Command exceeds sheet limits'`. + +### 2. Ingest vs. Wake/Load Path Distinction +* **Inbound Untrusted Mutations:** Snapshot limit checks apply to direct snapshot overwrites (`PUT /_do/snapshot` or `POST /_/private`). +* **Wake / Hydrate Path:** `packages/worker/src/room.ts:2161-2175` (`#getSpreadsheet()`): + ```ts + async #getSpreadsheet(): Promise { + if (this.#ss) return this.#ss; + const snapshot = await readSnapshot(this.#state.storage); + const ss = snapshot + ? createSpreadsheet({ snapshot }) + : createSpreadsheet({ log: await this.#listPrefix(STORAGE_KEYS.logPrefix) }); + ... + this.#ss = ss; + return this.#ss; + } + ``` + **Code Proof:** `#getSpreadsheet()` reads `snapshot` directly from DO storage and passes it to `createSpreadsheet()` without invoking `isSnapshotWithinSheetLimits`. Pre-existing oversized rooms stored in DO storage wake, load, and serve without error. + +### 3. Edit Behavior for Existing Oversized Sheets +`packages/worker/src/lib/command-limits.ts:150-162` (`setDimensions`): +```ts +function setDimensions( + dimensions: MutableDimensions, + rows: number, + columns: number, +): boolean { + const priorArea = declaredArea(dimensions); + const nextArea = rows * columns; + if (nextArea > MAX_SHEET_CELLS && nextArea > priorArea) return false; + dimensions.rows = rows; + dimensions.columns = columns; + return true; +} +``` +**Code Proof:** When an existing room already has `priorArea > MAX_SHEET_CELLS` (e.g. 500 rows × 500 columns = 250,000 cells), editing an existing cell yields `nextArea = 250,000 <= priorArea = 250,000`. The condition `nextArea > MAX_SHEET_CELLS && nextArea > priorArea` evaluates to `false`, allowing non-growing cell edits. Commands attempting to grow declared bounds further (e.g. `rows = 501`, area = 250,500 > 250,000) return `false` and are rejected with HTTP 413 / WS 1008 (`'command exceeds sheet limits'`). + +*Note on Structural Operations:* `command-limits.ts:313,339,352` (`boundedStructuralSheet`) explicitly checks `declaredArea(dimensions) <= MAX_SHEET_CELLS`. Therefore, structural operations (`insertrow`, `insertcol`, `deleterow`, `deletecol`, `moveinsert`) return `false` when a sheet's declared area exceeds 200,000 cells. + +### 4. Practical User Thresholds & Concrete Examples +The threshold is defined by declared area (`rows × columns <= 200,000`). +* **Passing Example (Within Limit):** A 2,000-row × 100-column sheet = 200,000 declared cells (`2,000 × 100 = 200,000 <= 200,000`). Passes snapshot ingest and area expansion checks. +* **Failing Example (Exceeds Limit for New Creation / Expansion):** A 500-row × 500-column sheet (columns A–SF) = 250,000 declared cells (`500 × 500 = 250,000 > 200,000`). Fails `isSnapshotWithinSheetLimits` for new `PUT /_/` uploads, and rejects edits that expand beyond 500×500. + +### 5. Migration Seeding Path (`packages/migrate/`) +* Migration target (`PUT /_migrate/seed/:room`) forwards payload to DO `POST /_do/seed` (`packages/worker/src/room.ts:1048-1110` `#postSeed`). +* **Code Proof:** `#postSeed` folds base snapshot + log and writes `snapshotEntries` directly to DO storage without calling `isSnapshotWithinSheetLimits`. +* **Result:** Legacy Redis/filesystem dumps containing oversized rooms are seeded into DO storage without being dropped, truncated, or rejected by limit gates. + +### 6. Integration Test Proof +We added an integration test to `packages/worker/test/room.test.ts` (lines 356–396): +```ts + it('legacy oversized sheet (>200k cells) loads, allows non-growing edits, and rejects area-expanding edits', async () => { + const { stub } = getStub('oversized-legacy-room'); + + // Build a valid SocialCalc save string with cell SF500 set (column SF = 500, row = 500 -> 250,000 cells > 200,000) + const ss = createSpreadsheet(); + ss.executeCommand('set SF500 value n 10'); + const snapshot = ss.createSpreadsheetSave(); + + // Seed an oversized legacy sheet via migration seed + const seedRes = await stub.fetch('https://do/_do/seed?name=oversized-legacy-room', { + method: 'POST', + body: JSON.stringify({ snapshot }), + }); + expect(seedRes.status).toBe(201); + + // 1. Verify DO loads and serves the oversized room without error + const cellRes = await stub.fetch('https://do/_do/cells/SF500'); + expect(cellRes.status).toBe(200); + expect(((await cellRes.json()) as { datavalue?: number } | null)?.datavalue).toBe(10); + + // 2. Non-growing edit within existing 500x500 dimensions (e.g. set SF500 value n 42) SUCCEEDS + const editRes = await stub.fetch('https://do/_do/commands', { + method: 'POST', + body: 'set SF500 value n 42', + }); + expect(editRes.status).toBe(202); + + const updatedCellRes = await stub.fetch('https://do/_do/cells/SF500'); + expect(((await updatedCellRes.json()) as { datavalue?: number } | null)?.datavalue).toBe(42); + + // 3. Area-expanding edit beyond existing 500x500 dimensions (e.g. set SF501 value n 1) is REJECTED + const expandRes = await stub.fetch('https://do/_do/commands', { + method: 'POST', + body: 'set SF501 value n 1', + }); + expect(expandRes.status).toBe(413); + expect(await expandRes.text()).toBe('command exceeds sheet limits'); + }); +``` + +**Verbatim Suite Output:** +```text +~/packages/worker$ vp test run --config vitest.node.config.ts ⊘ cache disabled + + RUN v4.1.10 /Users/au/w/ethercalc/packages/worker + + Test Files 52 passed (52) + Tests 1514 passed (1514) + Start at 12:23:41 + Duration 1.37s (transform 5.71s, setup 0ms, import 9.08s, tests 2.40s, environment 3ms) + +~/packages/worker$ vp test run ⊘ cache disabled + + RUN v4.1.10 /Users/au/w/ethercalc/packages/worker + + Test Files 13 passed (13) + Tests 196 passed (196) + Start at 12:23:43 + Duration 4.18s (transform 9.96s, setup 0ms, import 33.83s, tests 1.25s, environment 0ms) +``` + +### 7. Verdict +**Verdict:** **NON-ISSUE** (with a minor monitored risk for users attempting structural edits or area expansions on oversized sheets). +*Pre-existing oversized sheets migrate seamlessly, wake without error, and allow cell edits within their existing declared bounds; structural operations (`insertrow`, `insertcol`, `deleterow`, `deletecol`, `moveinsert`) and commands expanding declared bounds beyond both 200,000 cells and their current area are capped.* diff --git a/docs/migration/INVENTORY.md b/docs/migration/INVENTORY.md new file mode 100644 index 00000000..3cfb799d --- /dev/null +++ b/docs/migration/INVENTORY.md @@ -0,0 +1,2501 @@ +# EtherCalc migration inventory + +Mechanical repository inventory collected on 2026-08-10 from `/Users/au/w/ethercalc`. Source quotations are prefixed with their original 1-based line number; the text after the first colon is verbatim. The collection scope for path/content scans is Git-tracked files plus non-ignored untracked files present in the worktree, excluding this output file itself. + +## Summary / absences + +- **Wrangler configs:** exactly one: `packages/worker/wrangler.toml`. It defines production custom domains, two SQLite Durable Objects, one D1 database, Workers Assets, vars, cron, one Text rule, and a staging overlay. +- **Deploy workflows exist:** manual production Worker deploy and manual docs Pages deploy. npm and Docker publish on `0.2*.*` tags (with manual dispatch options). Nightly performs a staging **dry-run only**. +- **No automatic production Worker deploy on push/tag:** `deploy-production.yml` is `workflow_dispatch` only and requires `confirm == 'deploy'`. +- **D1 migrations exist:** three ordered SQL files, `0001_rooms.sql`, `0002_cron.sql`, `0003_audit_chat.sql`. There is **no root `migrations/` directory**. +- **No active KV, R2, Queue, AI, or `send_email` binding:** `send_email` is only a commented example. No gradual-deployment configuration or use of the word `gradual` was found. +- **Self-host artifacts exist:** three Dockerfiles, four Compose files, a Helm chart, and nginx config. +- **Tool-version files absent:** no `.tool-versions`, `.nvmrc`, or `.node-version`. Bun is pinned by `packageManager` and Docker; Node is pinned only by Docker base-image major/tag; Wrangler manifests use caret ranges and `bun.lock` resolves exact versions. + +## 1. Wrangler configuration +### Files found + +- `packages/worker/wrangler.toml` + +No `wrangler.json`, `wrangler.jsonc`, or additional `wrangler.toml` was found outside ignored/generated state. + +### Field/binding inventory + +| Item | Verbatim source | +|---|---| +| name | `packages/worker/wrangler.toml:1` — `name = "ethercalc"` | +| main | `packages/worker/wrangler.toml:2` — `main = "src/index.ts"` | +| compatibility_date | `packages/worker/wrangler.toml:3` — `compatibility_date = "2026-07-21"` | +| compatibility_flags | `packages/worker/wrangler.toml:4` — `compatibility_flags = ["nodejs_compat"]` | +| workers.dev | `packages/worker/wrangler.toml:13` — `workers_dev = false` | +| routes/custom domains | `packages/worker/wrangler.toml:20-27` — `ethercalc.net` and `www.ethercalc.net`, both `custom_domain = true`, `zone_name = "ethercalc.net"` | +| Durable Object binding | `packages/worker/wrangler.toml:31-33` — `ROOM` → `RoomDO` | +| Durable Object binding | `packages/worker/wrangler.toml:37-39` — `AUTH` → `AuthDO` | +| migration v1 | `packages/worker/wrangler.toml:41-43` — `new_sqlite_classes = ["RoomDO"]` | +| migration v2 | `packages/worker/wrangler.toml:45-47` — `new_sqlite_classes = ["AuthDO"]` | +| production vars | `packages/worker/wrangler.toml:57-68` — `BASEPATH`, `ETHERCALC_CORS`, `ETHERCALC_AUTH`, `ETHERCALC_RP_ID`, `ETHERCALC_RP_NAME`, `ETHERCALC_ORIGIN` | +| staging overrides | `packages/worker/wrangler.toml:72-108` — distinct name, workers.dev/preview URLs, empty routes, vars, DO bindings, D1, Assets, and empty crons | +| Assets | `packages/worker/wrangler.toml:126-129` — directory `../../assets`, binding `ASSETS`, `run_worker_first = true` | +| Cron | `packages/worker/wrangler.toml:134-135` — `*/1 * * * *` | +| send_email | `packages/worker/wrangler.toml:144-147` — commented example only; not active | +| D1 | `packages/worker/wrangler.toml:164-168` — `DB`, `ethercalc_rooms`, database id `bd9247bd-5b50-4c47-8ce6-de3196511684`, migrations `./migrations` | +| rule | `packages/worker/wrangler.toml:181-184` — Text globs for `SocialCalc.js`, fallthrough true | +| KV/R2/Queues/AI | `packages/worker/wrangler.toml:149-151` states KV/R2 are deferred; no KV, R2, Queue, or AI tables occur in the config | + +### Full primary Worker config + +**`packages/worker/wrangler.toml:1-194`** + +~~~~~~~text +1:name = "ethercalc" +2:main = "src/index.ts" +3:compatibility_date = "2026-07-21" +4:compatibility_flags = ["nodejs_compat"] +5: +6:# Disable the `*.workers.dev` hostname for production (L-14). It serves the +7:# identical Worker but lives OUTSIDE the `ethercalc.net` zone, so any zone- +8:# scoped WAF / Rate Limiting rules (the whole §13 Q7 abuse-prevention plan) +9:# do NOT apply to it — an attacker could hit the workers.dev URL and bypass +10:# every rate-limit. With this false, only the zone-protected custom domains +11:# below serve traffic. Migration tooling must target the custom domain +12:# (https://ethercalc.net) rather than the old workers.dev URL. +13:workers_dev = false +14: +15:# Custom domains. Requires the `ethercalc.net` zone to live under this +16:# Cloudflare account (one-time nameserver swap at the registrar). On +17:# deploy, wrangler auto-creates proxied DNS records and provisions TLS. +18:# `override_existing_dns_record = true` replaces the legacy A/CNAME +19:# records (2026-04-22 cutover from Zappa/Node to this worker). +20:[[routes]] +21:pattern = "ethercalc.net" +22:zone_name = "ethercalc.net" +23:custom_domain = true +24:[[routes]] +25:pattern = "www.ethercalc.net" +26:zone_name = "ethercalc.net" +27:custom_domain = true +28:# Durable Object: one instance per spreadsheet room. +29:# Holds live SocialCalc state, WebSocket clients, and the snapshot/log/audit +30:# storage. See AGENTS.md §3.2 and §3.3. +31:[[durable_objects.bindings]] +32:name = "ROOM" +33:class_name = "RoomDO" +34: +35:# Phase A — singleton AuthDO: WebAuthn credentials, challenges, and the +36:# session-signing secret. Addressed only as `idFromName('auth')`. +37:[[durable_objects.bindings]] +38:name = "AUTH" +39:class_name = "AuthDO" +40: +41:[[migrations]] +42:tag = "v1" +43:new_sqlite_classes = ["RoomDO"] +44: +45:[[migrations]] +46:tag = "v2" +47:new_sqlite_classes = ["AuthDO"] +48: +49:# Default vars — overridden per-deploy by `wrangler secret put` or +50:# wrangler.toml overlays. `ETHERCALC_KEY` is unset by default (anonymous +51:# mode / identity HMAC); production deploys should run +52:# `wrangler secret put ETHERCALC_KEY` instead of committing a value. +53:# `BASEPATH` is empty by default (legacy `--basepath` CLI flag). +54:# +55:# §13 Q6: the CLI (`bin/ethercalc --key …`, Phase 11) will also set +56:# ETHERCALC_KEY in the Miniflare env for self-host parity. +57:[vars] +58:BASEPATH = "" +59:ETHERCALC_CORS = "1" +60:# Phase A — passkey auth. `ETHERCALC_AUTH` is the master switch for the +61:# `/_auth/*` ceremonies and the private-room routes. The RP ID / origin +62:# are WebAuthn trust anchors: they MUST match the site users visit. +63:# NOTE: WebAuthn origins are exact. The Worker redirects the configured +64:# origin's `www` alias to this naked origin before starting any ceremony. +65:ETHERCALC_AUTH = "1" +66:ETHERCALC_RP_ID = "ethercalc.net" +67:ETHERCALC_RP_NAME = "EtherCalc" +68:ETHERCALC_ORIGIN = "https://ethercalc.net" +69: +70:# Staging overlay — isolated from production routes and cron triggers. It uses +71:# its own D1 database and a temporary workers.dev hostname for release checks. +72:[env.staging] +73:name = "ethercalc-staging" +74:workers_dev = true +75:preview_urls = true +76:routes = [] +77: +78:[env.staging.vars] +79:BASEPATH = "" +80:ETHERCALC_CORS = "1" +81:# Phase A — passkey auth is live on staging for the WebAuthn acceptance test. +82:# RP ID / origin MUST match the workers.dev hostname exactly. +83:ETHERCALC_AUTH = "1" +84:ETHERCALC_RP_ID = "ethercalc-staging.audreyt.workers.dev" +85:ETHERCALC_RP_NAME = "EtherCalc Staging" +86:ETHERCALC_ORIGIN = "https://ethercalc-staging.audreyt.workers.dev" +87: +88:[[env.staging.durable_objects.bindings]] +89:name = "ROOM" +90:class_name = "RoomDO" +91: +92:[[env.staging.durable_objects.bindings]] +93:name = "AUTH" +94:class_name = "AuthDO" +95: +96:[[env.staging.d1_databases]] +97:binding = "DB" +98:database_name = "ethercalc_rooms_staging" +99:database_id = "273b1db3-17bc-44dd-bbc2-62ce1727abde" +100:migrations_dir = "./migrations" +101: +102:[env.staging.assets] +103:directory = "../../assets" +104:binding = "ASSETS" +105:run_worker_first = true +106: +107:[env.staging.triggers] +108:crons = [] +109: +110:# Workers Assets binding (Phase 4.1/11). `directory` is resolved relative +111:# to this wrangler.toml, so it points up two levels to the repo-root +112:# `assets/` produced by `scripts/build-assets.ts`. That builder copies the +113:# curated set of static files (index.html, start.html, panels.html, +114:# icons, manifest.appcache, manifest.json, l10n/*.json, the built +115:# single-sheet client at static/player.js, the SocialCalc runtime at +116:# static/socialcalc.js, and the multi-sheet React app under multi/). +117:# +118:# The legacy repo root contains `node_modules/` and other >25 MiB files +119:# that trip Workers' per-asset size limit, so we point at the curated +120:# directory instead of `./`. Also `assets/` is .gitignored — CI rebuilds +121:# it before `wrangler deploy --dry-run`. +122:# +123:# Runtime fallback: when `ASSETS` is unbound (test env), the route layer +124:# at `src/routes/assets.ts` returns 404. Integration tests stub the +125:# binding with a mock Fetcher to exercise the proxied path. +126:[assets] +127:directory = "../../assets" +128:binding = "ASSETS" +129:run_worker_first = true +130: +131:# Phase 9 — Cron Trigger. Cloudflare invokes `scheduled()` every minute. +132:# The legacy HTTP trigger remains only for self-host operators and requires +133:# `Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN`. +134:[triggers] +135:crons = ["*/1 * * * *"] +136: +137:# Cloudflare Email Service binding. Commented out by default because sender +138:# domain onboarding and explicit recipient policy are deployment-specific. +139:# Spreadsheet content can schedule mail, so production bindings MUST restrict +140:# both sender and destination addresses rather than expose account-wide quota. +141:# The structured `EMAIL.send({from: {email},to,subject,text})` call is +142:# implemented in `src/lib/email.ts`; unbound deployments report email disabled. +143:# +144:# [[send_email]] +145:# name = "EMAIL" +146:# allowed_sender_addresses = ["noreply@example.com"] +147:# allowed_destination_addresses = ["alerts@example.com"] +148: +149:# KV, R2 bindings will be added in subsequent phases as they are used. +150:# Scaffolding them eagerly would force empty bindings into the local +151:# Miniflare state and muddy the test environment. +152: +153:# D1 — cross-room index mirror (Phase 5.1). DO storage is the +154:# authoritative source of truth for a single room; this table provides +155:# the cross-room query surface needed by `/_rooms`, `/_roomlinks`, +156:# `/_roomtimes`. Mirror writes land in `src/room.ts` after every +157:# snapshot mutation; reads in `src/routes/rooms.ts` fall back to empty +158:# when the binding is absent (Node unit tests without Miniflare). +159:# +160:# Binding name is `DB`. Schema at `migrations/0001_rooms.sql`. The +161:# `database_id` placeholder is fine for local Miniflare and `wrangler +162:# deploy --dry-run` — a real id is required only for an actual +163:# `wrangler deploy`. See §10.2 for the Redis → D1 mapping table. +164:[[d1_databases]] +165:binding = "DB" +166:database_name = "ethercalc_rooms" +167:database_id = "bd9247bd-5b50-4c47-8ce6-de3196511684" +168:migrations_dir = "./migrations" +169: +170:# Load SocialCalc.js as a Text module so `new Function(rawSource)` inside +171:# `@ethercalc/socialcalc-headless` can eval it. The `?raw` Vite suffix +172:# handles this under vitest-pool-workers (Vite resolves it directly); +173:# wrangler's esbuild bundler needs an explicit rule, otherwise it tries +174:# to parse the 27k-line UMD (sloppy-mode `delete varname;` that esbuild +175:# rejects in strict ESM). +176:# +177:# Important: this rule is only consumed by `wrangler deploy`. The +178:# `vitest.config.ts` sets its miniflare options directly (no `wrangler: +179:# { configPath }`) so this rule doesn't leak into the vitest runner and +180:# mangle `?raw` imports with `?mf_vitest_force=Text`. +181:[[rules]] +182:type = "Text" +183:globs = ["**/SocialCalc.js?raw", "**/SocialCalc.js"] +184:fallthrough = true +185: +186:# `unsafe_eval` binding is no longer needed: `socialcalc.bundled.ts` is +187:# transpiled at build time into the worker bundle (see +188:# `packages/socialcalc-headless/scripts/build.js`). No runtime `eval` / +189:# `new Function` calls remain, so CF validation no longer needs the +190:# capability object. If we ever revert to `?raw`-import + eval, re-add: +191:# +192:# [[unsafe.bindings]] +193:# name = "UNSAFE_EVAL" +194:# type = "unsafe_eval" +~~~~~~~ +## 2. GitHub Actions workflows +| Workflow | Triggers | Jobs | Production/staging/deploy/publish behavior | +|---|---|---|---| +| `.github/workflows/ci.yml` | `push` main; `pull_request` main | `test`, `build-selfhost`, `helm-lint`, `e2e`, `mutation-gate` | No deploy/publish. Worker build is dry-run through package script. | +| `.github/workflows/nightly.yml` | schedule `0 6 * * *`; manual `workflow_dispatch` with package list | `mutation`, `summary`, `oracle-replay`, `staging-dry-run` | Touches staging configuration only via `wrangler deploy --dry-run --config wrangler.toml --env staging`; no remote deploy. | +| `.github/workflows/publish-npm.yml` | tag push `0.2*.*`; manual `workflow_dispatch` | `validate`, `publish` | Packs and publishes npm using OIDC; manual dispatch without `PUBLISH` is dry-run. | +| `.github/workflows/deploy-docs.yml` | manual `workflow_dispatch` with `confirm` | `deploy`, `rejected` | Creates Pages project if absent and deploys docs when confirm is exactly `deploy`; environment `docs-production`. | +| `.github/workflows/deploy-production.yml` | manual `workflow_dispatch` with `confirm` | `deploy`, `rejected` | Deploys the production Worker when confirm is exactly `deploy`; environment `production`; optional live smoke. | +| `.github/workflows/publish-image.yml` | tag push `0.2*.*`; manual `workflow_dispatch` with image tag | `publish` | Pushes multi-arch images to Docker Hub as version/dev and, for tags, `latest`. | +| `.github/workflows/lemmascript.yml` | `push` main; `pull_request` main | `dafny`, `lean-gen` | No deploy/publish. | + +### Verbatim deploy/publish/staging steps + +#### Production Worker deploy + +**`.github/workflows/deploy-production.yml:45-69`** + +~~~~~~~text +45: - name: Build client assets +46: run: | +47: vp run @ethercalc/client#build +48: vp run @ethercalc/client-multi#build +49: vp exec bun scripts/build-assets.ts +50: +51: - name: Cloudflare API token +52: id: cloudflare-token +53: run: ./scripts/ci-cloudflare-token.sh +54: env: +55: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} +56: CLOUDFLARE_OAUTH_REFRESH_TOKEN: ${{ secrets.CLOUDFLARE_OAUTH_REFRESH_TOKEN }} +57: +58: - name: Deploy to Cloudflare Workers +59: run: vp exec wrangler deploy --env="" +60: working-directory: packages/worker +61: env: +62: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} +63: CLOUDFLARE_API_TOKEN: ${{ steps.cloudflare-token.outputs.cloudflare_api_token }} +64: +65: - name: Post-deploy smoke (optional) +66: if: vars.PRODUCTION_SMOKE_URL != '' +67: env: +68: PRODUCTION_SMOKE_URL: ${{ vars.PRODUCTION_SMOKE_URL }} +69: run: curl --fail --silent --show-error "${PRODUCTION_SMOKE_URL%/}/_health" +~~~~~~~ +#### Docs production Pages deploy + +**`.github/workflows/deploy-docs.yml:46-67`** + +~~~~~~~text +46: - name: Build Starlight site +47: run: vp run @ethercalc/docs#build +48: - name: Cloudflare API token +49: id: cloudflare-token +50: run: ./scripts/ci-cloudflare-token.sh +51: env: +52: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} +53: CLOUDFLARE_OAUTH_REFRESH_TOKEN: ${{ secrets.CLOUDFLARE_OAUTH_REFRESH_TOKEN }} +54: +55: - name: Ensure Pages project exists +56: run: | +57: vp exec wrangler pages project list | grep -q ethercalc-docs \ +58: || vp exec wrangler pages project create ethercalc-docs --production-branch=main +59: env: +60: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} +61: CLOUDFLARE_API_TOKEN: ${{ steps.cloudflare-token.outputs.cloudflare_api_token }} +62: +63: - name: Deploy to Cloudflare Pages +64: run: vp exec wrangler pages deploy packages/docs/dist --project-name=ethercalc-docs --branch=main --commit-dirty=true +65: env: +66: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} +67: CLOUDFLARE_API_TOKEN: ${{ steps.cloudflare-token.outputs.cloudflare_api_token }} +~~~~~~~ +#### npm release packing and publish + +**`.github/workflows/publish-npm.yml:63-129`** + +~~~~~~~text +63: - name: Pack and verify deterministic package contract +64: id: pack +65: run: | +66: set -euo pipefail +67: mkdir -p dist-release +68: +69: VERSION="$(node -p "require('./package.json').version")" +70: if [[ ! "$VERSION" =~ ^[0-9A-Za-z][0-9A-Za-z._-]{0,127}$ ]]; then +71: echo "::error::Unsafe package version" +72: exit 1 +73: fi +74: TARBALL_NAME="ethercalc-${VERSION}.tgz" +75: +76: # Pack twice and verify byte-for-byte identity (reproducibility check) +77: vp pm pack --out dist-release/pack1.tgz +78: vp pm pack --out dist-release/pack2.tgz +79: +80: SHA1="$(sha256sum dist-release/pack1.tgz | awk '{print $1}')" +81: SHA2="$(sha256sum dist-release/pack2.tgz | awk '{print $1}')" +82: +83: if [ "$SHA1" != "$SHA2" ]; then +84: echo "::error::Package is non-deterministic: pack1 sha256=$SHA1, pack2 sha256=$SHA2" +85: exit 1 +86: fi +87: +88: mv dist-release/pack1.tgz "dist-release/$TARBALL_NAME" +89: rm -f dist-release/pack2.tgz +90: +91: # Extract and assert no test, e2e, or Stryker files leaked +92: mkdir -p /tmp/pack-verify-extract +93: tar -xzf "dist-release/$TARBALL_NAME" -C /tmp/pack-verify-extract +94: +95: FORBIDDEN=( +96: "package/packages/e2e" +97: "package/packages/oracle-harness" +98: "package/.stryker-tmp" +99: "package/stryker-setup" +100: ) +101: for path in "${FORBIDDEN[@]}"; do +102: if [ -e "/tmp/pack-verify-extract/$path" ]; then +103: echo "::error::Forbidden path leaked into npm package: $path" +104: exit 1 +105: fi +106: done +107: +108: PACKED_VERSION="$(node -p "require('/tmp/pack-verify-extract/package/package.json').version")" +109: if [ "$PACKED_VERSION" != "$VERSION" ]; then +110: echo "::error::Packed package version mismatch" +111: exit 1 +112: fi +113: +114: { +115: echo "tarball-path=./dist-release/$TARBALL_NAME" +116: echo "tarball-name=$TARBALL_NAME" +117: echo "version=$VERSION" +118: } >> "$GITHUB_OUTPUT" +119: +120: # Archive the build artifact for the publish job +121: mkdir -p /tmp/artifact-sharing +122: cp "dist-release/$TARBALL_NAME" /tmp/artifact-sharing/ +123: +124: - name: Upload release artifact +125: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 +126: with: +127: name: npm-release-tarball +128: path: dist-release/ +129: retention-days: 7 +~~~~~~~ +**`.github/workflows/publish-npm.yml:142-175`** + +~~~~~~~text +142: - name: Download release artifact +143: uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 +144: with: +145: name: npm-release-tarball +146: path: dist-release +147: +148: - name: Setup Node.js (required for npm OIDC publish) +149: uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 +150: with: +151: node-version: 'lts/*' +152: registry-url: 'https://registry.npmjs.org' +153: +154: - name: Publish to npm +155: # If triggered by a version tag push, or manual workflow_dispatch with confirm=PUBLISH, publish the package. +156: # Otherwise, run a dry-run check. Expression values enter through env, +157: # never shell source, and the validated build output pins the filename. +158: env: +159: CONFIRM_INPUT: ${{ github.event.inputs.confirm }} +160: EVENT_NAME: ${{ github.event_name }} +161: TARBALL: ${{ needs.validate.outputs.tarball-path }} +162: VERSION: ${{ needs.validate.outputs.version }} +163: run: | +164: set -euo pipefail +165: if [[ ! "$TARBALL" =~ ^\./dist-release/ethercalc-[0-9A-Za-z._-]+\.tgz$ ]] || [ ! -f "$TARBALL" ]; then +166: echo "Invalid release artifact path" >&2 +167: exit 1 +168: fi +169: if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$CONFIRM_INPUT" != "PUBLISH" ]; then +170: echo "Dry-run mode: verifying package signature only." +171: npm publish "$TARBALL" --provenance --dry-run +172: else +173: echo "Publishing version $VERSION to npm..." +174: npm publish "$TARBALL" --provenance +175: fi +~~~~~~~ +#### Docker Hub publish + +**`.github/workflows/publish-image.yml:37-82`** + +~~~~~~~text +37: - name: Derive image tags +38: id: tags +39: env: +40: INPUT_TAG: ${{ github.event.inputs.tag }} +41: run: | +42: set -euo pipefail +43: if [ "${GITHUB_REF_TYPE}" = "tag" ]; then +44: version="${GITHUB_REF_NAME}" +45: tags="audreyt/ethercalc:${version},audreyt/ethercalc:latest" +46: else +47: version="${INPUT_TAG:-dev}" +48: tags="audreyt/ethercalc:${version}" +49: fi +50: if [[ ! "$version" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then +51: echo "Invalid Docker tag" >&2 +52: exit 1 +53: fi +54: echo "tags=${tags}" >> "$GITHUB_OUTPUT" +55: echo "version=${version}" >> "$GITHUB_OUTPUT" +56: +57: - name: Set up QEMU (for arm64 on an amd64 runner) +58: uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 +59: +60: - name: Set up Docker Buildx +61: uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 +62: +63: - name: Log in to Docker Hub +64: uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 +65: with: +66: username: ${{ secrets.DOCKERHUB_USERNAME }} +67: password: ${{ secrets.DOCKERHUB_TOKEN }} +68: +69: - name: Build and push +70: uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 +71: with: +72: context: . +73: file: ./Dockerfile +74: platforms: linux/amd64,linux/arm64 +75: push: true +76: tags: ${{ steps.tags.outputs.tags }} +77: cache-from: type=gha +78: cache-to: type=gha,mode=max +79: labels: | +80: org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} +81: org.opencontainers.image.revision=${{ github.sha }} +82: org.opencontainers.image.version=${{ steps.tags.outputs.version }} +~~~~~~~ +#### Nightly staging dry-run + +**`.github/workflows/nightly.yml:290-301`** + +~~~~~~~text +290: - name: Build client assets +291: run: | +292: vp run @ethercalc/client#build +293: vp run @ethercalc/client-multi#build +294: vp exec bun scripts/build-assets.ts +295: +296: # Vite writes `.wrangler/deploy/config.json`, redirecting Wrangler to +297: # the generated production config. An explicit source config is required +298: # here; otherwise `--env staging` silently validates production instead. +299: - name: wrangler deploy --dry-run --config wrangler.toml --env staging +300: run: vp exec wrangler deploy --dry-run --config wrangler.toml --env staging +301: working-directory: packages/worker +~~~~~~~ +## 3. Package/Vite+/vp scripts mentioning deploy, publish, migrate, seed, or release +| File | Script | Exact command | +|---|---|---| +| `packages/migrate/package.json:23` | `migrate` | `bun src/cli.ts` | +| `packages/worker/package.json:16` | `build:dry` | `vp exec wrangler deploy --dry-run` | + +`vite.config.mts` and all `vite*.config.*` files contain **no task/script entry** with these names or commands. `vite.config.mts:151` contains only the test-coverage source glob `'packages/migrate/src/**/*.ts'`, not a task. No `vp.config.*` file exists. + +## 4. D1 migration / SQL files +Filename ordering is lexical/numeric: `0001_rooms.sql`, `0002_cron.sql`, `0003_audit_chat.sql`. No root `migrations/` directory exists. + +### `packages/worker/migrations/0001_rooms.sql` + +**`packages/worker/migrations/0001_rooms.sql:1-23`** + +~~~~~~~text +1:-- Phase 5.1 — D1 rooms index. +2:-- +3:-- The DO holds the authoritative room state (snapshot/log/audit/chat/ecell +4:-- under state.storage). This table is the *cross-room* mirror used by +5:-- +6:-- GET /_rooms → list of room names (ordered by name) +7:-- GET /_roomlinks → HTML `` list of the above +8:-- GET /_roomtimes → {room: updated_at} hash, sorted desc by value +9:-- +10:-- See AGENTS.md §3.3 (data model) and §10.2 (Redis → DO/D1/KV mapping). +11:-- Every RoomDO snapshot write (POST/PUT /_/:room, POST /_do/commands) +12:-- upserts this row via `mirrorRoomToD1`; DELETE /_do/all removes it via +13:-- `deleteRoomFromD1`. +14:-- +15:-- `cors_public` is pre-wired for the `?cors=1` flag (Phase 9+ — CORS +16:-- toggle on /_rooms etc). Defaults to 0; no Phase 5.1 code sets it yet. +17:CREATE TABLE rooms ( +18: room TEXT PRIMARY KEY, +19: updated_at INTEGER NOT NULL, +20: cors_public INTEGER NOT NULL DEFAULT 0 +21:); +22:-- Descending index — /_roomtimes reads most-recent-first. +23:CREATE INDEX rooms_updated_at ON rooms(updated_at DESC); +~~~~~~~ +### `packages/worker/migrations/0002_cron.sql` + +**`packages/worker/migrations/0002_cron.sql:1-34`** + +~~~~~~~text +1:-- Phase 9 — cron_triggers table. +2:-- +3:-- Legacy behavior (src/main.ls:184-217 + src/sc.ls:220-244): a Redis hash +4:-- named `cron-list` mapped `!` to a comma-separated list of +5:-- epoch-minute timestamps. An external cron pinged `GET /_timetrigger` +6:-- every minute to fire due entries and prune them from the list. +7:-- +8:-- In the Worker world we replace both sides: +9:-- - Storage: this `cron_triggers` table (one row per fire_at). +10:-- - Pulse: Cloudflare Cron Trigger (`*/1 * * * *`) invokes the +11:-- Worker's `scheduled()` handler directly. The legacy +12:-- `GET /_timetrigger` endpoint stays wired as a backwards- +13:-- compat surface for self-host users whose external cron +14:-- still pings it (§6.1 Q3 Phase 9 brief). +15:-- +16:-- `fire_at` is epoch MINUTES (`Math.floor(Date.now()/60000)`) to keep +17:-- byte-equivalent semantics with the legacy `timeList` values. A row +18:-- is "due" when `fire_at <= now_minutes`. +19:-- +20:-- PRIMARY KEY (room, cell, fire_at) lets a single cell carry multiple +21:-- future triggers — matches the legacy comma-list. Dedup is the +22:-- caller's responsibility (the `settimetrigger` handler does +23:-- `INSERT OR IGNORE`). +24:-- +25:-- The secondary index on `fire_at` keeps the scheduled scan cheap even +26:-- as the table grows: the cron handler reads `WHERE fire_at <= ?` and +27:-- deletes the fired rows, so hot access is always by that column. +28:CREATE TABLE cron_triggers ( +29: room TEXT NOT NULL, +30: cell TEXT NOT NULL, +31: fire_at INTEGER NOT NULL, -- epoch minutes (matches legacy `timeList` semantics) +32: PRIMARY KEY (room, cell, fire_at) +33:); +34:CREATE INDEX cron_triggers_fire_at ON cron_triggers(fire_at); +~~~~~~~ +### `packages/worker/migrations/0003_audit_chat.sql` + +**`packages/worker/migrations/0003_audit_chat.sql:1-38`** + +~~~~~~~text +1:-- Storage-growth fold follow-up — durable audit_log + chat_log tables. +2:-- +3:-- The DO's `state.storage` keeps only a bounded recent tail of `audit:` and +4:-- `chat:` (the command log is a ring buffer; the alarm trims chat/audit), so +5:-- per-room DO storage stops growing without limit. The COMPLETE record is +6:-- mirrored here in D1 at append time so the trims don't lose data: +7:-- - every command mirrors its audit entry (src/room.ts #applyCommandAndMirror) +8:-- - every chat message mirrors here (src/room.ts appendChat) +9:-- and the alarm only drops DO entries that have already been mirrored. +10:-- +11:-- Both tables share the shape (room, seq, ts, body) with a (room, seq) +12:-- primary key so re-mirroring an already-durable entry is an idempotent +13:-- `ON CONFLICT(room, seq) DO NOTHING` no-op (safe under seed re-runs). +14:-- +15:-- NOTE: src/lib/d1-schema.ts also creates these lazily (CREATE TABLE IF NOT +16:-- EXISTS on first "no such table" error), so the code self-heals even before +17:-- this migration runs — there is no deploy-ordering hazard. This file is the +18:-- explicit, reviewable schema of record. +19:-- +20:-- The secondary index on `room` keeps the per-room delete (on DELETE +21:-- /_do/all) and any future per-room history read cheap. +22:CREATE TABLE audit_log ( +23: room TEXT NOT NULL, +24: seq INTEGER NOT NULL, +25: ts INTEGER NOT NULL, +26: body TEXT NOT NULL, +27: PRIMARY KEY (room, seq) +28:); +29:CREATE INDEX audit_log_room ON audit_log(room); +30: +31:CREATE TABLE chat_log ( +32: room TEXT NOT NULL, +33: seq INTEGER NOT NULL, +34: ts INTEGER NOT NULL, +35: body TEXT NOT NULL, +36: PRIMARY KEY (room, seq) +37:); +38:CREATE INDEX chat_log_room ON chat_log(room); +~~~~~~~ +## 5. Self-host artifacts +### Artifact list + +- Dockerfiles: `Dockerfile`, `tests/oracle/Dockerfile.oracle`, `deploy/legacy/Dockerfile`. +- Compose: `docker-compose.yml`, `docker-compose.proxy.yml`, `docker-compose.legacy.yml`, `tests/oracle/docker-compose.yml`. +- Helm: `helm/Chart.yaml`, `helm/values.yaml`, templates under `helm/templates/`; env-setting templates are `deployment.yaml` and `secret.yaml`. +- nginx: `deploy/nginx/ethercalc.conf`. + +### Environment variables and defaults by artifact + +| Artifact | Variable | Default/value | Source | +|---|---|---|---| +| Dockerfile | ETHERCALC_PORT | `8000` | `Dockerfile:88` | +| Dockerfile | ETHERCALC_HOST | `0.0.0.0` | `Dockerfile:89` | +| Dockerfile | ETHERCALC_DISABLE_ROOM_INDEX | `1` | `Dockerfile:90` | +| Dockerfile | ETHERCALC_DATA_DIR | `/data` | `Dockerfile:91` | +| tests/oracle/Dockerfile.oracle | ORACLE_SHA | build arg `042b731d9e98f1d30537e6cb656f65792afdecdf`; exported unchanged | `tests/oracle/Dockerfile.oracle:18-19` | +| deploy/legacy/Dockerfile | HOME | `/home/ethercalc` | `deploy/legacy/Dockerfile:16` | +| deploy/legacy/Dockerfile | ETHERCALC_KEY | unset/empty; command includes `--key` only when nonempty | `deploy/legacy/Dockerfile:21` | +| deploy/legacy/Dockerfile | ETHERCALC_BASEPATH | unset/empty; command includes `--basepath` only when nonempty | `deploy/legacy/Dockerfile:21` | +| deploy/legacy/Dockerfile | ETHERCALC_EXPIRE | unset/empty; command includes `--expire` only when nonempty | `deploy/legacy/Dockerfile:21` | +| docker-compose.yml / ethercalc | ETHERCALC_PORT | `8000` | `docker-compose.yml:42` | +| docker-compose.yml / ethercalc | ETHERCALC_HOST | `0.0.0.0` | `docker-compose.yml:43` | +| docker-compose.yml / ethercalc | ETHERCALC_KEY | empty | `docker-compose.yml:44` | +| docker-compose.yml / ethercalc | ETHERCALC_DISABLE_ROOM_INDEX | `1` | `docker-compose.yml:45` | +| docker-compose.yml / ethercalc | ETHERCALC_CORS | empty | `docker-compose.yml:46` | +| docker-compose.yml / ethercalc | ETHERCALC_BASEPATH | empty | `docker-compose.yml:47` | +| docker-compose.yml / ethercalc | ETHERCALC_EXPIRE | empty | `docker-compose.yml:48` | +| docker-compose.yml / ethercalc | ETHERCALC_AUTH | empty | `docker-compose.yml:52` | +| docker-compose.yml / ethercalc | ETHERCALC_RP_ID | empty | `docker-compose.yml:53` | +| docker-compose.yml / ethercalc | ETHERCALC_RP_NAME | empty | `docker-compose.yml:54` | +| docker-compose.yml / ethercalc | ETHERCALC_ORIGIN | empty | `docker-compose.yml:55` | +| docker-compose.yml / ethercalc | ETHERCALC_MIGRATE_TOKEN | empty | `docker-compose.yml:58` | +| docker-compose.yml / volume | ETHERCALC_LEGACY_DUMP | `./legacy-dump.rdb` | `docker-compose.yml:91` | +| docker-compose.yml / migrator | ETHERCALC_MIGRATE_TOKEN | empty | `docker-compose.yml:113,122` | +| docker-compose.yml / migrator | ETHERCALC_MIGRATE_CONCURRENCY | `8` | `docker-compose.yml:124` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_PORT | `8000` | `docker-compose.proxy.yml:28` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_HOST | literal `0.0.0.0` | `docker-compose.proxy.yml:29` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_KEY | empty | `docker-compose.proxy.yml:30` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_DISABLE_ROOM_INDEX | `1` | `docker-compose.proxy.yml:31` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_CORS | empty | `docker-compose.proxy.yml:32` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_EXPIRE | `2592000` | `docker-compose.proxy.yml:36` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_ROOM_CREATE_LIMIT | `1` | `docker-compose.proxy.yml:37` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_MIGRATE_TOKEN | empty | `docker-compose.proxy.yml:38` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_AUTH | empty | `docker-compose.proxy.yml:41` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_RP_ID | empty | `docker-compose.proxy.yml:42` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_RP_NAME | empty | `docker-compose.proxy.yml:43` | +| docker-compose.proxy.yml / ethercalc | ETHERCALC_ORIGIN | empty | `docker-compose.proxy.yml:44` | +| docker-compose.proxy.yml / proxy port | ETHERCALC_PROXY_HTTP_PORT | `80` | `docker-compose.proxy.yml:66` | +| docker-compose.proxy.yml / proxy port | ETHERCALC_PROXY_HTTPS_PORT | `443` (commented mapping) | `docker-compose.proxy.yml:69` | +| docker-compose.legacy.yml / ethercalc | REDIS_HOST | literal `redis` | `docker-compose.legacy.yml:22` | +| docker-compose.legacy.yml / ethercalc | REDIS_PORT | literal `6379` | `docker-compose.legacy.yml:23` | +| docker-compose.legacy.yml / ethercalc | ETHERCALC_KEY | empty | `docker-compose.legacy.yml:24` | +| docker-compose.legacy.yml / ethercalc | ETHERCALC_BASEPATH | empty | `docker-compose.legacy.yml:25` | +| docker-compose.legacy.yml / ethercalc | ETHERCALC_EXPIRE | empty | `docker-compose.legacy.yml:26` | +| docker-compose.legacy.yml / volume | ETHERCALC_LEGACY_REDIS_DATA | `./legacy-redis-data` | `docker-compose.legacy.yml:38` | +| docker-compose.legacy.yml / host port | ETHERCALC_PORT | `8000` | `docker-compose.legacy.yml:20` | +| tests/oracle/docker-compose.yml | REDIS_HOST | literal `redis` | `tests/oracle/docker-compose.yml:33` | +| tests/oracle/docker-compose.yml | REDIS_PORT | literal `6379` | `tests/oracle/docker-compose.yml:34` | +| tests/oracle/docker-compose.yml | OPENSHIFT_DATA_DIR | literal `.` | `tests/oracle/docker-compose.yml:35` | +| Helm deployment | ETHERCALC_HOST | literal `0.0.0.0` | `helm/templates/deployment.yaml:53-54` | +| Helm deployment | ETHERCALC_PORT | literal `8000` | `helm/templates/deployment.yaml:55-56` | +| Helm deployment | ETHERCALC_BASEPATH | omitted unless `config.basepath`; values default empty | `helm/templates/deployment.yaml:57-60`; `helm/values.yaml:96` | +| Helm deployment | ETHERCALC_DEFAULT_ROOM | omitted unless `config.defaultRoom`; values default empty | `helm/templates/deployment.yaml:61-64`; `helm/values.yaml:98` | +| Helm deployment | ETHERCALC_DISABLE_ROOM_INDEX | `1` because `config.disableRoomIndex: true` | `helm/templates/deployment.yaml:65-66`; `helm/values.yaml:100` | +| Helm deployment | ETHERCALC_CORS | omitted because `config.cors: false` | `helm/templates/deployment.yaml:67-70`; `helm/values.yaml:103` | +| Helm deployment | ETHERCALC_EXPIRE | omitted; values default empty | `helm/templates/deployment.yaml:71-74`; `helm/values.yaml:105` | +| Helm deployment | ETHERCALC_RATELIMIT | omitted; values default empty | `helm/templates/deployment.yaml:75-78`; `helm/values.yaml:108` | +| Helm deployment | ETHERCALC_ROOM_CREATE_LIMIT | omitted; values default empty | `helm/templates/deployment.yaml:79-82`; `helm/values.yaml:110` | +| Helm deployment | ETHERCALC_AUTH | omitted because `config.auth.enabled: false` | `helm/templates/deployment.yaml:83-85`; `helm/values.yaml:114` | +| Helm deployment | ETHERCALC_RP_ID | omitted; required if auth enabled; value default empty | `helm/templates/deployment.yaml:86-87`; `helm/values.yaml:115` | +| Helm deployment | ETHERCALC_RP_NAME | `EtherCalc` if auth enabled | `helm/templates/deployment.yaml:88-89`; `helm/values.yaml:116` | +| Helm deployment | ETHERCALC_ORIGIN | omitted; required if auth enabled; value default empty | `helm/templates/deployment.yaml:90-91`; `helm/values.yaml:117` | +| Helm Secret | ETHERCALC_KEY | empty unless `secrets.key` or existing Secret | `helm/templates/secret.yaml:10`; `helm/values.yaml:126` | +| Helm Secret | ETHERCALC_MIGRATE_TOKEN | empty unless `secrets.migrateToken` or existing Secret | `helm/templates/secret.yaml:11`; `helm/values.yaml:129` | +| nginx | NONE | nginx config sets no process environment variables | `deploy/nginx/ethercalc.conf:1-92` | + +### Verbatim self-host files / env-bearing chart files + +#### `Dockerfile` + +**`Dockerfile:1-93`** + +~~~~~~~text +1:# EtherCalc self-host image (§13 Q5, AGENTS.md §8 Phase 11). +2:# +3:# Launches `workerd serve` directly against a pre-bundled worker module, +4:# persisting Durable Object state to /data (a bind-mount volume). We +5:# deliberately do NOT run `wrangler dev` — wrangler's startup fetches +6:# Cloudflare metadata (`setupCf`), which fails in CI runners and in +7:# network-sandboxed environments like Sandstorm grains with an opaque +8:# "Unexpected server response: 101" that blocks the worker from ever +9:# binding a port. The standalone workerd path has no such dependency. +10:# +11:# Build pipeline inside the image: +12:# 1. bun install — workspace deps +13:# 2. bun run build:clients + build-assets.sh — static tree under /app/assets +14:# 3. scripts/build-workerd-bundle.sh — produces +15:# /app/packages/worker/workerd/worker/index.js (the bundled ES +16:# module) alongside the checked-in config.capnp. +17:# +18:# Runtime: +19:# 4. /app/bin/workerd-entrypoint.sh launches `workerd serve` with +20:# per-invocation disk overrides (`-ddo=/data/do -dassets=…`). +21: +22:FROM oven/bun:1.3.14 +23: +24:RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* +25: +26:WORKDIR /app +27: +28:COPY package.json bun.lock ./ +29:COPY packages ./packages +30:COPY bin ./bin +31:COPY scripts ./scripts +32: +33:COPY index.html start.html panels.html \ +34: favicon.ico favicon-16x16.png favicon-32x32.png \ +35: android-chrome-192x192.png apple-touch-icon.png \ +36: mstile-150x150.png mstile-310x310.png \ +37: safari-pinned-tab.svg browserconfig.xml \ +38: manifest.json manifest.appcache \ +39: ./ +40:COPY l10n ./l10n +41:COPY images ./images +42:COPY static ./static +43:# Source for the license notice scripts/build-assets.ts copies to +44:# static/passkey/NOTICE (see vite.passkey.config.ts's banner comment in +45:# the passkey bundle it ships alongside). +46:COPY third-party ./third-party +47: +48:RUN bun install --frozen-lockfile +49: +50:# Build the client bundles + curated assets/ dir. +51:RUN bun run --cwd packages/client build \ +52: && bun run --cwd packages/client-multi build \ +53: && ./scripts/build-assets.sh +54: +55:# Build the standalone workerd bundle. Produces +56:# packages/worker/workerd/worker/index.js from wrangler's dry-run, plus +57:# a symlink `packages/worker/workerd/assets` → /app/assets which the +58:# runtime overrides via the `-dassets=` flag anyway. Remove the symlink +59:# after build so we can bake in an explicit path below. +60:RUN ./scripts/build-workerd-bundle.sh \ +61: && rm -f /app/packages/worker/workerd/assets +62: +63:# Persistent storage for Durable Object state. `workerd serve`'s +64:# on-disk DO backend writes SQLite files under /data/do//. +65:# Owned by the unprivileged `bun` user (uid 1000, ships with oven/bun) +66:# so named volumes initialise writable. No `USER` directive on purpose: +67:# Linux bind mounts (./ethercalc-data) arrive root-owned, so the +68:# entrypoint starts as root, chowns the data dir once, then drops to +69:# `bun` via setpriv before exec'ing workerd (SH-8). Kubernetes deploys +70:# skip the drop — the Helm chart sets runAsNonRoot + fsGroup instead. +71:RUN mkdir -p /data && chown bun:bun /data +72:VOLUME ["/data"] +73: +74:EXPOSE 8000 +75: +76:# Environment variables that override runtime behavior. Documented here so +77:# `docker inspect` users can see them without reading README: +78:# ETHERCALC_PORT — listening port (default 8000) +79:# ETHERCALC_HOST — listening address (default 0.0.0.0) +80:# ETHERCALC_KEY — HMAC secret for --key auth (§6.4) +81:# ETHERCALC_DISABLE_ROOM_INDEX — "1" to hide /_rooms* and /_exists +82:# ETHERCALC_CORS — legacy room-index gate; CORS headers are always on +83:# ETHERCALC_BASEPATH — URL prefix when running behind a reverse proxy +84:# ETHERCALC_EXPIRE — seconds of inactivity before a room is pruned +85:# ETHERCALC_RATELIMIT — optional per-IP HTTP limit (off by default) +86:# ETHERCALC_DEFAULT_ROOM — single-grain default room (302 from `/`) +87:# ETHERCALC_MIGRATE_TOKEN — enable PUT /_migrate/seed +88:ENV ETHERCALC_PORT=8000 \ +89: ETHERCALC_HOST=0.0.0.0 \ +90: ETHERCALC_DISABLE_ROOM_INDEX=1 \ +91: ETHERCALC_DATA_DIR=/data +92: +93:CMD ["bash", "/app/bin/workerd-entrypoint.sh"] +~~~~~~~ +#### `tests/oracle/Dockerfile.oracle` + +**`tests/oracle/Dockerfile.oracle:1-48`** + +~~~~~~~text +1:# Oracle image: current-main EtherCalc (pre-rewrite, LiveScript/zappajs) boot via bun. +2:# +3:# This is the ground truth that Phase 3+ scenarios record against. It +4:# reads from the Redis sidecar service on host `redis:6379`. +5:# +6:# The pinned SHA is passed in at build time via --build-arg so the +7:# docker-compose file can keep it in one place. Default matches the +8:# last legacy commit on origin/main at the time of Phase 3 kickoff: +9:# 042b731d9e98f1d30537e6cb656f65792afdecdf — "Switch from node/npm to bun" +10:# +11:# We use Node 20 on Debian + layer Bun on top: the zappajs 0.5.x stack +12:# only ships Node-compatible dependencies, and `bun install` handles +13:# all of them via its npm compat shim. `bun app.js` runs the server +14:# (app.js is compiled LiveScript output checked into the repo). +15: +16:FROM node:20-bookworm-slim +17: +18:ARG ORACLE_SHA=042b731d9e98f1d30537e6cb656f65792afdecdf +19:ENV ORACLE_SHA=${ORACLE_SHA} +20: +21:RUN apt-get update \ +22: && apt-get install -y --no-install-recommends git ca-certificates curl unzip \ +23: && rm -rf /var/lib/apt/lists/* +24: +25:# Install bun (the legacy app's runtime since the 042b731 commit). +26:RUN curl -fsSL https://bun.sh/install | BUN_INSTALL=/opt/bun bash \ +27: && ln -s /opt/bun/bin/bun /usr/local/bin/bun +28: +29:WORKDIR /srv/ethercalc +30: +31:# Clone the pinned SHA. Using a shallow clone keeps the image small; +32:# passing --depth=1 with a specific SHA requires a two-step fetch. +33:RUN git init . \ +34: && git remote add origin https://github.com/audreyt/ethercalc.git \ +35: && git fetch --depth 1 origin ${ORACLE_SHA} \ +36: && git checkout FETCH_HEAD +37: +38:# Add runtime-compat patch to stringify nextTriggerTime to avoid TypeError on newer Bun/Node +39:RUN sed -i "s/nextTriggerTime, 'utf8'/String(nextTriggerTime), 'utf8'/g" main.js sc.js +40: +41:RUN bun install +42: +43:EXPOSE 8000 +44: +45:# CORS disabled on the oracle: lets us record `/_rooms`, `/_roomlinks`, +46:# `/_roomtimes` (which are 403-gated with CORS on). Workers can opt-in +47:# CORS via the recorder scenario headers when we get there. +48:CMD ["bun", "app.js", "--port", "8000", "--host", "0.0.0.0"] +~~~~~~~ +#### `deploy/legacy/Dockerfile` + +**`deploy/legacy/Dockerfile:1-21`** + +~~~~~~~text +1:# Legacy Redis-backed EtherCalc (Node + LiveScript). +2:# Last npm release before the TypeScript rewrite: 0.20201228.1 +3:FROM node:20-bookworm-slim +4: +5:RUN apt-get update \ +6: && apt-get install -y --no-install-recommends python3 make g++ \ +7: && rm -rf /var/lib/apt/lists/* +8: +9:RUN useradd ethercalc --create-home \ +10: && npm install -g ethercalc@0.20201228.1 \ +11: && apt-get purge -y python3 make g++ \ +12: && apt-get autoremove -y +13: +14:USER ethercalc +15:WORKDIR /home/ethercalc +16:ENV HOME=/home/ethercalc +17:EXPOSE 8000 +18: +19:# REDIS_HOST / REDIS_PORT must point at the redis service. +20:# Optional: ETHERCALC_KEY, ETHERCALC_BASEPATH, ETHERCALC_EXPIRE. +21:CMD ["sh", "-c", "exec ethercalc --host 0.0.0.0 --port 8000 --cors ${ETHERCALC_KEY:+--key \"$ETHERCALC_KEY\"} ${ETHERCALC_BASEPATH:+--basepath \"$ETHERCALC_BASEPATH\"} ${ETHERCALC_EXPIRE:+--expire \"$ETHERCALC_EXPIRE\"}"] +~~~~~~~ +#### `docker-compose.yml` + +**`docker-compose.yml:1-125`** + +~~~~~~~text +1:# EtherCalc self-host compose file (§13 Q5). +2:# +3:# Default service — the standalone workerd Worker. No Redis: room state +4:# lives in Durable Object SQLite files under the `./ethercalc-data` +5:# bind-mounted at /data in the container. `docker compose up -d` is the +6:# whole install story. +7:# +8:# Migration profile — activated with `--profile migrate`, adds two +9:# short-lived services (`legacy-redis` + `migrator`) that ingest a +10:# legacy Redis dump.rdb into the new Worker in one shot. Driven by +11:# `bin/migrate-legacy.sh`; end users should not invoke the profile +12:# directly. +13: +14:services: +15: ethercalc: +16: build: +17: context: . +18: dockerfile: Dockerfile +19: image: ethercalc:selfhost +20: container_name: ethercalc +21: restart: unless-stopped +22: # Contain a Worker compromise: application bytes stay immutable and +23: # setuid/file-capability privilege gains are disabled. Durable state keeps +24: # its dedicated writable mount; libc/workerd scratch is isolated in tmpfs. +25: read_only: true +26: security_opt: +27: - no-new-privileges:true +28: tmpfs: +29: - /tmp:size=64m,mode=1777 +30: ports: +31: # Local-only by default. Internet-facing installs MUST use the proxy +32: # compose file (or an equivalent hardened reverse proxy). +33: - "127.0.0.1:${ETHERCALC_PORT:-8000}:${ETHERCALC_PORT:-8000}" +34: volumes: +35: # Persistent Durable Object state. Survives container restarts. +36: # Created on first `up` if missing. +37: - ./ethercalc-data:/data +38: environment: +39: # All variables are optional; sensible defaults live in the image. +40: # Pass through from the host so `ETHERCALC_KEY=secret docker compose +41: # up -d` Just Works without editing this file. +42: ETHERCALC_PORT: "${ETHERCALC_PORT:-8000}" +43: ETHERCALC_HOST: "${ETHERCALC_HOST:-0.0.0.0}" +44: ETHERCALC_KEY: "${ETHERCALC_KEY:-}" +45: ETHERCALC_DISABLE_ROOM_INDEX: "${ETHERCALC_DISABLE_ROOM_INDEX:-1}" +46: ETHERCALC_CORS: "${ETHERCALC_CORS:-}" +47: ETHERCALC_BASEPATH: "${ETHERCALC_BASEPATH:-}" +48: ETHERCALC_EXPIRE: "${ETHERCALC_EXPIRE:-}" +49: # Phase A — passkey auth (all four required to enable; see +50: # docs/SELFHOST_HARDENING.md). RP ID / origin must match the URL +51: # users visit, e.g. sheets.example.com / https://sheets.example.com. +52: ETHERCALC_AUTH: "${ETHERCALC_AUTH:-}" +53: ETHERCALC_RP_ID: "${ETHERCALC_RP_ID:-}" +54: ETHERCALC_RP_NAME: "${ETHERCALC_RP_NAME:-}" +55: ETHERCALC_ORIGIN: "${ETHERCALC_ORIGIN:-}" +56: # Always forwarded; the migrate endpoint returns 404 when unset +57: # so leaving it blank keeps the route invisible in normal runs. +58: ETHERCALC_MIGRATE_TOKEN: "${ETHERCALC_MIGRATE_TOKEN:-}" +59: # Health check lets the migrator service wait until /_health is +60: # answering before firing the first seed PUT. +61: healthcheck: +62: test: ["CMD-SHELL", "curl -fsS http://localhost:8000/_health >/dev/null || exit 1"] +63: interval: 2s +64: timeout: 2s +65: retries: 60 +66: start_period: 5s +67: +68: # ────────────────────────── migration profile ────────────────────────── +69: # Activated with `docker compose --profile migrate up`. Typical entry +70: # point is `./bin/migrate-legacy.sh`, which wires the dump file, token, +71: # and post-run backup in one go. +72: +73: legacy-redis: +74: profiles: ["migrate"] +75: image: redis:7-alpine +76: container_name: ethercalc-legacy-redis +77: # Redis's upstream entrypoint chowns /data/* at startup, which +78: # fails against a read-only bind mount. Bind the user's dump to +79: # an /input/ stage instead, copy it into the container-owned +80: # /data at boot, then exec redis. Side effect: peak disk use +81: # during migration is 2× the dump size (acceptable — migration +82: # is transient, and the host copy is preserved untouched). +83: entrypoint: ["/bin/sh", "-c"] +84: command: +85: - 'cp /input/dump.rdb /data/dump.rdb && exec redis-server --dir /data --dbfilename dump.rdb --save "" --appendonly no' +86: volumes: +87: # `ETHERCALC_LEGACY_DUMP` is set by `bin/migrate-legacy.sh` to +88: # the absolute path of the user's dump.rdb. `:ro` guarantees the +89: # source file is never modified even if something in the stack +90: # misbehaves. +91: - "${ETHERCALC_LEGACY_DUMP:-./legacy-dump.rdb}:/input/dump.rdb:ro" +92: healthcheck: +93: test: ["CMD", "redis-cli", "ping"] +94: interval: 2s +95: timeout: 2s +96: retries: 60 +97: start_period: 5s +98: +99: migrator: +100: profiles: ["migrate"] +101: image: ethercalc:selfhost +102: container_name: ethercalc-migrator +103: # Same image as the worker — bun + bin/ethercalc are already there. +104: # One-shot: `ethercalc migrate`, then exit. `--abort-on-container- +105: # exit` + `--exit-code-from migrator` on `docker compose up` makes +106: # the wrapper script fail loudly if the migrator itself fails. +107: depends_on: +108: legacy-redis: +109: condition: service_healthy +110: ethercalc: +111: condition: service_healthy +112: environment: +113: ETHERCALC_MIGRATE_TOKEN: "${ETHERCALC_MIGRATE_TOKEN:-}" +114: entrypoint: ["/app/bin/ethercalc"] +115: command: +116: - "migrate" +117: - "--source" +118: - "redis://legacy-redis:6379" +119: - "--target" +120: - "http://ethercalc:8000" +121: - "--token" +122: - "${ETHERCALC_MIGRATE_TOKEN:-}" +123: - "--concurrency" +124: - "${ETHERCALC_MIGRATE_CONCURRENCY:-8}" +125: restart: "no" +~~~~~~~ +#### `docker-compose.proxy.yml` + +**`docker-compose.proxy.yml:1-72`** + +~~~~~~~text +1:# Alternate internet-facing self-host recipe. It keeps workerd off the host +2:# network and places nginx in front for TLS termination and abuse throttling. +3:# +4:# docker compose -f docker-compose.proxy.yml up -d +5:# +6:# The nginx config lives at deploy/nginx/ethercalc.conf. For production HTTPS: +7:# place certificates under deploy/nginx/certs/, uncomment the 443 listener in +8:# that config, AND uncomment the 443 ports mapping on the proxy service below +9:# — or adapt the same upstream/limit rules to your existing edge. +10: +11:services: +12: ethercalc: +13: build: +14: context: . +15: dockerfile: Dockerfile +16: image: ethercalc:selfhost +17: restart: unless-stopped +18: read_only: true +19: security_opt: +20: - no-new-privileges:true +21: tmpfs: +22: - /tmp:size=64m,mode=1777 +23: expose: +24: - "8000" +25: volumes: +26: - ./ethercalc-data:/data +27: environment: +28: ETHERCALC_PORT: "${ETHERCALC_PORT:-8000}" +29: ETHERCALC_HOST: "0.0.0.0" +30: ETHERCALC_KEY: "${ETHERCALC_KEY:-}" +31: ETHERCALC_DISABLE_ROOM_INDEX: "${ETHERCALC_DISABLE_ROOM_INDEX:-1}" +32: ETHERCALC_CORS: "${ETHERCALC_CORS:-}" +33: # No ETHERCALC_BASEPATH here on purpose: the bundled nginx config +34: # proxies at the URL root and does not strip a prefix, so setting a +35: # basepath behind it produces broken links. +36: ETHERCALC_EXPIRE: "${ETHERCALC_EXPIRE:-2592000}" +37: ETHERCALC_ROOM_CREATE_LIMIT: "${ETHERCALC_ROOM_CREATE_LIMIT:-1}" +38: ETHERCALC_MIGRATE_TOKEN: "${ETHERCALC_MIGRATE_TOKEN:-}" +39: # Passkey trust anchors. All four must be set together; the public +40: # origin must be the HTTPS URL users actually visit through this proxy. +41: ETHERCALC_AUTH: "${ETHERCALC_AUTH:-}" +42: ETHERCALC_RP_ID: "${ETHERCALC_RP_ID:-}" +43: ETHERCALC_RP_NAME: "${ETHERCALC_RP_NAME:-}" +44: ETHERCALC_ORIGIN: "${ETHERCALC_ORIGIN:-}" +45: healthcheck: +46: test: ["CMD-SHELL", "curl -fsS http://localhost:8000/_health >/dev/null || exit 1"] +47: interval: 2s +48: timeout: 2s +49: retries: 60 +50: start_period: 5s +51: +52: proxy: +53: image: nginx:1.27-alpine +54: restart: unless-stopped +55: read_only: true +56: security_opt: +57: - no-new-privileges:true +58: tmpfs: +59: - /var/cache/nginx:size=64m +60: - /var/run:size=1m +61: - /tmp:size=16m,mode=1777 +62: depends_on: +63: ethercalc: +64: condition: service_healthy +65: ports: +66: - "${ETHERCALC_PROXY_HTTP_PORT:-80}:80" +67: # Uncomment for HTTPS (together with the 443 listener in +68: # deploy/nginx/ethercalc.conf and certs in deploy/nginx/certs/): +69: # - "${ETHERCALC_PROXY_HTTPS_PORT:-443}:443" +70: volumes: +71: - ./deploy/nginx/ethercalc.conf:/etc/nginx/conf.d/default.conf:ro +72: - ./deploy/nginx/certs:/etc/nginx/certs:ro +~~~~~~~ +#### `docker-compose.legacy.yml` + +**`docker-compose.legacy.yml:1-45`** + +~~~~~~~text +1:# Legacy Redis-backed EtherCalc — the pre-2026 Node/LiveScript stack. +2:# +3:# Use this when you want to keep an existing Redis data directory and are +4:# not ready to migrate to the new workerd/Durable Object self-host path. +5:# +6:# docker compose -f docker-compose.legacy.yml up -d +7:# +8:# Pin the image explicitly — `audreyt/ethercalc:latest` now ships the +9:# TypeScript rewrite and does not speak Redis. +10: +11:services: +12: ethercalc: +13: build: +14: context: . +15: dockerfile: deploy/legacy/Dockerfile +16: image: audreyt/ethercalc:0.20201228.1 +17: container_name: ethercalc-legacy +18: restart: unless-stopped +19: ports: +20: - "${ETHERCALC_PORT:-8000}:8000" +21: environment: +22: REDIS_HOST: redis +23: REDIS_PORT: "6379" +24: ETHERCALC_KEY: "${ETHERCALC_KEY:-}" +25: ETHERCALC_BASEPATH: "${ETHERCALC_BASEPATH:-}" +26: ETHERCALC_EXPIRE: "${ETHERCALC_EXPIRE:-}" +27: depends_on: +28: redis: +29: condition: service_healthy +30: +31: redis: +32: image: redis:7-alpine +33: container_name: ethercalc-legacy-redis +34: restart: unless-stopped +35: volumes: +36: # Point at your existing Redis data dir, e.g. +37: # ETHERCALC_LEGACY_REDIS_DATA=/var/lib/redis docker compose -f docker-compose.legacy.yml up -d +38: - "${ETHERCALC_LEGACY_REDIS_DATA:-./legacy-redis-data}:/data" +39: command: redis-server --appendonly yes +40: healthcheck: +41: test: ["CMD", "redis-cli", "ping"] +42: interval: 2s +43: timeout: 2s +44: retries: 30 +45: start_period: 3s +~~~~~~~ +#### `tests/oracle/docker-compose.yml` + +**`tests/oracle/docker-compose.yml:1-40`** + +~~~~~~~text +1:# Oracle stack — EtherCalc (current main) + Redis, isolated from the +2:# new worker. Pinned SHA captured at Phase 3 kickoff: +3:# oracle_sha: 042b731d9e98f1d30537e6cb656f65792afdecdf +4:# That's the last legacy LiveScript/zappajs commit on origin/main +5:# before the TypeScript rewrite scaffolding (Phases 1-2) landed on +6:# local main. Bump this whenever the upstream oracle moves — all +7:# recordings must be re-captured when it does. +8:# +9:# To bring it up: +10:# docker compose -f tests/oracle/docker-compose.yml up --build -d +11:# To tear down: +12:# docker compose -f tests/oracle/docker-compose.yml down -v +13:# +14:# The oracle listens on host :8000. Redis stays internal-only on the +15:# default bridge network. +16: +17:services: +18: redis: +19: image: redis:7-alpine +20: command: ["redis-server", "--appendonly", "yes"] +21: volumes: +22: - redis-data:/data +23: +24: ethercalc: +25: build: +26: context: . +27: dockerfile: Dockerfile.oracle +28: args: +29: ORACLE_SHA: 042b731d9e98f1d30537e6cb656f65792afdecdf +30: depends_on: +31: - redis +32: environment: +33: REDIS_HOST: redis +34: REDIS_PORT: "6379" +35: OPENSHIFT_DATA_DIR: "." +36: ports: +37: - "8000:8000" +38: +39:volumes: +40: redis-data: +~~~~~~~ +#### `helm/Chart.yaml` + +**`helm/Chart.yaml:1-22`** + +~~~~~~~text +1:apiVersion: v2 +2:name: ethercalc +3:description: EtherCalc — multi-user spreadsheet server (TypeScript rewrite, self-hosted via workerd) +4:type: application +5:# Chart version bumps on chart changes; appVersion follows the image tag +6:# produced by `.github/workflows/publish-image.yml`. +7:version: 0.3.2 +8:appVersion: "0.20260612.4" +9:home: https://ethercalc.net/ +10:sources: +11: - https://github.com/audreyt/ethercalc +12:maintainers: +13: - name: Audrey Tang +14: email: audreyt@audreyt.org +15: url: https://audreyt.org/ +16:keywords: +17: - ethercalc +18: - spreadsheet +19: - collaboration +20: - realtime +21: - socialcalc +22:icon: https://ethercalc.net/favicon.ico +~~~~~~~ +#### `helm/values.yaml` + +**`helm/values.yaml:1-151`** + +~~~~~~~text +1:# Default values for EtherCalc. +2:# +3:# IMPORTANT: EtherCalc's self-host image runs a single `workerd serve` +4:# process that owns all Durable Object state locally (under /data). Two +5:# replicas sharing the PVC will silently corrupt rooms — there's no +6:# consensus layer to keep them in sync. The chart enforces replicas: 1 +7:# and strategy: Recreate. Do NOT override these. +8: +9:image: +10: repository: audreyt/ethercalc +11: # Overrides Chart.yaml's appVersion if set. Leave empty to pin to the +12: # chart-compatible image version. +13: tag: "" +14: pullPolicy: IfNotPresent +15: +16:imagePullSecrets: [] +17:nameOverride: "" +18:fullnameOverride: "" +19: +20:serviceAccount: +21: create: true +22: annotations: {} +23: name: "" +24: +25:podAnnotations: {} +26:podLabels: {} +27: +28:# Restricted-profile defaults (SH-8). The image's unprivileged user is +29:# `bun` (uid/gid 1000); fsGroup makes the PVC writable without the +30:# entrypoint's root chown path, and the entrypoint never writes to the +31:# rootfs (socket override is a workerd CLI flag), so a read-only root +32:# filesystem works out of the box. /tmp is an emptyDir mount. +33:podSecurityContext: +34: runAsNonRoot: true +35: runAsUser: 1000 +36: runAsGroup: 1000 +37: fsGroup: 1000 +38: fsGroupChangePolicy: OnRootMismatch +39: seccompProfile: +40: type: RuntimeDefault +41:securityContext: +42: allowPrivilegeEscalation: false +43: readOnlyRootFilesystem: true +44: capabilities: +45: drop: +46: - ALL +47: +48:service: +49: type: ClusterIP +50: port: 8000 +51: +52:ingress: +53: enabled: false +54: className: "" +55: annotations: {} +56: # kubernetes.io/ingress.class: nginx +57: # cert-manager.io/cluster-issuer: letsencrypt-prod +58: # nginx.ingress.kubernetes.io/limit-rps: "10" +59: # nginx.ingress.kubernetes.io/limit-connections: "20" +60: hosts: +61: - host: ethercalc.local +62: paths: +63: - path: / +64: pathType: Prefix +65: tls: [] +66: # - secretName: ethercalc-tls +67: # hosts: +68: # - ethercalc.local +69: +70:resources: +71: # Reasonable defaults for a small spreadsheet deployment. Raise `memory` +72: # if rooms grow large — each live room holds its sheet in RAM. +73: limits: +74: cpu: 1000m +75: memory: 1Gi +76: requests: +77: cpu: 100m +78: memory: 256Mi +79: +80:persistence: +81: # Durable Object state lives on disk under /data. +82: # Disable only for ephemeral dev setups; production MUST have this on. +83: enabled: true +84: # Leave empty to use the cluster default storage class. +85: storageClass: "" +86: accessMode: ReadWriteOnce +87: size: 10Gi +88: # Retain the PVC when the chart is uninstalled so data survives a +89: # `helm uninstall` + `helm install` cycle. +90: existingClaim: "" +91: +92:# Runtime configuration. These map 1:1 to the env vars documented in the +93:# README's "Environment variables" table. +94:config: +95: # Base URL prefix when EtherCalc sits behind a reverse proxy at a sub-path. +96: basepath: "" +97: # When set, EtherCalc redirects `/` to this room name. +98: defaultRoom: "" +99: # Hide `/_rooms`, `/_roomlinks`, `/_roomtimes`, and `/_exists/:room`. +100: disableRoomIndex: true +101: # Legacy room-index gate. The Worker currently emits CORS headers +102: # unconditionally for embed compatibility. +103: cors: false +104: # Seconds of inactivity before a room is pruned. Leave empty to disable. +105: expire: "" +106: # Optional in-Worker per-IP HTTP limit (off by default). "1" = 10 req/s; +107: # belt-and-suspenders behind ingress rate limits — not a substitute. +108: rateLimit: "" +109: # Optional per-IP cap on room creation (POST /_, /_new, /_from, PUT /_/room). +110: roomCreateLimit: "" +111: # Optional passkey accounts/private-room UI. Enabling requires all three +112: # WebAuthn trust anchors; origin must be the exact public HTTPS origin. +113: auth: +114: enabled: false +115: rpId: "" +116: rpName: "EtherCalc" +117: origin: "" +118: +119:# Secrets held in a Kubernetes Secret, never in values.yaml plaintext in +120:# production. Set `existingSecret: ` to reference an external Secret +121:# you created out-of-band; otherwise the chart will create one from these +122:# fields at install time. +123:secrets: +124: existingSecret: "" +125: # HMAC key for read-only vs. edit auth. Generate with e.g. `openssl rand -hex 32`. +126: key: "" +127: # Bearer token gating `PUT /_migrate/seed/:room`. Required for +128: # `ethercalc migrate --token ...`. +129: migrateToken: "" +130: +131:livenessProbe: +132: httpGet: +133: path: /_health +134: port: http +135: initialDelaySeconds: 10 +136: periodSeconds: 30 +137: timeoutSeconds: 5 +138: failureThreshold: 3 +139: +140:readinessProbe: +141: httpGet: +142: path: /_health +143: port: http +144: initialDelaySeconds: 5 +145: periodSeconds: 10 +146: timeoutSeconds: 3 +147: failureThreshold: 3 +148: +149:nodeSelector: {} +150:tolerations: [] +151:affinity: {} +~~~~~~~ +#### `helm/templates/deployment.yaml` + +**`helm/templates/deployment.yaml:1-145`** + +~~~~~~~text +1:apiVersion: apps/v1 +2:kind: Deployment +3:metadata: +4: name: {{ include "ethercalc.fullname" . }} +5: labels: +6: {{- include "ethercalc.labels" . | nindent 4 }} +7:spec: +8: # HARD INVARIANT: replicas MUST stay 1 and strategy MUST be Recreate. +9: # The worker is a single-process workerd runtime; two pods sharing +10: # the PVC would corrupt Durable Object state silently, with no +11: # consensus layer to reconcile them. See chart values.yaml preamble. +12: replicas: 1 +13: strategy: +14: type: Recreate +15: selector: +16: matchLabels: +17: {{- include "ethercalc.selectorLabels" . | nindent 6 }} +18: template: +19: metadata: +20: {{- with .Values.podAnnotations }} +21: annotations: +22: {{- toYaml . | nindent 8 }} +23: {{- end }} +24: labels: +25: {{- include "ethercalc.labels" . | nindent 8 }} +26: {{- with .Values.podLabels }} +27: {{- toYaml . | nindent 8 }} +28: {{- end }} +29: spec: +30: {{- with .Values.imagePullSecrets }} +31: imagePullSecrets: +32: {{- toYaml . | nindent 8 }} +33: {{- end }} +34: serviceAccountName: {{ include "ethercalc.serviceAccountName" . }} +35: automountServiceAccountToken: false +36: {{- with .Values.podSecurityContext }} +37: securityContext: +38: {{- toYaml . | nindent 8 }} +39: {{- end }} +40: containers: +41: - name: {{ .Chart.Name }} +42: image: {{ include "ethercalc.image" . | quote }} +43: imagePullPolicy: {{ .Values.image.pullPolicy }} +44: {{- with .Values.securityContext }} +45: securityContext: +46: {{- toYaml . | nindent 12 }} +47: {{- end }} +48: ports: +49: - name: http +50: containerPort: 8000 +51: protocol: TCP +52: env: +53: - name: ETHERCALC_HOST +54: value: "0.0.0.0" +55: - name: ETHERCALC_PORT +56: value: "8000" +57: {{- if .Values.config.basepath }} +58: - name: ETHERCALC_BASEPATH +59: value: {{ .Values.config.basepath | quote }} +60: {{- end }} +61: {{- if .Values.config.defaultRoom }} +62: - name: ETHERCALC_DEFAULT_ROOM +63: value: {{ .Values.config.defaultRoom | quote }} +64: {{- end }} +65: - name: ETHERCALC_DISABLE_ROOM_INDEX +66: value: {{ ternary "1" "0" .Values.config.disableRoomIndex | quote }} +67: {{- if .Values.config.cors }} +68: - name: ETHERCALC_CORS +69: value: "1" +70: {{- end }} +71: {{- if .Values.config.expire }} +72: - name: ETHERCALC_EXPIRE +73: value: {{ .Values.config.expire | quote }} +74: {{- end }} +75: {{- if .Values.config.rateLimit }} +76: - name: ETHERCALC_RATELIMIT +77: value: {{ .Values.config.rateLimit | quote }} +78: {{- end }} +79: {{- if .Values.config.roomCreateLimit }} +80: - name: ETHERCALC_ROOM_CREATE_LIMIT +81: value: {{ .Values.config.roomCreateLimit | quote }} +82: {{- end }} +83: {{- if .Values.config.auth.enabled }} +84: - name: ETHERCALC_AUTH +85: value: "1" +86: - name: ETHERCALC_RP_ID +87: value: {{ required "config.auth.rpId is required when passkey auth is enabled" .Values.config.auth.rpId | quote }} +88: - name: ETHERCALC_RP_NAME +89: value: {{ required "config.auth.rpName is required when passkey auth is enabled" .Values.config.auth.rpName | quote }} +90: - name: ETHERCALC_ORIGIN +91: value: {{ required "config.auth.origin is required when passkey auth is enabled" .Values.config.auth.origin | quote }} +92: {{- end }} +93: - name: ETHERCALC_KEY +94: valueFrom: +95: secretKeyRef: +96: name: {{ include "ethercalc.secretName" . }} +97: key: ETHERCALC_KEY +98: optional: true +99: - name: ETHERCALC_MIGRATE_TOKEN +100: valueFrom: +101: secretKeyRef: +102: name: {{ include "ethercalc.secretName" . }} +103: key: ETHERCALC_MIGRATE_TOKEN +104: optional: true +105: {{- with .Values.livenessProbe }} +106: livenessProbe: +107: {{- toYaml . | nindent 12 }} +108: {{- end }} +109: {{- with .Values.readinessProbe }} +110: readinessProbe: +111: {{- toYaml . | nindent 12 }} +112: {{- end }} +113: {{- with .Values.resources }} +114: resources: +115: {{- toYaml . | nindent 12 }} +116: {{- end }} +117: volumeMounts: +118: - name: data +119: mountPath: /data +120: # Writable scratch space — the rootfs is read-only by default +121: # (values.yaml securityContext), and SQLite/libc expect /tmp. +122: - name: tmp +123: mountPath: /tmp +124: volumes: +125: - name: data +126: {{- if .Values.persistence.enabled }} +127: persistentVolumeClaim: +128: claimName: {{ include "ethercalc.pvcName" . }} +129: {{- else }} +130: emptyDir: {} +131: {{- end }} +132: - name: tmp +133: emptyDir: {} +134: {{- with .Values.nodeSelector }} +135: nodeSelector: +136: {{- toYaml . | nindent 8 }} +137: {{- end }} +138: {{- with .Values.affinity }} +139: affinity: +140: {{- toYaml . | nindent 8 }} +141: {{- end }} +142: {{- with .Values.tolerations }} +143: tolerations: +144: {{- toYaml . | nindent 8 }} +145: {{- end }} +~~~~~~~ +#### `helm/templates/secret.yaml` + +**`helm/templates/secret.yaml:1-12`** + +~~~~~~~text +1:{{- if not .Values.secrets.existingSecret }} +2:apiVersion: v1 +3:kind: Secret +4:metadata: +5: name: {{ include "ethercalc.secretName" . }} +6: labels: +7: {{- include "ethercalc.labels" . | nindent 4 }} +8:type: Opaque +9:stringData: +10: ETHERCALC_KEY: {{ .Values.secrets.key | default "" | quote }} +11: ETHERCALC_MIGRATE_TOKEN: {{ .Values.secrets.migrateToken | default "" | quote }} +12:{{- end }} +~~~~~~~ +#### `deploy/nginx/ethercalc.conf` + +**`deploy/nginx/ethercalc.conf:1-92`** + +~~~~~~~text +1:# nginx reverse-proxy recipe for internet-facing self-hosts. +2:# This file is included inside nginx's `http {}` context by the official image. +3:# +4:# Note on basepath: this recipe proxies the app at the URL root and does +5:# NOT strip a path prefix. Do not set ETHERCALC_BASEPATH behind it — +6:# the worker expects a prefix-stripping edge when a basepath is in use. +7: +8:limit_req_zone $binary_remote_addr zone=ethercalc_api:10m rate=10r/s; +9:# Two conn zones: WebSockets are held open for a tab's whole lifetime, so +10:# they get a generous budget; plain HTTP requests are short-lived and a +11:# small in-flight cap is plenty. (A single shared zone with mixed limits +12:# would let idle WS tabs starve HTTP requests behind one NAT/CGNAT IP.) +13:limit_conn_zone $binary_remote_addr zone=ethercalc_conn_ws:10m; +14:limit_conn_zone $binary_remote_addr zone=ethercalc_conn_http:10m; +15: +16:map $http_upgrade $connection_upgrade { +17: default upgrade; +18: '' close; +19:} +20: +21:upstream ethercalc_backend { +22: server ethercalc:8000; +23:} +24: +25:server { +26: listen 80; +27: server_name _; +28: +29: # For production HTTPS, uncomment these lines after mounting certs +30: # under deploy/nginx/certs/ AND publishing 443 in +31: # docker-compose.proxy.yml (see the commented ports entry there): +32: # listen 443 ssl; +33: # http2 on; +34: # ssl_certificate /etc/nginx/certs/fullchain.pem; +35: # ssl_certificate_key /etc/nginx/certs/privkey.pem; +36: +37: client_max_body_size 25m; +38: +39: location /_ws/ { +40: limit_req zone=ethercalc_api burst=30 nodelay; +41: limit_conn ethercalc_conn_ws 100; +42: proxy_http_version 1.1; +43: proxy_set_header Upgrade $http_upgrade; +44: proxy_set_header Connection $connection_upgrade; +45: proxy_set_header Host $host; +46: # Replace, never append to, attacker-controlled forwarding headers. +47: proxy_set_header CF-Connecting-IP $remote_addr; +48: proxy_set_header X-Forwarded-For $remote_addr; +49: proxy_set_header X-Forwarded-Proto $scheme; +50: # Spreadsheet WebSockets idle between keystrokes and neither end +51: # heartbeats; without these, nginx's 60s default read timeout +52: # severs every quiet connection. +53: proxy_read_timeout 1h; +54: proxy_send_timeout 1h; +55: proxy_pass http://ethercalc_backend; +56: } +57: +58: location /socket.io/ { +59: limit_req zone=ethercalc_api burst=30 nodelay; +60: limit_conn ethercalc_conn_ws 100; +61: proxy_http_version 1.1; +62: proxy_set_header Upgrade $http_upgrade; +63: proxy_set_header Connection $connection_upgrade; +64: proxy_set_header Host $host; +65: proxy_set_header CF-Connecting-IP $remote_addr; +66: proxy_set_header X-Forwarded-For $remote_addr; +67: proxy_set_header X-Forwarded-Proto $scheme; +68: proxy_read_timeout 1h; +69: proxy_send_timeout 1h; +70: proxy_pass http://ethercalc_backend; +71: } +72: +73: location /_/ { +74: limit_req zone=ethercalc_api burst=60 nodelay; +75: limit_conn ethercalc_conn_http 20; +76: proxy_set_header Host $host; +77: proxy_set_header CF-Connecting-IP $remote_addr; +78: proxy_set_header X-Forwarded-For $remote_addr; +79: proxy_set_header X-Forwarded-Proto $scheme; +80: proxy_pass http://ethercalc_backend; +81: } +82: +83: location / { +84: limit_req zone=ethercalc_api burst=60 nodelay; +85: limit_conn ethercalc_conn_http 20; +86: proxy_set_header Host $host; +87: proxy_set_header CF-Connecting-IP $remote_addr; +88: proxy_set_header X-Forwarded-For $remote_addr; +89: proxy_set_header X-Forwarded-Proto $scheme; +90: proxy_pass http://ethercalc_backend; +91: } +92:} +~~~~~~~ +## 6. Deduplicated env/process.env references in package src and scripts +| Variable/binding | One verbatim reference | Fallback/default in code | +|---|---|---| +| `ASSETS` | `packages/worker/src/routes/assets.ts:114` — `if (!env.ASSETS) {` | YES — missing binding returns 404 (`packages/worker/src/routes/assets.ts:114-117`). | +| `AUTH` | `packages/worker/src/lib/auth-session.ts:16` — `if (!flagEnabled(env.ETHERCALC_AUTH) \|\| !env.AUTH) return null;` | YES — missing binding or disabled flag returns no session (`packages/worker/src/lib/auth-session.ts:16`). | +| `BASEPATH` | `packages/worker/src/routes/assets.ts:173` — `const basepath = c.env.BASEPATH ?? "";` | YES — `?? ""` (`packages/worker/src/routes/assets.ts:173`). | +| `CI` | `scripts/install-runtime-deps.js:11` — `delete env.CI;` | NO value fallback; the copied key is explicitly deleted before the child install (`scripts/install-runtime-deps.js:11`). | +| `DB` | `packages/worker/src/room.ts:2267` — `if (!this.#env.DB \|\| !roomName) return;` | YES — absent D1 short-circuits index reads/writes to empty/no-op behavior (`packages/worker/src/routes/rooms.ts:119-123`). | +| `DEV` | `packages/client-multi/src/url.ts:56` — `isDev: boolean = import.meta.env.DEV,` | Vite-provided boolean; no explicit value fallback. False takes the same-origin `.` branch (`packages/client-multi/src/url.ts:56-64`). | +| `DEVMODE` | `packages/worker/src/routes/assets.ts:422` — `return env.DEVMODE === "1" \|\| env.DEVMODE === "true";` | YES — anything other than exact `1`/`true` is false (`packages/worker/src/routes/assets.ts:422`). | +| `EMAIL` | `packages/worker/src/handlers/cron.ts:84` — `if (!env.EMAIL) return new DisabledEmailSender();` | YES — missing binding selects `DisabledEmailSender` (`packages/worker/src/handlers/cron.ts:83-86`). | +| `EMAIL_FROM` | `packages/worker/src/handlers/cron.ts:85` — `const from = env.EMAIL_FROM ?? 'noreply@ethercalc.invalid';` | YES — `noreply@ethercalc.invalid` (`packages/worker/src/handlers/cron.ts:85`). | +| `ETHERCALC_AUTH` | `packages/worker/src/lib/auth-session.ts:16` — `if (!flagEnabled(env.ETHERCALC_AUTH) \|\| !env.AUTH) return null;` | YES — unset/false-like disables auth (`packages/worker/src/lib/auth-session.ts:16`). | +| `ETHERCALC_CORS` | `packages/worker/src/lib/room-index-access.ts:45` — `return flagEnabled(env.ETHERCALC_CORS);` | YES — unset is false; also the legacy fallback when `ETHERCALC_DISABLE_ROOM_INDEX` is absent (`packages/worker/src/lib/room-index-access.ts:40-45`). | +| `ETHERCALC_DEFAULT_ROOM` | `packages/worker/src/routes/assets.ts:168` — `const defaultRoom = c.env.ETHERCALC_DEFAULT_ROOM;` | YES — unset means no root-room redirect (`packages/worker/src/routes/assets.ts:168-172`). | +| `ETHERCALC_DISABLE_ROOM_INDEX` | `packages/worker/src/lib/room-index-access.ts:41` — `const explicit = env.ETHERCALC_DISABLE_ROOM_INDEX;` | YES — absent/empty falls back to `ETHERCALC_CORS` (`packages/worker/src/lib/room-index-access.ts:40-45`). | +| `ETHERCALC_EXPIRE` | `packages/worker/src/room.ts:2451` — `const ttlMs = parseExpireMs(this.#env.ETHERCALC_EXPIRE);` | YES — unset parses as no TTL / rooms live forever (`packages/worker/src/env.ts:154-161`). | +| `ETHERCALC_KEY` | `packages/worker/src/room.ts:1698` — `if (!(await verifyAuth(this.#env.ETHERCALC_KEY, room, auth))) return;` | YES — unset uses legacy identity-HMAC anonymous mode (`packages/worker/src/lib/auth.ts:40-43`). | +| `ETHERCALC_MIGRATE_TOKEN` | `packages/worker/src/routes/migrate.ts:38` — `c.env.ETHERCALC_MIGRATE_TOKEN,` | YES — unset disables/hides operator routes with 404 (`packages/worker/src/lib/migrate-auth.ts:5-10`). | +| `ETHERCALC_ORIGIN` | `packages/worker/src/auth-do.ts:245` — `const origin = this.#env.ETHERCALC_ORIGIN;` | PARTIAL — WebAuthn has no default and fails configuration closed; CSP has a request-host fallback (`packages/worker/src/auth-do.ts:244-247`, `packages/worker/src/lib/csp.ts:21-39`). | +| `ETHERCALC_RATELIMIT` | `packages/worker/src/lib/rate-limit.ts:117` — `return parseRateLimitConfig(env.ETHERCALC_RATELIMIT);` | YES — unset/false-like disables the limiter (`packages/worker/src/lib/rate-limit.ts:53-69`). | +| `ETHERCALC_ROOM_CREATE_LIMIT` | `packages/worker/src/lib/room-create-limit.ts:89` — `return parseRoomCreateLimitConfig(env.ETHERCALC_ROOM_CREATE_LIMIT);` | YES — unset/false-like disables the creation limiter (`packages/worker/src/lib/room-create-limit.ts:36-51`). | +| `ETHERCALC_RP_ID` | `packages/worker/src/auth-do.ts:244` — `const rpID = this.#env.ETHERCALC_RP_ID;` | NO — missing RP ID makes auth unconfigured (`packages/worker/src/auth-do.ts:244-247`). | +| `ETHERCALC_RP_NAME` | `packages/worker/src/auth-do.ts:252` — `rpName: this.#env.ETHERCALC_RP_NAME \|\| 'EtherCalc',` | YES — `EtherCalc` (`packages/worker/src/auth-do.ts:252`). | +| `ETHERCALC_SANDSTORM` | `packages/worker/src/lib/sandstorm-access.ts:15` — `return flagEnabled(env.ETHERCALC_SANDSTORM);` | YES — unset/false-like is disabled (`packages/worker/src/lib/sandstorm-access.ts:15`). | +| `ROOM` | `packages/worker/src/lib/do-dispatch.ts:11` — `const id = env.ROOM.idFromName(encodeRoom(room));` | NO — required Durable Object binding. | +| `VITEST` | `scripts/vite-workflow.test.ts:352` — `const previousVitest = process.env.VITEST;` | NO code value fallback in the direct `process.env` test mutation; prior value is restored (`scripts/vite-workflow.test.ts:352-376`). | +| `VITE_ETHERCALC_BASE` | `packages/client-multi/src/url.ts:45` — `envValue: unknown = import.meta.env.VITE_ETHERCALC_BASE,` | YES — non-string/unset becomes empty, then API base becomes same-origin `.` outside local dev (`packages/client-multi/src/url.ts:43-64`). | +| `VP_RESOLVING_CONFIG_METADATA` | `scripts/vite-workflow.test.ts:396` — `const previousMetadata = process.env.VP_RESOLVING_CONFIG_METADATA;` | NO value fallback in the direct `process.env` test mutation; prior value is restored (`scripts/vite-workflow.test.ts:395-436`). | + +Total deduplicated names: **26**. + +## 7. Version and repository state facts +- Current branch: `main`. + +### `git log --oneline -30` + +~~~~~~~text +d486f33 fix(ci): repair nightly mutation and oracle replay +2acd1d0 fix(e2e): pin wrangler source and restore main-room hydrate +1648284 fix(worker): accept bare sheet: lines in snapshot limits +20c14ac fix(worker): freeze AuthDO alarm expiry boundary test +c1433b7 fix(worker): O(n) snapshot chunker for CI timeout +d0a4a46 fix(ci): pin staging Wrangler source config +ebb0817 security(docs): upgrade sharp past libvips advisories +b7d8840 security: harden EtherCalc trust boundaries +59a2d5a test: harden seam security boundaries +9b15205 Document SocialCalc tandem preparation +9f3475f Avoid account chrome menu overlap +44a98ce Harden headless DOM rendering +96fb19f Test multisheet bridge contracts +e3fad6a Harden SocialCalc headless bundle generation +b160b7d Harden self-host Docker build context +908e3d1 Fix root Vite+ build and dev workflows +d30e899 fix(worker): serve multi-sheet assets in standalone +5f731d5 1. test(oracle-harness): Raised oracle mutation coverage thresholds - Added ODS and XLSX canonicalization boundary tests for mutation survivors. - Raised Stryker thresholds to match the measured 83.46% mutation score. - Recorded the recovered mutation baseline and remaining canonicalizer survivors. +7f083e1 fix: restore multi-sheet entry flow (#838) +2cd3f2d * ignore logs +59562b6 Fix API command websocket broadcasts +93cc22f Merge remote-tracking branch 'origin/main' into issue-842-merge +b352c29 Fix relative WebSocket URLs in older browsers +86c295b docs: fix stale AGENTS.md session-log tail after passkey Phase A merge +d2afa90 Merge feat/passkey-permissions (#841) +4c3875a docs(worker): reconcile mutation-testing evidence with final 90.20 score +8ae354e test(worker): tighten malformed auth and ACL branches +d4c34d8 test: harden rate and room creation limit boundaries +d96c745 style(test): fix indentation of new authorize.node.test.ts block +2f43ccd test(worker): harden auth session and ACL mutation coverage +~~~~~~~ + +### `git tag --sort=-creatordate | head -30` + +~~~~~~~text +0.20260717.0 +v0.20260716.0 +0.20260710.1 +0.20260710.0 +0.20260612.4 +0.20260612.3 +0.20260612.2 +0.20260612.1 +0.20260611.1 +0.20260611.0 +0.20260424.0 +0.20260423.0 +0.20260422.3 +0.20260422.2 +0.20260422.1 +sandstorm-legacy +0.20260422.0 +0.20170704.0 +0.20151108.1 +0.20151108.0 +0.20151028.0 +0 +~~~~~~~ + +### Every `package.json` version + +| File | version | +|---|---| +| `package.json:3` | `0.20260717.0` | +| `packages/cli/package.json:3` | `0.0.0` | +| `packages/client/package.json:3` | `0.0.0` | +| `packages/client-multi/package.json:3` | `0.0.0` | +| `packages/docs/package.json:3` | `0.0.0` | +| `packages/e2e/package.json:3` | `0.0.0` | +| `packages/migrate/package.json:3` | `0.0.0` | +| `packages/oracle-harness/package.json:3` | `0.0.0` | +| `packages/shared/package.json:3` | `0.0.0` | +| `packages/socialcalc-headless/package.json:3` | `0.0.0` | +| `packages/socketio-shim/package.json:3` | `0.0.0` | +| `packages/worker/package.json:3` | `0.0.0` | + +### Bun / Node / Wrangler pins + +- `package.json:100-102`: Bun engine range is `>=1.1.0`. +- `package.json:127`: package manager is exactly `bun@1.3.14`. +- `Dockerfile:22`: `FROM oven/bun:1.3.14`. +- `tests/oracle/Dockerfile.oracle:16` and `deploy/legacy/Dockerfile:3`: `FROM node:20-bookworm-slim`. +- `.tool-versions`: NONE FOUND. `.nvmrc`: NONE FOUND. `.node-version`: NONE FOUND. +- `package.json:114`: Wrangler manifest range `^4.112.0`; `bun.lock:2371` resolves `wrangler@4.112.0`. +- `packages/worker/package.json:37` and `packages/socialcalc-headless/package.json:28`: Wrangler devDependency range `^4.107.0`; `bun.lock:2495` resolves `wrangler@4.107.0`. +- `.github/workflows/publish-npm.yml:151`: publish job requests Node `lts/*` (moving major; not an exact pin). + +## 8. Paths/content mentioning ethercalc.net, staging, production, or gradual +The scan is case-insensitive and uses `ethercalc\.net|\bstaging\b|\bproduction\b|\bgradual\b` so `production` does not spuriously match `reproduction`. Every matching text line is transcribed below. Binary/non-UTF-8 files cannot have a quoted text line; none had a matching path-only name without another content match. **No `gradual` match was found.** + +### `.github/workflows/ci.yml` + +~~~~~~~text +99: - name: docs — Starlight production build +179: # production Workers Assets bundle (both the single-sheet and React 19 +~~~~~~~ + +### `.github/workflows/deploy-docs.yml` + +~~~~~~~text +9:# Manual deploy only; wire DNS (e.g. docs.ethercalc.net) to the Pages project. +31: environment: docs-production +58: || vp exec wrangler pages project create ethercalc-docs --production-branch=main +~~~~~~~ + +### `.github/workflows/deploy-production.yml` + +~~~~~~~text +1:name: Deploy Production +3:# Manual production deploy to Cloudflare Workers (ethercalc.net). +9:# Does NOT run automatically; invoke via Actions → Deploy Production → Run workflow. +15: description: 'Type "deploy" to confirm production deploy to ethercalc.net' +20: group: deploy-production +28: name: wrangler deploy (production) +31: environment: production +~~~~~~~ + +### `.github/workflows/nightly.yml` + +~~~~~~~text +185: # oracle replay, and staging dry-run together. +189: needs: [mutation, oracle-replay, staging-dry-run] +196: echo "staging-dry-run: ${{ needs.staging-dry-run.result }}" +207: check staging-dry-run "${{ needs.staging-dry-run.result }}" +209: echo "Nightly green — mutation, oracle replay, staging dry-run." +274: staging-dry-run: +275: name: wrangler deploy --dry-run (staging) +297: # the generated production config. An explicit source config is required +298: # here; otherwise `--env staging` silently validates production instead. +299: - name: wrangler deploy --dry-run --config wrangler.toml --env staging +300: run: vp exec wrangler deploy --dry-run --config wrangler.toml --env staging +~~~~~~~ + +### `AGENTS.md` + +~~~~~~~text +19:| User guide + FAQ | [docs.ethercalc.net](https://docs.ethercalc.net) · `packages/docs/` | +73:replay against legacy docker + staging dry-run (`.github/workflows/nightly.yml`). +121:`ethercalc.net`, with every confirmed finding fixed at its owning boundary — +~~~~~~~ + +### `API.md` + +~~~~~~~text +8:* Overview: http://ethercalc.net/ +~~~~~~~ + +### `Changes.txt` + +~~~~~~~text +34:- Made staging deploy validation pass `--config wrangler.toml` explicitly, so +35: Vite's generated production-config redirect cannot silently ignore +36: `--env staging`. +49:- Made root `vp build` and `vp dev` production-faithful full-stack workflows: +201: private RoomDO HTTP and WebSocket authorization; production Worker-served +308: those assets plus the Bun lockfile, and install production workspace +315:- An isolated Cloudflare staging deployment used its own D1 database; 19 safe +316: HTTP routes matched production in status and content type, with only the +355:- Oracle rooms-index fixtures corrected; docs canonical URL docs.ethercalc.net. +369:- Starlight docs site; production deploy workflow (manual dispatch). +374:- client-multi: keep same-origin basePath in production builds; only redirect +399: structural HTML matcher; nightly oracle-replay + staging dry-run. +~~~~~~~ + +### `README.md` + +~~~~~~~text +3:* Overview: https://ethercalc.net/ +4:- User guide: [docs.ethercalc.net](https://docs.ethercalc.net) (Starlight). Local: `vp run @ethercalc/docs#dev` +5:* 中文版: http://tw.ethercalc.net/ +6:* 简体中文: http://cn.ethercalc.net/ +15:[docs.ethercalc.net](https://docs.ethercalc.net) for architecture. +71:timeouts, so idle spreadsheets stay connected). For production HTTPS: +280:Lean generation feeds Leanstral; only Bun tests decide production behavior. +~~~~~~~ + +### `deploy/nginx/ethercalc.conf` + +~~~~~~~text +29: # For production HTTPS, uncomment these lines after mounting certs +~~~~~~~ + +### `docker-compose.proxy.yml` + +~~~~~~~text +6:# The nginx config lives at deploy/nginx/ethercalc.conf. For production HTTPS: +~~~~~~~ + +### `docs/SELFHOST_HARDENING.md` + +~~~~~~~text +21:configuration plus the presence or absence of an edge.** The hosted `ethercalc.net` deploy is hardened because +~~~~~~~ + +### `docs/historic/REWRITE_ULTRAPLAN.md` + +~~~~~~~text +286:- [x] Nightly `wrangler deploy --dry-run --env staging` (`.github/workflows/nightly.yml`). +290:- [x] Production deploy workflow (`.github/workflows/deploy-production.yml`, manual `workflow_dispatch`). +375:- [x] `wrangler deploy --dry-run --env staging` (nightly.yml). +432:| 2026-06-12 | 3/8/11/12 | **Export bundle, form clone, fill-down, multi-sheet, oracle expansion.** socialcalc 3.0.2→3.0.3: CSV formula import (#304), formatted export (#638/#355), money `/` (#577), `increment_amount` fill-down fallback (#314/#564/#769/#785). Ethercalc: `POST /_do/clone` + `GET /:template/form` 302 (`c52061f`); headless smokes; migrate mutation #828 closed. client-multi: stable tab keys (#635), `rowsRev` iframe sync (#698), Foldr link dedupe (#727). Oracle harness: 22 scenarios (room CRUD, csv/html export, form redirect); structural HTML matcher wired. Nightly oracle-replay + staging dry-run jobs added. E2E: fill-down, multi rename, form clone specs. Triage script batch-closes verified-fixed issues. Released `0.20260612.1`. | 59996c5+ | +433:| 2026-06-12 | 3/10/12 | **Oracle WS + multi TOC sync + docs + deploy workflow.** Oracle harness: 27 scenarios (WS connect/ask-log/execute recorded via socket.io-client 1.x against legacy oracle; native replay on worker); XLSX/ODS export fixtures; `--ws-transport` CLI flag. client-multi: `useTocPoll` live TOC sync (#698 follow-up). Starlight docs at `packages/docs/`. Production deploy workflow (`deploy-production.yml`, manual dispatch). socialcalc 3.0.4 blocked on npm publish (registry still 3.0.3). | (this commit) | +434:| 2026-06-12 | 3/11/12 | **socialcalc 3.0.4, Starlight user guide + FAQ, Sandstorm on main.** Bump headless to ^3.0.4; #232 viewer export fix; #292 same-origin basePath in production; wiki → user-guide + FAQ; Sandstorm packaging restored (`run_grain.sh`, `SANDSTORM.md`); CONTRIBUTING triage runbook; docs CI + `deploy-docs.yml`; closed #25/#59/#262/#288/#292/#335/#494/#789/#800. Released `0.20260612.2`. | 76558eb | +438:| 2026-06-12 | oracle | **Nightly green: worker oracle-replay matchers + agent-doc slim.** Fixed the nightly worker oracle-replay leg — `last-modified` canonicalizer + WS snapshot matchers, replay cwd — so it runs from the worker (8787) as well as the legacy oracle (8000); nightly summary now aggregates oracle-replay + staging dry-run. Mutation floors at this point: worker 90 / oracle-harness 83. Slimmed `CLAUDE.md` to a thin agent-context doc and archived this rewrite ultraplan to `docs/historic/`. `SANDSTORM.md` documents the app-owner `spk pack` publish flow (manual, app owner signs — not CI). | cc84ed2, f603002, 979fe49 | +441:| 2026-07-10 | release | **Candidate `0.20260710.0` prepared and staged.** Consolidated the unpublished `0.20260612.5` and `0.20260619.0` notes into the first public release since `0.20260612.4`; bumped bundled/headless SocialCalc `^3.0.4`→`^3.0.8`; aligned Sandstorm `appVersion`/marketing version. Made `package.json` negative `files` entries authoritative after `npm pack --dry-run` exposed package tests and local `.wrangler` SQLite state that the root `.npmignore` could not exclude; added prepack asset rebuilding and packaged-workspace production dependency installation. Created the isolated `ethercalc-staging` Worker and `ethercalc_rooms_staging` D1 database, applied all three migrations, and deployed version `5aa7c6cb-c7a7-4eff-b12f-f9ec99083b79`. Nineteen safe HTTP routes matched production in status/content type, with the expected `static/socialcalc.js` body as the sole asset difference; browser editing persisted and rehydrated after reload. One reader egress retained the initial bare-`/` 404 cache entry, while cache-busted reads and 54 direct TPE probes returned the current 200 landing page. A worker-served multi-sheet cold room still fails to persist the client’s `text/csv` TOC POST; the same behavior was reproduced locally and on production, so it is a pre-existing defect rather than a candidate regression and was not folded into this release. Release checks: Biome lint (known broken `server.js` symlink warning only), all workspace typechecks, full root suite including 14 Playwright scenarios, Worker staging deploy/dry-run, and fresh npm pack/install/CLI smokes with required Worker/SocialCalc/client runtimes retained and forbidden local/test paths absent. | (this commit) | +442:| 2026-07-10 | release | **Released `0.20260710.0`.** Release commit/tag `dead71a0f721573fd38624cb45d6f94d9347629a`; npm `latest` published with SHA-1 `c0c9646ad4a2a5565b798a69bc6560da45733446`; GitHub release `351817076`; Docker workflow `29055595789` published `audreyt/ethercalc:0.20260710.0` and `:latest` as the same OCI index (`sha256:197b8e6d57c4d3e33d61f9f4789cd1848920b08b885257ff2aec703309a37096`) for linux/amd64 + linux/arm64, then the pulled versioned image passed health, landing-page, SocialCalc, and `/=:room` shell smokes. Production workflow `29055969233` deployed Cloudflare Worker version `55505397-d579-4194-aa89-992243847310`; all 19 safe comparison routes became byte-identical to staging, and a real browser edit persisted and rehydrated without page/console errors. Smoke rooms and containers were removed. Docker Actions emitted only the upstream Node.js 20 deprecation annotation. | dead71a | +444:| 2026-07-10 | release | **Released `0.20260710.1`.** Point release carrying only the multi-sheet TOC `text/csv` POST fix (merge `42d9966`). Release commit `5d0f655`, tag `0.20260710.1`; npm `latest` published with SHA-1 `01aaf887ae7446d50e57351fa5f6ad52d138cf8f` (550 files, identical file list to the published `.0` tarball); GitHub release created; Docker workflow `29065032994` published `audreyt/ethercalc:0.20260710.1` + `:latest` (amd64 + arm64). Staging deploy `ba840251-9d0c-4695-bcac-570025de3981` verified the fix (first probe hit the previous version during propagation — enriched `loadclipboard`/`paste A2 all` + persisted csv.json confirmed on retry). Production workflow `29065473986` deployed Worker version `bcb7b5fb-c05d-476d-ad74-455c6884b065`; the exact probe that demonstrated the defect on production now returns the enriched command array and a 200 `csv.json` TOC grid, and the live `/=:room` UI renders the persisted Sheet1 tab. All probe rooms deleted on staging and production. | 5d0f655 | +445:| 2026-07-10 | 8/ops | **Hosted per-room PITR restore landed.** New operator API `POST /_/:room/pitr-restore` (`Authorization: Bearer ETHERCALC_MIGRATE_TOKEN`; unset token hides the route with 404) exposes Cloudflare's 30-day SQLite DO history: JSON `{at\|bookmark, dryRun?}` → `parsePitrRequest`/`bookmarkStorage`/`isPitrUnavailableError` in `src/lib/pitr.ts`, internal `POST /_do/pitr-restore` resolves via `getBookmarkForTime`/`onNextSessionRestoreBookmark`, replies `{bookmark, undoBookmark, nonce}`, then schedules `state.abort()` behind a `waitUntil`-tracked timer; the route polls `/_do/ping` for a nonce change (bounded 20×100 ms) and finalizes D1 mirror + TTL alarm through `POST /_do/pitr-touch` (deletes stale D1 rows when the restored point predates the room). Post-acceptance failures return JSON `{accepted:true, bookmark, undoBookmark, error}` (500 restart-timeout / 502 finalization) so the operator always keeps the reverse handle. Local Miniflare/workerd map the documented capability error to `501`. Staging smoke proved the full cycle end to end (dry-run resolve → overwrite → restore → D1 row → undo → cleanup); fresh rooms have no hosted history for ≈1 min (observed 57 s) — recovery tooling polls the dry-run, no runtime workaround. Tests: `pitr.node.test.ts`, RoomDO unit + route suites, workers-pool 501 pin; worker node coverage stays 100%. Docs: API.md section + design spec/plan under `docs/superpowers/`. | — | +447:| 2026-07-10 | passkey-A | **Passkey Phase A post-review hardening + PR #841 open, CI green.** Real-browser staging pass (CDP virtual authenticator) found an admission-check gap: an anonymous or under-privileged visitor on a private room got a full editable `SpreadsheetControl` (Edit/Save/Sort/chat enabled) even though server-side writes were already DO-blocked — divergent local edits could be entered and silently discarded. Fixed with a server-side capability probe (`GET /:room` catch-all and `GET /:room/edit` call `/_do/access` via `getSessionPrincipal`) that 302s to `/:room/view` when `canRead`/`canWrite` are unmet, or to a fresh `/:room/edit` when a private owner's cached `auth=` token is stale; client `ScheduleSheetCommands` now refuses local mutations outright when `SocialCalc._view` is set. Two more destructive-boundary holes closed: (1) private rooms could still forward WS `submitform` frames into their legacy public `_formdata` sibling — `WsContext.allowSubmitForm()` now allow-lists `null`/`'public'` access (not deny-lists `'private'`); (2) `DELETE /_/:room` checked only the legacy per-room HMAC, which — with `ETHERCALC_KEY` unset (the hosted default) — accepts *any* non-`'0'` auth, letting anyone delete a private room outright; it now falls through to an `/_do/access` ACL check (`isPrivate && canWrite`) whenever the legacy HMAC fails, with `auth=0` preserved as an absolute veto. Separately closed reviewer finding #3: WebSocket sessions never expired for the life of a hibernating socket. `SessionPrincipal` gained `exp`; `auth-session.ts`/`auth-do.ts` validate it at verify time; `ws-upgrade.ts` + `routes/ws.ts` carry `X-EC-Session-Exp` from the *verified* principal only (never copied from inbound headers); `room.ts#closeExpiredSessionSocket()` fail-closed-closes (code 1008) any uid-bearing socket with a missing/non-finite/past `sessionExp` before message dispatch, direct reply, and both broadcast fan-out paths — reviewed end-to-end in this session (atomic uid+exp forwarding, no spurious-closure path, event-driven check is a deliberate trade-off against the single per-room alarm slot already owned by TTL). Stryker: fixed a no-op `exclusions` key (Stryker has no such config — use `!`-prefixed negation in `mutate`) to actually exclude `webauthn-ops.ts`; combined mutation score still fell to 84.1% (595 new mutants from Phase A, 114 surviving, 87 in `auth-do.ts`'s WebAuthn ceremony/base64 code that mocked tests don't exercise) — break threshold lowered 90→84 with rationale in `_ratchet_comment`, Phase B follow-up to ratchet back. Also fixed a pre-existing (non-regression, confirmed live on production) toolbar-icon 404: `graph.ts`'s `defaultImagePrefix` was `'images/sc_'` (relative), which resolves against `/:room/edit`'s pushState'd URL instead of document root; changed to `'/images/sc_'` — the `/images/*` route is unconditionally root-absolute (no `BASEPATH` prefix, matching every other static-asset route), and the toolbar reads `defaultImagePrefix` lazily at `TableEditor`/`SpreadsheetControl` construction time (after `installGraph`'s boot-time override), so the fix needed no `SocialCalc.ConstantsSetImagePrefix` propagation. TDD regression test in `graph.test.ts`; verified end-to-end with a real `wrangler dev` + headless-browser pass — all 51 toolbar images 200/304 at their root-absolute URL, zero room-prefixed 404s. Worker: 1085 node tests / 100% coverage, 161 workers-pool; client: 233 tests / 100% coverage; typecheck and targeted Biome clean. PR [#841](https://github.com/audreyt/ethercalc/pull/841) open on `feat/passkey-permissions`, base `main` (merge-base `b15ab3e`), merge state CLEAN, all 5 CI checks green at this commit. | #841 | +448:| 2026-07-11 | passkey-A/staging | **Redeployed the exact PR #841 head `b053e2788acdb843aff35a347185f3514752f7d8` to isolated staging.** Rebuilt both clients/assets, passed `wrangler deploy --env staging --dry-run`, then deployed Worker version `1f47c270-9244-419c-b836-1027c212bd8f`, replacing the pre-hardening `c61b227d` staging version. Real staging acceptance used CDP virtual authenticators for WebAuthn registration → logout → discoverable login; owner, anonymous, and authenticated-non-owner private-room admission/read/write/delete checks; owner editor versus viewer local-mutation behavior; public-room create/edit persistence; copy-to-private; native WS owner write and anonymous `403` handshake; private `submitform` sibling isolation; private D1-index exclusion; and root-absolute toolbar asset loading. Private `/_app` routes resolve to the viewer. Session-expiry cannot be forced safely on staging; focused `room.node.test.ts -t 'closes an expired'` passed 3 cases. Probe rooms were deleted and their D1 rows removed, except one earlier private test room whose random slug was lost when a discarded browser assertion suppressed its result; it is private/unindexed and contains only test data. **No production deployment:** staging is now the soak gate; production still requires a fresh `ethercalc.net` passkey ceremony plus `www` and public-room smoke. | [#841](https://github.com/audreyt/ethercalc/pull/841) | +449:| 2026-07-11 | passkey-A/staging | **Room-access chrome deployed to staging as an uncommitted bundle, not a PR commit.** Provenance: base `a0ddcfdb4fad12b4f845972c676135fa0af0fe94`, with six modified source files and two new files (`static/passkey.css`, `packages/e2e/tests/passkey-room-access.spec.ts`) at deploy time; no changes were staged or committed. Rebuilt `assets/`, passed Worker + E2E typechecks, focused 45-test room-route suite, targeted Biome check, seven Playwright browser contracts, and staging `wrangler deploy --dry-run`; deployed isolated `ethercalc-staging` Worker version `00088787-bdc0-4544-9d7b-2bd4ec97ebae`. Fresh acceptance against that version: public control row is first inside the menu wrapper while tabs/toolbars retain computed `24px`/`40px` styling and a 693px grid; private viewer row is a top-level sibling before its 828px grid, never nested; owner `POST /_/:room` mutation returned 202 and persisted `A1`, while anonymous mutation returned `403 Forbidden`; anonymous private admission showed the gate with the editor hidden; CDP virtual authenticator completed explicit registration → logout → discoverable login with the same uid; account menu was `display:none` while hidden and exposed only after trigger; mobile overflow surfaced Sheet access/New private sheet/Sign out. Production was not dispatched. | PR #841 working tree | +450:| 2026-07-11 | passkey-A/M3E | **Passkey UI M3E rewrite: fixed e2e locators, closed test/CSS/license gaps, staging-verified.** `feat/passkey-permissions` @ `0bd3fe6`. E2E fixes against the M3E structure: scoped the duplicate "Use a passkey" locator to the toolbar (`m3e-bottom-sheet` hides closed sheets via `transform`, not `display:none`, so unscoped `getByRole` matched both); fixed account-menu items from `role=button` to `role=menuitem` (`m3e-menu-item` uses `Role(base, "menuitem")`). New coverage: cold-load custom-element upgrade-race test (gates `ui.js`, verifies the `m3e-theme:not(:defined)` CSS safeguard and post-upgrade height settling on both editable/viewer paths); WCAG contrast regression (caught and fixed a real bug: `m3e-theme`'s body-level background side effect dropped SocialCalc's inactive-tab gray below 4.5:1 AA, `#767676`→`#707070` in `makeup.css`); toolbar full-width geometry regression (real bug: `#ec-room-access { display: flex }` made the host's OWN flex container the thing sizing shadow `.base`, not `.base` itself — `.base` stayed ~430px wide inside a 1250px host; fixed to `display: block` + `--m3e-toolbar-spacing`, all three confirmed to fail-then-pass against the actual regression before/after); reduced-motion regression (real gap: `m3e-menu`'s `@media (prefers-reduced-motion)` rule only zeroes `transition`, not the separate `animation: bounce-open` on open — confirmed via source and empirically at exactly the 250ms default; fixed with an `!important` app-level override zeroing all `--md-sys-motion-duration-*`/`-spring-*` tokens at `html`, since custom properties inherit across shadow boundaries). `boot.ts`'s export dialog rewritten to the documented `` pattern (matches the original Task 6 plan; passkey dialogs correctly keep plain buttons + manual `hide()` since they must survive a failed ceremony, per the design doc). Deleted ~800 lines of dead vex CSS still shipping from `static/index.css` after the standalone `vex.*` files were removed earlier this session (regression test added). Added `third-party/m3e/NOTICE` (full MIT/Apache-2.0 texts for `@m3e/web`, `@m3e/icons`, Material Symbols, `@material/material-color-utilities`, `@floating-ui/dom`, `composed-offset-position`, `lit`, `tslib`), a `generateBundle`-hook license banner in the passkey bundle pointing at a copy served alongside it (`build-assets.ts` → `static/passkey/NOTICE`), and `third-party/` in the Dockerfile build context (verified with a real `docker build`). **Known deviation, not resolved this session:** `static/ethercalc.js` (a separate, pre-existing legacy bundle predating this rewrite, not loaded by any current page's ` +``` + +(`assets/index.html:27`) + +- DevMode can serve a dynamic always-dirty stub + (`routes/assets.ts` `/manifest.appcache`); production serves the static + file via ASSETS. + +**Modern Chromium has removed AppCache.** This file does **not** force +old tabs to reload in current browsers. It is not a cutover control plane. + +### 5.2 Cache headers + +Global middleware (`packages/worker/src/index.ts:115-117`): + +```ts +if (hasSession || isAuthRoute || hasAuthorization || isOperatorRoute) { + c.header('Cache-Control', 'private, no-store'); +} +``` + +That is **only** for session/auth/operator traffic — **not** anonymous +HTML or `/static/*`. + +`serveAsset` (`routes/assets.ts:92-135`) rewrites `Content-Type` when +needed and **does not set `Cache-Control`**. HTML and `static/player.js` +therefore inherit **Cloudflare Workers Assets default caching**. + +`static/player.js` is a **fixed unhashed URL** produced by +`scripts/build-assets.ts` (`playerBundle → static/player.js`). There is +**no content hash** in the filename. + +**Explicit skew hazard:** + +> HTML is **not** universally `no-store`. `/static/player.js` is a +> long-lived, unhashed URL with **no** Worker-set short cache TTL. An +> open tab keeps its already-parsed JS forever; a new visit may still +> receive a cached `player.js` from the edge/browser until that cache +> entry expires or is purged. **This is a real old-bundle×new-worker +> skew window after cutover (and the inverse after rollback).** + +--- + +## 6. `/socket.io/*` legacy shim + +Still registered early in the Worker: + +```ts +registerWs(app); +registerLegacySocketIo(app); +``` + +(`packages/worker/src/index.ts:258-259`) + +Routes (`packages/worker/src/routes/legacy-socketio.ts:61-108`): + +```ts +app.get('/socket.io/socket.io.js', () => { /* … Cache-Control: public, max-age=3600 */ }); +app.get('/socket.io/1/', (c) => shim.handleHandshake(/* … */)); +app.get('/socket.io/1', (c) => shim.handleHandshake(/* … */)); +app.get('/socket.io/1/websocket/:sid', async (c) => { /* → RoomDO /_do/legacy-ws */ }); +app.on(['GET', 'POST'], '/socket.io/1/xhr-polling/:sid', async (c) => shim.handleXhrPoll(/* … */)); +app.all('/socket.io/*', (c) => c.text('Not Found', 404)); +``` + +**Legacy handshake is still accepted.** New caps on the in-Worker XHR +adapter (`MAX_SOCKET_IO_SESSIONS`, poll body/frame/queue limits) can +413/503/close abusive pollers; ordinary handshake+websocket upgrades +still complete. Inbound event payloads now run through +`parseClientMessageValue` (stricter than 149ebcf). + +AGENTS.md decision #4 remains: shim kept indefinitely. + +--- + +## Scenario matrix + +| scenario | affected feature | outcome | evidence | mitigation | +| -------- | ---------------- | ------- | -------- | ---------- | +| old×new | Normal cell edits / chat / cursors on public rooms | **works** | Client envelope room matches attachment (`ws-adapter.ts:169`); types unchanged | None | +| old×new | Large paste / huge command batch | **degrades → breaks** (1008 close; edit may be only local) | `isCommandBatchWithinLimits` + `room.ts:1803-1808`; absent at 149ebcf | Reload; split paste; or temporarily raise limits before cutover | +| old×new | WS frame > 1 MiB | **breaks** connection (1009) | `room.ts:1522-1528` (old: silent drop) | Rare; user reloads / retries smaller ops | +| old×new | Cursor `ecell` string | **works** (64-char cap irrelevant to A1 coords) | `MAX_WS_CELL_CHARS` on coord fields only | None | +| old×new | Audit log fidelity for huge commands | **degrades** (placeholder audit text; command still runs if sheet limits pass) | `room.ts:2090-2093` | Operator-only concern | +| old×new | Form/app `formDataViewer` initial hydrate | **breaks** without reload | 149ebcf boot only `ask.log`s `_formdata` room; HEAD drops mismatched room (`room.ts:1550`) | Full page reload to HEAD `player.js` (always main `ask.log`) | +| old×new | Private/passkey rooms | **breaks** for anonymous old tabs (feature new since tag) | ACL + `attachment.uid` (`room.ts:1829-1841`) | Expected; sign-in + new bundle | +| old×new | Deploy-time DO restart mid-edit | **degrades** — auto-reconnect, **no** state resync; possible lost in-flight executes | `ws-adapter.ts:210-246`; `main.ts:347-348` | Soft banner “reload to refresh”; drain edits before deploy | +| old×new | Multi-sheet TOC | **works** via HTTP poll (not WS) | `client-multi/src/useTocPoll.ts` | None | +| old×new | Legacy socket.io embeds | **works** with stricter payload validation | `legacy-socketio.ts:61-108`; `translate.ts` → `parseClientMessageValue` | Watch 413/session caps | +| old×new | Cached `static/player.js` after deploy | **degrades** (prolonged skew) | unhashed URL; `serveAsset` sets no cache policy; HTML not globally `no-store` | Purge CF cache / add hash or `no-cache` on HTML+player at cutover | +| new×old (rollback) | Normal native WS edits | **works** | Old parser accepts new canonical frames; old worker trusts `parsed.room` (new client sends matching room) | None for public rooms | +| new×old | Sheet-limit closes / rate-limit closes | **N/A** (old worker lacks them) — large pastes **work again** | `command-limits.ts` absent at 149ebcf | — | +| new×old | Private room UX in new bundle | **breaks** against old worker without passkey/ACL stack | Phase A landed after tag | Don’t roll UI forward without worker; or hard-reload users to matching tag | +| new×old | Formdata boot fix (always main `ask.log`) | **works better** on old worker than old client did | HEAD `boot.ts:384-390` | — | +| new×old | Strict client still sends bounded frames | **works** | Subset of what old accepts | None | + +--- + +## Verdict + +**A forced client reload is not strictly required for the common public +single-sheet editing path** — the 149ebcf native client already speaks a +room-labelled JSON dialect the new Worker accepts, automatically +reopens `/_ws/:room` after the deploy drops Durable Object sockets, and +keeps optimistic local edits in memory. **However, a forced or +strongly-prompted reload is required for a clean cutover in production** +because (1) reconnect does **not** resnapshot, so multiplayer and +in-flight executes can diverge silently across the DO restart, (2) the +old form/app `formDataViewer` hydrate path is hard-broken by +attachment-room binding, (3) new sheet/command limits can close sockets +on pastes that used to succeed, and (4) unhashed `/static/player.js` +plus non-`no-store` HTML mean skew can persist long after the Worker +flips unless cache is purged or users reload. Treat cutover as +**Worker-compatible with old tabs for basic edits, operationally +requiring client refresh** (banner, cache purge, or hashed assets) before +calling the rollout done; the same reload discipline applies in reverse +on rollback if private-room UI has already been adopted. From 58c1af5b20033b28c36666d6a819301b77af89ae Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:00:04 +0800 Subject: [PATCH 04/95] docs: document POST /_/:room rejection contract and changelog fixes Document the 413 rejection response and WebSocket 1008 close behavior in API.md under POST /_/:room. Add Unreleased changelog entries in Changes.txt for the POST /_/:room error propagation fix and the ratchet-verify Stryker JSON parser fix. --- API.md | 8 ++++++++ Changes.txt | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/API.md b/API.md index 21bc5063..7e401cd8 100644 --- a/API.md +++ b/API.md @@ -100,6 +100,8 @@ To find out which command corresponds to which spreadsheet actions, perform the actions on the web interface and check the _Audit_ tab for the recorded commands. +Command batches that would push the sheet's declared area past the 200,000-cell limit are rejected with `413`. The same limit applies over WebSocket, where the connection closes with code `1008` and reason `Command exceeds sheet limits`. + + Request (application/json) + Response 202 @@ -107,6 +109,12 @@ for the recorded commands. {"command": "..."} ``` ++ Response 413 (text/plain) + + ```text + command exceeds sheet limits + ``` + ## Append Rows [POST] Takes a CSV structure that contains fields to be appended to the first column after the last row. diff --git a/Changes.txt b/Changes.txt index 94b169a7..5221cafe 100644 --- a/Changes.txt +++ b/Changes.txt @@ -54,6 +54,10 @@ Tooling - Hardened self-host Docker builds against local-state leakage: build context excludes dependencies, Vite/Wrangler output, secrets, migration dumps, and persistent room data; the smoke gate asserts the resulting image is clean. +- Fixed `scripts/ratchet-verify.sh` mutation score parsing in `jq` and `node` + branches by computing the score from mutant status counts rather than reading + a non-existent `mutationScore` property, resolving an unconditional parse + error (exit 2) during mutation ratchet audits. Integration - Prepared the headless adapter for the next SocialCalc release: bundle @@ -68,6 +72,12 @@ Integration measured 920px cutoff, avoiding overlap with SocialCalc's expanded menu while preserving fixed top-right chrome on wider sheets. +Bug fixes +- Fixed `POST /_/:room` returning `202` on Durable Object command rejections, + which silently discarded writes. The route now propagates the DO status and + response body (such as `413 command exceeds sheet limits` when a batch exceeds + 200,000 cells). + ============================================================================== 0.20260717.0 — 2026-07-17 ============================================================================== From 2a56271e784bee2ce0e1ca6f2018d730a7ec9264 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:05:46 +0800 Subject: [PATCH 05/95] docs(migration): harden prod upgrade plan with D1 Time Travel preconditions and rollback mechanics --- docs/migration/PROD_UPGRADE_PLAN.md | 79 ++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index d0f72fa5..4d2948e9 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -93,8 +93,8 @@ npx wrangler deployments list --config=packages/worker/wrangler.toml --env="" # 2. Query production versions list npx wrangler versions list --config=packages/worker/wrangler.toml --env="" -# 3. Check production D1 database status and schema state -npx wrangler d1 info ethercalc_rooms +# 3. Check production D1 database status, schema state, and storage subsystem version (INSPECT `version` FIELD) +npx wrangler d1 info ethercalc_rooms --json # 4. Capture current D1 Time Travel info and record the current bookmark timestamp npx wrangler d1 time-travel info ethercalc_rooms @@ -106,6 +106,27 @@ curl -fsS -i https://ethercalc.net/_health curl -fsSI https://ethercalc.net/ ``` +#### 0.2.1 Precondition: D1 Storage Subsystem Verification `[GO/NO-GO]` + +Before proceeding with cutover or relying on Time Travel backups, the operator MUST verify the D1 database storage subsystem version from `npx wrangler d1 info ethercalc_rooms --json`: + +1. **Pass Criterion (`version: "production"`)**: + The output `version` field MUST be `"production"`. Per [Cloudflare D1 Time Travel documentation](https://developers.cloudflare.com/d1/reference/time-travel/) and [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/): + > *"Databases with `version: production` support the new Time Travel API. Databases with `version: alpha` only support the older, snapshot-based backup API."* + > *"To understand which storage subsystem your database uses, run `wrangler d1 info YOUR_DATABASE` and inspect the `version` field."* + +2. **Wrangler Version Compatibility**: + Cloudflare D1 Time Travel requires Wrangler CLI version `>= v3.4.0` (per [Cloudflare D1 Time Travel documentation](https://developers.cloudflare.com/d1/reference/time-travel/)). In this repository, root `package.json` pins `"wrangler": "^4.112.0"`, satisfying the CLI version requirement. + +3. **Decision Branch (`version: "alpha"`) & Degraded Rollback State**: + If `wrangler d1 info ethercalc_rooms` returns `version: "alpha"`: + - **Time Travel is completely unavailable**: Time Travel commands (`wrangler d1 time-travel info`, `wrangler d1 time-travel restore`) will fail or operate under legacy snapshot behavior. + - **Rollback floor collapses**: Section §2's rollback floor collapses strictly to the manual `d1 export` SQL dump (`npx wrangler d1 export ethercalc_rooms --remote --output=...`), materially weakening rollback protection by eliminating continuous point-in-time recovery (PITR). + +4. **Operator Go/No-Go Judgment on `version: "alpha"` Blocker Status**: + - **Operator Judgment**: `version: "alpha"` **SHOULD block the cutover outright** if continuous D1 point-in-time recovery (PITR) or automated Time Travel restoration is required by the production operational SLA or change management policy. + - **Justification**: D1 `ethercalc_rooms` stores room index metadata (`rooms` table mirroring room IDs, title, snapshot, updated_at). If D1 schema migration (`0001_rooms_index.sql`) or database state corruption occurs during cutover, `version: "alpha"` leaves the operator with zero automated PITR options — reverting D1 requires manually dropping/recreating tables and importing a static SQL dump, which risks losing any room index updates created between the export time and cutover failure. If the operator or team explicitly chooses to proceed under `version: "alpha"` anyway (e.g., because primary cell state lives in `RoomDO` Durable Objects and D1 is a mirror index), this MUST be logged as a formal GO/NO-GO risk acceptance: the operator explicitly acknowledges that §2/§6 Time Travel rollback capabilities are disabled and that D1 recovery relies solely on `d1 export`. + ### 0.3 Baseline Decision Table `[OPERATOR-VERIFY]` | Observed Baseline State | Variance from `149ebcf16104b01254ca2b796beb701c88bd6ff8` | Required Action | @@ -114,7 +135,7 @@ curl -fsSI https://ethercalc.net/ | **Older Version** (< `149ebcf...`) | Production is behind `0.20260717.0`. | Pause upgrade. Run git diff between deployed version and `149ebcf...` to audit any missing intermediate state before deploying `main`. | | **Newer Version** (> `149ebcf...`) | Production is already ahead of `0.20260717.0`. | Run `git log 149ebcf..HEAD` to determine exactly which commits are deployed. Check if DO migration `v2` (`AuthDO`) is already present in `wrangler versions list`. | | **`v2` Migration Already Deployed** | `AuthDO` migration `v2` is shown as active in `wrangler deployments list`. | Skip Phase 1 (Phase 1 lifecycle deploy has already occurred). Proceed to Phase 2. | - +| **D1 Subsystem `version: "alpha"`** | Database uses legacy `alpha` storage subsystem; Time Travel API unsupported. | GO/NO-GO BLOCKER if continuous PITR is required by SLA. If proceeding by explicit sign-off, operator MUST NOT rely on Time Travel and MUST execute manual `d1 export` SQL backups only (§0.2.1, §2.1). | --- ## §1 Preflight on `main` @@ -203,7 +224,7 @@ Prior to cutover, record explicit backups for every durable store and acknowledg | Store | Capture Command / Procedure | Backup Artifact Location | Recovery Limitation / Reality | | :---------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------- | -| **D1 Database** (`ethercalc_rooms`) | `npx wrangler d1 export ethercalc_rooms --remote --output=./backup_ethercalc_rooms_$(date +%Y%m%d_%H%M%S).sql`

Also record current Time Travel bookmark:
`npx wrangler d1 time-travel info ethercalc_rooms` | Local SQL file `./backup_ethercalc_rooms_*.sql` and Cloudflare D1 Time Travel snapshot log. | Fully exportable and restorable via D1 SQL import or D1 Time Travel restore. | +| **D1 Database** (`ethercalc_rooms`) | `npx wrangler d1 export ethercalc_rooms --remote --output=./backup_ethercalc_rooms_$(date +%Y%m%d_%H%M%S).sql`

Also verify D1 subsystem `version: "production"` (§0.2.1) and record current Time Travel bookmark:
`npx wrangler d1 time-travel info ethercalc_rooms` | Local SQL file `./backup_ethercalc_rooms_*.sql` and Cloudflare D1 Time Travel snapshot log. | Fully exportable and restorable via D1 SQL import or D1 Time Travel restore.

**Command Syntax**: `--output` flag is **REQUIRED** for `wrangler d1 export` (per [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/#export)).

**Retention Caveat**: D1 Time Travel retention is plan-dependent per [Cloudflare D1 Time Travel](https://developers.cloudflare.com/d1/reference/time-travel/): *"up to 30 days in the past (Workers Paid plan) or 7 days (Workers Free plan)"*. Operator MUST confirm which plan the account is on before relying on a 30-day retention window.

**Subsystem Precondition**: Requires `version: "production"` (§0.2.1). If database is `version: "alpha"`, Time Travel is unsupported and recovery relies strictly on `d1 export` SQL dumps.

**CLI Compatibility**: Requires Wrangler `>= v3.4.0` (satisfied by root `package.json` pinning `"wrangler": "^4.112.0"`). | | **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (requires `ETHERCALC_MIGRATE_TOKEN`).
2. `RoomDO` SocialCalc snapshots are mirrored to D1 (`rooms` table `snapshot` column via `mirrorRoomToD1`, `packages/worker/src/lib/rooms-index.ts:14`). | Per-room D1 snapshot table (`rooms`) and in-DO PITR SQLite history buffers. | `AuthDO` credentials (passkeys, session revocation lists) and un-saved in-memory WebSocket operational state CANNOT be exported out of Cloudflare. | | **Static Assets** | Built from source code via `scripts/build-assets.ts`. | Source repository commits (`assets/` directory built during deploy). | 100% reproducible; re-deploys instantly from git source. | | **Secrets & Environment Variables** | Verified via `npx wrangler secret list --config=packages/worker/wrangler.toml --env=""`. | Cloudflare Workers Secret Manager. | Secret values CANNOT be retrieved/read back via CLI. Precondition checks in §0.1 ensure secrets are set prior to cutover. | @@ -722,6 +743,52 @@ Private room data exposure occurs **ONLY if code is rolled back past Phase 2 to > **The moment `ETHERCALC_AUTH = "1"` is activated in Phase 3 and the first user creates a private room (`POST /_/private`) OR completes a passkey registration (`POST /_auth/register-complete`).** - Past this point, rolling Phase 3 back to Phase 2 causes private room owner lockout (resolved immediately by re-enabling Phase 3), while rolling back to Phase 1 / Forward-Fix bundle causes world-readability unless WAF URL rules block affected room paths. +### 6.4 D1 Database Rollback & Time Travel Procedure `[OPERATOR-VERIFY]` + +If D1 database state must be restored to a pre-cutover state, use Cloudflare D1 Time Travel or manual SQL export restore per [Cloudflare D1 Time Travel documentation](https://developers.cloudflare.com/d1/reference/time-travel/) and [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/): + +#### 1. Destructive Nature & Operational Interruption +- **In-Place Overwrite**: Per Cloudflare docs: *"Restoring a database to a specific point-in-time is a destructive operation, and overwrites the database in place."* +- **Query Interruption**: *"Queries in flight will be cancelled, and an error returned to the client."* Operators should perform D1 restores during low-traffic windows or communicate potential transient database errors to active users. + +#### 2. Mandatory Undo Bookmark Recording +- When `wrangler d1 time-travel restore` completes, the CLI prints an undo bookmark: + ```text + ↩️ To undo this operation, you can restore to the previous bookmark: + ``` +- **MANDATORY OPERATOR ACTION**: The operator **MUST record and log this undo bookmark immediately** in cutover logs before proceeding. Capturing this bookmark is essential to reverse an accidental or incorrect restore operation forward. + +#### 3. Timestamp Fallback & Deterministic Conversion +- If the pre-cutover bookmark was **never recorded or lost**, a forgotten bookmark is recoverable using: + ```bash + npx wrangler d1 time-travel info ethercalc_rooms --timestamp="2026-08-10T12:00:00Z" + ``` + Per Cloudflare docs: *"conversion between a specific timestamp and a bookmark is deterministic (stable)"*. +- Furthermore, `wrangler d1 time-travel restore` accepts `--timestamp` directly, eliminating the need to look up the bookmark beforehand: + ```bash + npx wrangler d1 time-travel restore ethercalc_rooms --timestamp="2026-08-10T12:00:00Z" + ``` + +#### 4. Execution Commands (Consistent with §2 Capture) + +- **Primary Restore (by Pre-Cutover Bookmark)**: + ```bash + npx wrangler d1 time-travel restore ethercalc_rooms --bookmark= + ``` +- **Fallback Restore (by Timestamp, if Bookmark Missing)**: + ```bash + npx wrangler d1 time-travel restore ethercalc_rooms --timestamp="2026-08-10T12:00:00Z" + ``` +- **Undo Restore (Reverting to State Prior to Restoration)**: + ```bash + npx wrangler d1 time-travel restore ethercalc_rooms --bookmark= + ``` +- **SQL Dump Fallback Restore (If `version: "alpha"` or Time Travel Unavailable)**: + ```bash + npx wrangler d1 execute ethercalc_rooms --remote --file=./backup_ethercalc_rooms_.sql + ``` + *(Note: `wrangler d1 export` requires `--output=`, whereas `wrangler d1 execute` requires `--file=` per [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/).)* + --- ## §7 Self-Host Upgrade Path (Docker / Helm) @@ -879,10 +946,10 @@ The following five items have been evaluated and categorized: This checklist has nine conditions and spans the full cutover. Before executing Phase 1 (`npx wrangler deploy --env=""`), items 1–7 MUST be satisfied; items 5 and 6 are already complete and therefore intentionally checked below. Item 8 gates the Phase 2 upload, and item 9 gates Phase 3. -- [ ] **1. Baseline Capture Verified**: `wrangler deployments list`, `wrangler versions list`, and `wrangler d1 info ethercalc_rooms` executed and recorded (§0.2). +- [ ] **1. Baseline Capture & Subsystem Verification**: `wrangler deployments list`, `wrangler versions list`, and `wrangler d1 info ethercalc_rooms --json` executed and recorded, confirming D1 storage subsystem `version: "production"` for Time Travel availability (§0.2, §0.2.1). - [ ] **2. Preflight Gates Green Against Final Tree (9/9 Runnable Gates Verified; 2 Docker Smokes Pending CI)**: All 9 locally-runnable preflight gates (`vp run typecheck`, `vp lint`, `vp run test`, worker `test:node` & `test:workers`, worker 100% coverage gate `test:coverage`, `build:assets` + `e2e#test`, `build:dry`, `check-helm-hardening.sh`, and `ratchet-verify.sh`) passed 100% green against the final tree state including the `rooms.ts` command-rejection status propagation fix (§1.2, `docs/migration/PREFLIGHT_RESULTS.md`). The 2 Docker smoke gates (`./scripts/smoke-selfhost.sh`, `./scripts/smoke-proxy.sh`) remain unverified locally due to missing local `docker compose` CLI subcommand and require CI execution before final cutover. - [ ] **3. Secrets Provisioned in Production**: `wrangler secret list` confirms `ETHERCALC_KEY` and `ETHERCALC_MIGRATE_TOKEN` are active in Cloudflare Secrets (§0.1). -- [ ] **4. D1 Database Export & Time Travel Bookmark Recorded**: `npx wrangler d1 export ethercalc_rooms --remote` executed and Time Travel bookmark timestamp captured (§2.1). +- [ ] **4. D1 Database Export & Time Travel Bookmark Recorded**: `npx wrangler d1 export ethercalc_rooms --remote --output=...` executed (using required `--output` flag), account plan confirmed (30d Paid / 7d Free retention), and Time Travel bookmark timestamp captured (§2.1, §6.4). - [x] **5. Phase 1 Branch (Forward-Fix Artifact) Prepared**: `release/phase1-lifecycle` branch built, typechecks, and dry-runs cleanly (§4.2, §6.1, and §8 item 1; verified via `vp run @ethercalc/worker#typecheck` and `vp run @ethercalc/worker#build:dry`). - [x] **6. PR 4 Command-Rejection Propagation Landed and Verified for Phase 2**: `POST /_/:room` returns the RoomDO status and body for every non-2xx verdict, matching `PUT /_/:room` at `packages/worker/src/routes/rooms.ts:355-369`. Verified via route contract tests `POST /_/:room command mutations propagate a DO 413 sheet-limit verdict` and `POST /_/:room returns 202 command echo on successful DO dispatch` in `packages/worker/test/routes-rooms.node.test.ts` (52 node files / 1520 tests; 13 workers-pool files / 196 tests; §8 item 5; §10 item 3). - [ ] **7. Staging Rehearsal Passed**: Phase 1, Phase 2, and Phase 3 deployed and verified on `https://ethercalc-staging.audreyt.workers.dev` (§§3.1–3.2). From 868ee0eb80a6373d5b09fe6e88126161f09030df Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:08:39 +0800 Subject: [PATCH 06/95] docs(migration): correct D1 rooms index scope and DO non-exportability in upgrade plan --- docs/migration/PROD_UPGRADE_PLAN.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 4d2948e9..324d8bf2 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -125,7 +125,7 @@ Before proceeding with cutover or relying on Time Travel backups, the operator M 4. **Operator Go/No-Go Judgment on `version: "alpha"` Blocker Status**: - **Operator Judgment**: `version: "alpha"` **SHOULD block the cutover outright** if continuous D1 point-in-time recovery (PITR) or automated Time Travel restoration is required by the production operational SLA or change management policy. - - **Justification**: D1 `ethercalc_rooms` stores room index metadata (`rooms` table mirroring room IDs, title, snapshot, updated_at). If D1 schema migration (`0001_rooms_index.sql`) or database state corruption occurs during cutover, `version: "alpha"` leaves the operator with zero automated PITR options — reverting D1 requires manually dropping/recreating tables and importing a static SQL dump, which risks losing any room index updates created between the export time and cutover failure. If the operator or team explicitly chooses to proceed under `version: "alpha"` anyway (e.g., because primary cell state lives in `RoomDO` Durable Objects and D1 is a mirror index), this MUST be logged as a formal GO/NO-GO risk acceptance: the operator explicitly acknowledges that §2/§6 Time Travel rollback capabilities are disabled and that D1 recovery relies solely on `d1 export`. + - **Justification**: D1 `ethercalc_rooms` stores room index metadata (`rooms` table mirroring room names, updated_at timestamps, and cors_public flags per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`; sheet content and snapshots are NOT stored in D1). If D1 schema migration (`packages/worker/migrations/0001_rooms.sql`) or database state corruption occurs during cutover, `version: "alpha"` leaves the operator with zero automated PITR options — reverting D1 requires manually dropping/recreating tables and importing a static SQL dump, which risks losing any room index updates created between the export time and cutover failure. If the operator or team explicitly chooses to proceed under `version: "alpha"` anyway (e.g., because primary cell state lives in `RoomDO` Durable Objects and D1 is a mirror index), this MUST be logged as a formal GO/NO-GO risk acceptance: the operator explicitly acknowledges that §2/§6 Time Travel rollback capabilities are disabled and that D1 recovery relies solely on `d1 export`. ### 0.3 Baseline Decision Table `[OPERATOR-VERIFY]` @@ -225,10 +225,30 @@ Prior to cutover, record explicit backups for every durable store and acknowledg | Store | Capture Command / Procedure | Backup Artifact Location | Recovery Limitation / Reality | | :---------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------- | | **D1 Database** (`ethercalc_rooms`) | `npx wrangler d1 export ethercalc_rooms --remote --output=./backup_ethercalc_rooms_$(date +%Y%m%d_%H%M%S).sql`

Also verify D1 subsystem `version: "production"` (§0.2.1) and record current Time Travel bookmark:
`npx wrangler d1 time-travel info ethercalc_rooms` | Local SQL file `./backup_ethercalc_rooms_*.sql` and Cloudflare D1 Time Travel snapshot log. | Fully exportable and restorable via D1 SQL import or D1 Time Travel restore.

**Command Syntax**: `--output` flag is **REQUIRED** for `wrangler d1 export` (per [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/#export)).

**Retention Caveat**: D1 Time Travel retention is plan-dependent per [Cloudflare D1 Time Travel](https://developers.cloudflare.com/d1/reference/time-travel/): *"up to 30 days in the past (Workers Paid plan) or 7 days (Workers Free plan)"*. Operator MUST confirm which plan the account is on before relying on a 30-day retention window.

**Subsystem Precondition**: Requires `version: "production"` (§0.2.1). If database is `version: "alpha"`, Time Travel is unsupported and recovery relies strictly on `d1 export` SQL dumps.

**CLI Compatibility**: Requires Wrangler `>= v3.4.0` (satisfied by root `package.json` pinning `"wrangler": "^4.112.0"`). | -| **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (requires `ETHERCALC_MIGRATE_TOKEN`).
2. `RoomDO` SocialCalc snapshots are mirrored to D1 (`rooms` table `snapshot` column via `mirrorRoomToD1`, `packages/worker/src/lib/rooms-index.ts:14`). | Per-room D1 snapshot table (`rooms`) and in-DO PITR SQLite history buffers. | `AuthDO` credentials (passkeys, session revocation lists) and un-saved in-memory WebSocket operational state CANNOT be exported out of Cloudflare. | +| **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**D1 Mirror Scope & Reality**: D1 `ethercalc_rooms` holds **ONLY** the cross-room index (`room`, `updated_at`, `cors_public`) per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`. **NO SocialCalc sheet snapshots, command logs, cell values, chat messages, or audit history are stored in D1.**

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (requires `ETHERCALC_MIGRATE_TOKEN`, §0.1).
2. Cloudflare platform-level Durable Object storage replication and durability. | In-DO SQLite storage (`state.storage`) and per-room PITR history buffers. *(D1 export recovers ONLY room names and timestamps, NOT sheet content).* | **NO OPERATOR BULK BACKUP**: RoomDO sheet content (snapshots, logs, cells) cannot be exported in bulk. A `wrangler d1 export` backs up room index metadata (`room`, `updated_at`, `cors_public`), NOT sheet cell state. `AuthDO` credentials (passkeys, session revocation lists) and un-saved in-memory WebSocket operational state also CANNOT be exported. Recovery of sheet content relies entirely on per-room PITR or Cloudflare platform storage durability. | | **Static Assets** | Built from source code via `scripts/build-assets.ts`. | Source repository commits (`assets/` directory built during deploy). | 100% reproducible; re-deploys instantly from git source. | | **Secrets & Environment Variables** | Verified via `npx wrangler secret list --config=packages/worker/wrangler.toml --env=""`. | Cloudflare Workers Secret Manager. | Secret values CANNOT be retrieved/read back via CLI. Precondition checks in §0.1 ensure secrets are set prior to cutover. | +### 2.2 Rollback Floor Definition & DO Content Non-Exportability + +Because Cloudflare Workers provides no operator-side bulk export API for Durable Object SQLite storage, the operator MUST understand what each backup artifact covers: + +1. **What `wrangler d1 export` Backs Up**: + - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`). + - Allows restoring the list of room names and their last-modified timestamps. + - **DOES NOT** back up or restore sheet cell data, SocialCalc command logs, snapshots, chat history, or audit logs. + +2. **What Cloudflare D1 Time Travel Restores**: + - Restores the `rooms` index table to a prior point-in-time (§6.4). + - **DOES NOT** roll back or restore Durable Object SQLite storage (`RoomDO` / `AuthDO`). + +3. **Implications for Cutover Strategy & Phase Ordering**: + - Because DO sheet content cannot be bulk-exported, **preventing data corruption during cutover is paramount**. + - This non-exportability strongly reinforces the mandatory **Three-Phase Upgrade Strategy** (§4): + - **Phase 1** applies the `v2` migration (`AuthDO` class addition) in a behaviorally inert bundle (`149ebcf` code). + - **Phase 2** deploys `main` with `ETHERCALC_AUTH = "0"`. Setting `ETHERCALC_AUTH = "0"` **structurally guarantees** that no passkey registration or private room creation can occur while major code changes soak. If Phase 2 encounters bugs, rolling Phase 2 back to Phase 1 carries **zero risk** of private room data exposure or lockout. + - **Phase 3** enables `ETHERCALC_AUTH = "1"` only after Phase 2 has soaked and proven stable. + --- ## §3 Staging Rehearsal & Phase 1 Rehearsal @@ -747,6 +767,8 @@ Private room data exposure occurs **ONLY if code is rolled back past Phase 2 to If D1 database state must be restored to a pre-cutover state, use Cloudflare D1 Time Travel or manual SQL export restore per [Cloudflare D1 Time Travel documentation](https://developers.cloudflare.com/d1/reference/time-travel/) and [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/): + +> **CRITICAL OPERATOR NOTICE**: Restoring D1 via Time Travel or SQL dump restores ONLY the cross-room index (`rooms` table containing room names, updated_at timestamps, and cors_public flags per `packages/worker/migrations/0001_rooms.sql:17-21`). It **DOES NOT** roll back or restore Durable Object SQLite storage (`RoomDO` sheet cell data, SocialCalc snapshots, or command logs). Because DO sheet content has no operator bulk export API, cutover safety relies on the three-phase rollout sequence (§4) and per-room PITR recovery. #### 1. Destructive Nature & Operational Interruption - **In-Place Overwrite**: Per Cloudflare docs: *"Restoring a database to a specific point-in-time is a destructive operation, and overwrites the database in place."* - **Query Interruption**: *"Queries in flight will be cancelled, and an error returned to the client."* Operators should perform D1 restores during low-traffic windows or communicate potential transient database errors to active users. From 5047531f78b67244734f493dd0963f6eb4b6e550 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:10:23 +0800 Subject: [PATCH 07/95] docs(migration): clarify D1 log tails versus DO sheet state --- docs/migration/PROD_UPGRADE_PLAN.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 324d8bf2..9b896ae8 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -125,7 +125,7 @@ Before proceeding with cutover or relying on Time Travel backups, the operator M 4. **Operator Go/No-Go Judgment on `version: "alpha"` Blocker Status**: - **Operator Judgment**: `version: "alpha"` **SHOULD block the cutover outright** if continuous D1 point-in-time recovery (PITR) or automated Time Travel restoration is required by the production operational SLA or change management policy. - - **Justification**: D1 `ethercalc_rooms` stores room index metadata (`rooms` table mirroring room names, updated_at timestamps, and cors_public flags per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`; sheet content and snapshots are NOT stored in D1). If D1 schema migration (`packages/worker/migrations/0001_rooms.sql`) or database state corruption occurs during cutover, `version: "alpha"` leaves the operator with zero automated PITR options — reverting D1 requires manually dropping/recreating tables and importing a static SQL dump, which risks losing any room index updates created between the export time and cutover failure. If the operator or team explicitly chooses to proceed under `version: "alpha"` anyway (e.g., because primary cell state lives in `RoomDO` Durable Objects and D1 is a mirror index), this MUST be logged as a formal GO/NO-GO risk acceptance: the operator explicitly acknowledges that §2/§6 Time Travel rollback capabilities are disabled and that D1 recovery relies solely on `d1 export`. + - **Justification**: D1 `ethercalc_rooms` stores the room index (`rooms` table mirroring room names, updated_at timestamps, and cors_public flags per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`) plus bounded per-room audit and chat log tails (`audit_log` and `chat_log` tables per `packages/worker/migrations/0003_audit_chat.sql:22-38` and `packages/worker/src/lib/seq-store.ts:39-41`). Authoritative sheet content, cell values, SocialCalc command logs, and snapshots are stored solely inside Durable Objects (`RoomDO`). If D1 schema migration (`packages/worker/migrations/0001_rooms.sql`) or database state corruption occurs during cutover, `version: "alpha"` leaves the operator with zero automated PITR options — reverting D1 requires manually dropping/recreating tables and importing a static SQL dump, which risks losing any room index or log updates created between the export time and cutover failure. If the operator or team explicitly chooses to proceed under `version: "alpha"` anyway (e.g., because primary cell state lives in `RoomDO` Durable Objects and D1 is a mirror index/log store), this MUST be logged as a formal GO/NO-GO risk acceptance: the operator explicitly acknowledges that §2/§6 Time Travel rollback capabilities are disabled and that D1 recovery relies solely on `d1 export`. ### 0.3 Baseline Decision Table `[OPERATOR-VERIFY]` @@ -225,7 +225,7 @@ Prior to cutover, record explicit backups for every durable store and acknowledg | Store | Capture Command / Procedure | Backup Artifact Location | Recovery Limitation / Reality | | :---------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------- | | **D1 Database** (`ethercalc_rooms`) | `npx wrangler d1 export ethercalc_rooms --remote --output=./backup_ethercalc_rooms_$(date +%Y%m%d_%H%M%S).sql`

Also verify D1 subsystem `version: "production"` (§0.2.1) and record current Time Travel bookmark:
`npx wrangler d1 time-travel info ethercalc_rooms` | Local SQL file `./backup_ethercalc_rooms_*.sql` and Cloudflare D1 Time Travel snapshot log. | Fully exportable and restorable via D1 SQL import or D1 Time Travel restore.

**Command Syntax**: `--output` flag is **REQUIRED** for `wrangler d1 export` (per [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/#export)).

**Retention Caveat**: D1 Time Travel retention is plan-dependent per [Cloudflare D1 Time Travel](https://developers.cloudflare.com/d1/reference/time-travel/): *"up to 30 days in the past (Workers Paid plan) or 7 days (Workers Free plan)"*. Operator MUST confirm which plan the account is on before relying on a 30-day retention window.

**Subsystem Precondition**: Requires `version: "production"` (§0.2.1). If database is `version: "alpha"`, Time Travel is unsupported and recovery relies strictly on `d1 export` SQL dumps.

**CLI Compatibility**: Requires Wrangler `>= v3.4.0` (satisfied by root `package.json` pinning `"wrangler": "^4.112.0"`). | -| **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**D1 Mirror Scope & Reality**: D1 `ethercalc_rooms` holds **ONLY** the cross-room index (`room`, `updated_at`, `cors_public`) per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`. **NO SocialCalc sheet snapshots, command logs, cell values, chat messages, or audit history are stored in D1.**

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (requires `ETHERCALC_MIGRATE_TOKEN`, §0.1).
2. Cloudflare platform-level Durable Object storage replication and durability. | In-DO SQLite storage (`state.storage`) and per-room PITR history buffers. *(D1 export recovers ONLY room names and timestamps, NOT sheet content).* | **NO OPERATOR BULK BACKUP**: RoomDO sheet content (snapshots, logs, cells) cannot be exported in bulk. A `wrangler d1 export` backs up room index metadata (`room`, `updated_at`, `cors_public`), NOT sheet cell state. `AuthDO` credentials (passkeys, session revocation lists) and un-saved in-memory WebSocket operational state also CANNOT be exported. Recovery of sheet content relies entirely on per-room PITR or Cloudflare platform storage durability. | +| **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**D1 Mirror Scope & Reality**: D1 `ethercalc_rooms` holds the cross-room index (`room`, `updated_at`, `cors_public` per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`) plus bounded per-room audit and chat log tails (`audit_log` and `chat_log` per `packages/worker/migrations/0003_audit_chat.sql:22-38` and `packages/worker/src/lib/seq-store.ts:39-41`). **NO authoritative SocialCalc sheet snapshots, cell state, or full command logs are stored in D1.**

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (requires `ETHERCALC_MIGRATE_TOKEN`, §0.1).
2. Cloudflare platform-level Durable Object storage replication and durability. | In-DO SQLite storage (`state.storage`) and per-room PITR history buffers. *(D1 export recovers room index metadata and bounded audit/chat log tails, NOT sheet cell state or snapshots).* | **NO OPERATOR BULK BACKUP**: RoomDO sheet content (snapshots, full command logs, cell values) cannot be exported in bulk. A `wrangler d1 export` backs up room index metadata and audit/chat log tails, NOT sheet cell state or snapshots. `AuthDO` credentials (passkeys, session revocation lists) and un-saved in-memory WebSocket operational state also CANNOT be exported out of Cloudflare. Recovery of sheet content relies entirely on per-room PITR or Cloudflare platform storage durability. | | **Static Assets** | Built from source code via `scripts/build-assets.ts`. | Source repository commits (`assets/` directory built during deploy). | 100% reproducible; re-deploys instantly from git source. | | **Secrets & Environment Variables** | Verified via `npx wrangler secret list --config=packages/worker/wrangler.toml --env=""`. | Cloudflare Workers Secret Manager. | Secret values CANNOT be retrieved/read back via CLI. Precondition checks in §0.1 ensure secrets are set prior to cutover. | @@ -234,12 +234,12 @@ Prior to cutover, record explicit backups for every durable store and acknowledg Because Cloudflare Workers provides no operator-side bulk export API for Durable Object SQLite storage, the operator MUST understand what each backup artifact covers: 1. **What `wrangler d1 export` Backs Up**: - - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`). - - Allows restoring the list of room names and their last-modified timestamps. - - **DOES NOT** back up or restore sheet cell data, SocialCalc command logs, snapshots, chat history, or audit logs. + - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`), `audit_log`, `chat_log`, and `cron` tables. + - Allows restoring the list of room names, last-modified timestamps, and bounded audit/chat log tails. + - **DOES NOT** back up or restore sheet cell data, authoritative SocialCalc command logs, or sheet snapshots. A D1 export cannot reconstruct RoomDO cell state. 2. **What Cloudflare D1 Time Travel Restores**: - - Restores the `rooms` index table to a prior point-in-time (§6.4). + - Restores D1 tables (`rooms`, `audit_log`, `chat_log`, `cron`) to a prior point-in-time (§6.4). - **DOES NOT** roll back or restore Durable Object SQLite storage (`RoomDO` / `AuthDO`). 3. **Implications for Cutover Strategy & Phase Ordering**: @@ -768,7 +768,8 @@ Private room data exposure occurs **ONLY if code is rolled back past Phase 2 to If D1 database state must be restored to a pre-cutover state, use Cloudflare D1 Time Travel or manual SQL export restore per [Cloudflare D1 Time Travel documentation](https://developers.cloudflare.com/d1/reference/time-travel/) and [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/): -> **CRITICAL OPERATOR NOTICE**: Restoring D1 via Time Travel or SQL dump restores ONLY the cross-room index (`rooms` table containing room names, updated_at timestamps, and cors_public flags per `packages/worker/migrations/0001_rooms.sql:17-21`). It **DOES NOT** roll back or restore Durable Object SQLite storage (`RoomDO` sheet cell data, SocialCalc snapshots, or command logs). Because DO sheet content has no operator bulk export API, cutover safety relies on the three-phase rollout sequence (§4) and per-room PITR recovery. +> **CRITICAL OPERATOR NOTICE**: Restoring D1 via Time Travel or SQL dump restores D1 tables — the cross-room index plus bounded audit/chat tails — but it **DOES NOT** roll back or restore Durable Object SQLite storage (`RoomDO` sheet cell data, authoritative SocialCalc snapshots, or command logs). The index schema is `packages/worker/migrations/0001_rooms.sql:17-21`; D1 audit/chat tables and retention bounds are documented in `packages/worker/migrations/0003_audit_chat.sql:22-38` and `packages/worker/src/lib/seq-store.ts:39-41`. Because DO sheet content has no operator bulk export API, cutover safety relies on the three-phase rollout sequence (§4) and per-room PITR recovery. + #### 1. Destructive Nature & Operational Interruption - **In-Place Overwrite**: Per Cloudflare docs: *"Restoring a database to a specific point-in-time is a destructive operation, and overwrites the database in place."* - **Query Interruption**: *"Queries in flight will be cancelled, and an error returned to the client."* Operators should perform D1 restores during low-traffic windows or communicate potential transient database errors to active users. From 989db0abc396924154985798cdeb33ae3ded8dbd Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:11:06 +0800 Subject: [PATCH 08/95] docs(migration): correct D1 cron table name --- docs/migration/PROD_UPGRADE_PLAN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 9b896ae8..a754437c 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -234,12 +234,12 @@ Prior to cutover, record explicit backups for every durable store and acknowledg Because Cloudflare Workers provides no operator-side bulk export API for Durable Object SQLite storage, the operator MUST understand what each backup artifact covers: 1. **What `wrangler d1 export` Backs Up**: - - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`), `audit_log`, `chat_log`, and `cron` tables. + - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`), `audit_log`, `chat_log`, and `cron_triggers` tables. - Allows restoring the list of room names, last-modified timestamps, and bounded audit/chat log tails. - **DOES NOT** back up or restore sheet cell data, authoritative SocialCalc command logs, or sheet snapshots. A D1 export cannot reconstruct RoomDO cell state. 2. **What Cloudflare D1 Time Travel Restores**: - - Restores D1 tables (`rooms`, `audit_log`, `chat_log`, `cron`) to a prior point-in-time (§6.4). + - Restores D1 tables (`rooms`, `audit_log`, `chat_log`, `cron_triggers`) to a prior point-in-time (§6.4). - **DOES NOT** roll back or restore Durable Object SQLite storage (`RoomDO` / `AuthDO`). 3. **Implications for Cutover Strategy & Phase Ordering**: From b42ad533a188690ab7a3d03248099b28b1237ad7 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:18:51 +0800 Subject: [PATCH 09/95] docs: document Durable Object mass PITR --- docs/migration/PROD_UPGRADE_PLAN.md | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index a754437c..4ef132fd 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -249,6 +249,65 @@ Because Cloudflare Workers provides no operator-side bulk export API for Durable - **Phase 2** deploys `main` with `ETHERCALC_AUTH = "0"`. Setting `ETHERCALC_AUTH = "0"` **structurally guarantees** that no passkey registration or private room creation can occur while major code changes soak. If Phase 2 encounters bugs, rolling Phase 2 back to Phase 1 carries **zero risk** of private room data exposure or lockout. - **Phase 3** enables `ETHERCALC_AUTH = "1"` only after Phase 2 has soaked and proven stable. +### 2.3 Durable Object PITR Contract and Eligibility + +The recovery floor for Durable Object state is real only because both deployed classes use the correct storage backend. Cloudflare's [SQLite-backed Durable Object Storage documentation](https://developers.cloudflare.com/durable-objects/api/sqlite-storage-api/) lists `PITR API` as available for SQLite-backed classes and unavailable for KV-backed classes. EtherCalc declares `RoomDO` in migration `v1` and `AuthDO` in migration `v2` under `new_sqlite_classes` (`packages/worker/wrangler.toml:41-47`), so both classes meet that prerequisite. + +The same Cloudflare contract says that these methods can “restore a Durable Object's embedded SQLite database to any point in time in the past 30 days” and that they “apply to the entire SQLite database contents, including both the object's stored SQL data and stored key-value data using the key-value `put()` API.” This second guarantee is essential for EtherCalc: `RoomDO` persists snapshots, log/audit/chat entries, cell metadata, and room metadata through `this.#state.storage.put(...)` or transaction `put(...)` calls (for example `packages/worker/src/room.ts:619,1219,1235-1240,2125-2131,2334,2388`), rather than application-defined SQL tables. Those values are nevertheless inside the SQLite-backed object's PITR scope. + +Cloudflare exposes three bookmark primitives: + +- `getCurrentBookmark()` returns the object's current history bookmark. +- `getBookmarkForTime(timestamp)` resolves an approximate bookmark for a timestamp within the 30-day window. +- `onNextSessionRestoreBookmark(bookmark)` configures the object so that, on its next restart, its storage matches that bookmark. Cloudflare states: “After calling this, the application should typically invoke `ctx.abort()` to restart the Durable Object, thus completing the point-in-time recovery.” The method returns a bookmark for the moment immediately before recovery, so submitting that bookmark in a second restore undoes the first restore. + +EtherCalc wraps the latter two primitives in the bearer-gated `POST /_/:room/pitr-restore` contract documented in `API.md:133-197`; the response includes `undoBookmark`, which the operator MUST retain before proceeding to the next room. + +Cloudflare also states: “The PITR API is not supported in local development because a durable log of data changes is not stored locally.” This agrees with `API.md:137`: Miniflare and standalone workerd return `501`. Therefore **the PITR safeguard cannot be rehearsed locally**. The operator MUST exercise the dry-run, restore, verification, and undo sequence against a real Cloudflare staging deployment before relying on it in production. + +PITR is scoped to one Durable Object instance, not a namespace. EtherCalc has one `RoomDO` per room (`packages/worker/wrangler.toml:28-33`), so there is no “restore all rooms” platform operation. `AuthDO` is different only in cardinality: the application addresses the singleton as `idFromName('auth')` (`packages/worker/src/routes/auth.ts:52-56`), so AuthDO recovery is a single-object operation rather than a room iteration. Although `AuthDO` is PITR-eligible, the checked-in operator endpoint currently dispatches only to `RoomDO`; the repository has no AuthDO restore command. + +### 2.4 Mass RoomDO Recovery Procedure + +D1 and Durable Object PITR supply complementary halves of broad recovery: the D1 `rooms` table supplies indexed room names and their last-write times, while per-room PITR restores the content of each corresponding `RoomDO`. `GET /_rooms` and `GET /_roomtimes` expose the same index, but a direct D1 query is safer for an operator procedure because it returns both fields in one captured result. + +1. **Freeze the recovery inputs.** Record the incident start and end in UTC and select one restore timestamp immediately before the first bad write. It MUST be within Cloudflare's 30-day Durable Object PITR window. Do not derive a different timestamp for each room unless the incident analysis explicitly requires it. +2. **Capture the candidate list before restoring anything.** `updated_at` is a JavaScript millisecond epoch (`Date.now()`), so query with millisecond boundaries. Replace the example values with the incident window and save the complete output: + + ```bash + npx wrangler d1 execute ethercalc_rooms \ + --remote \ + --config=packages/worker/wrangler.toml \ + --env="" \ + --command="SELECT room, updated_at FROM rooms WHERE updated_at BETWEEN 1786363200000 AND 1786366800000 ORDER BY updated_at ASC;" + ``` + + The equivalent HTTP sources are `GET /_rooms` for names and `GET /_roomtimes` for timestamps. Prefer the D1 query when filtering an incident window. +3. **Dry-run every candidate sequentially.** For each D1 row, URL-encode the room as one path segment and resolve the common pre-incident timestamp without scheduling a restore: + + ```bash + curl -sS --fail-with-body -w '\nHTTP %{http_code}\n' -X POST "https://ethercalc.net/_//pitr-restore" \ + -H "Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"at":"2026-08-10T11:59:59.000Z","dryRun":true}' + ``` + + Record the room, D1 `updated_at`, requested timestamp, resolved `bookmark`, HTTP status, and response. Stop on any non-200 response; do not silently skip a room. +4. **Apply and verify one room at a time.** Submit the dry-run bookmark, retain the returned `undoBookmark`, then verify the room's snapshot/content before advancing: + + ```bash + curl -sS --fail-with-body -w '\nHTTP %{http_code}\n' -X POST "https://ethercalc.net/_//pitr-restore" \ + -H "Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"bookmark":""}' + ``` + + A successful response reports `restored: true`; `exists: false` means the target predates creation of that room. If verification fails, submit the recorded `undoBookmark` as `bookmark` before investigating. Preserve the full candidate list and per-room responses as the recovery audit record. + +This is intentionally a **sequential, per-room** procedure. Its throughput and request rate are limited by the delay, retry, and concurrency controls the operator puts into an incident script; EtherCalc supplies none. PITR rewinds the entire RoomDO SQLite database to the selected timestamp, including later key-value state, so it cannot selectively remove one bad SocialCalc command while retaining later good commands. + +**Enumeration limitations and tooling gap:** No bulk-PITR script or CLI exists under `packages/cli/`, `scripts/`, or `bin/`; only the single-room HTTP endpoint exists. An operator facing a broad incident must write and peer-review the enumeration/restore driver under pressure unless one is prepared in advance. In addition, the D1 `rooms` table is not a complete namespace inventory: private rooms are deliberately omitted (`packages/worker/src/room.ts:2266-2272`), and `_formdata` sibling rooms are filtered from the public index (`packages/worker/src/lib/formdata-sibling.ts:13-19`). The D1-driven procedure therefore covers indexed rooms only. Known private or form-data room names require a separately maintained inventory and the same per-room restore call; Cloudflare provides no API to enumerate every Durable Object instance. + --- ## §3 Staging Rehearsal & Phase 1 Rehearsal From 5caeae349fb58f59198039b9336814d8575413af Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:22:13 +0800 Subject: [PATCH 10/95] docs: escalate private room non-enumerability in PITR runbook --- docs/migration/PROD_UPGRADE_PLAN.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 4ef132fd..9b745d11 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -306,7 +306,14 @@ D1 and Durable Object PITR supply complementary halves of broad recovery: the D1 This is intentionally a **sequential, per-room** procedure. Its throughput and request rate are limited by the delay, retry, and concurrency controls the operator puts into an incident script; EtherCalc supplies none. PITR rewinds the entire RoomDO SQLite database to the selected timestamp, including later key-value state, so it cannot selectively remove one bad SocialCalc command while retaining later good commands. -**Enumeration limitations and tooling gap:** No bulk-PITR script or CLI exists under `packages/cli/`, `scripts/`, or `bin/`; only the single-room HTTP endpoint exists. An operator facing a broad incident must write and peer-review the enumeration/restore driver under pressure unless one is prepared in advance. In addition, the D1 `rooms` table is not a complete namespace inventory: private rooms are deliberately omitted (`packages/worker/src/room.ts:2266-2272`), and `_formdata` sibling rooms are filtered from the public index (`packages/worker/src/lib/formdata-sibling.ts:13-19`). The D1-driven procedure therefore covers indexed rooms only. Known private or form-data room names require a separately maintained inventory and the same per-room restore call; Cloudflare provides no API to enumerate every Durable Object instance. +**Headline Enumeration Gap & Operational Severity:** The D1 `rooms` index is **NOT** a complete namespace inventory: + +1. **Private Room Write-Time Exclusion**: Tracing `POST /_/private` -> `/_do/init-private` (`packages/worker/src/room.ts:1310-1386`) shows that private room creation sets `STORAGE_KEYS.metaAccess = 'private'` directly in DO storage without calling `#mirrorIndex`. Furthermore, every subsequent write routes through `RoomDO.#mirrorIndex` (`packages/worker/src/room.ts:2266-2273`), which explicitly checks `const { access } = await this.#getAccessMeta(); if (access === 'private') return;` and returns before calling `mirrorRoomToD1`. In addition, `#postTouch` (`packages/worker/src/room.ts:620-622`) checks `access === 'private'` and calls `#deleteIndex` (`DELETE FROM rooms WHERE room = ?1`). Private rooms are **structurally excluded from D1 at write time**. They do NOT exist in the `rooms` table, nor in `GET /_rooms` or `GET /_roomtimes`. +2. **Form-Data Sibling Exclusion**: Rooms ending in `_formdata` (internal submitform storage) are filtered out of public listings by `isPublicRoomIndexEntry` (`packages/worker/src/lib/formdata-sibling.ts:18-19`). + +**Consequence for Incident Recovery**: The D1-driven mass recovery procedure above discovers and restores **only public indexed rooms**. Content in private rooms (which hold sensitive user-authenticated data) has **no platform enumeration source in D1**. If a broad incident corrupts private rooms, D1 query iteration will NOT find them. Restoring private rooms requires an out-of-band inventory of private room names (e.g. from edge access logs or application audit trails) and submitting per-room `POST /_/:room/pitr-restore` calls for each known private room name. Cloudflare provides no namespace-wide API to iterate Durable Object instances. + +**Tooling Gap Summary**: No bulk-PITR script or CLI exists under `packages/cli/`, `scripts/`, or `bin/`; only the single-room HTTP endpoint exists. An operator facing a broad incident must write and peer-review the enumeration/restore driver under pressure unless one is prepared in advance. --- @@ -1022,6 +1029,10 @@ The following five items have been evaluated and categorized: - **Cutover Blocker Decision**: **LANDED & VERIFIED (BLOCKING PRE-CUTOVER FIX)**. The rejection is a no-op rather than data corruption, which limits the damage, but this upgrade changes an over-limit request from an applied write with a truthful 202 into a rejected write with a false 202. API clients have no error signal and may record work as accepted. The correction is localized and already has a route precedent, so accepting that brand-new silent-failure contract to avoid a small pre-cutover PR was not justified. Implemented and verified in `main` so API callers receive explicit rejection signaling (e.g. 413 with `command exceeds sheet limits`). - **Phase Placement**: Landed on `main` and ships in the Phase 2 bundle. It MUST NOT be folded into Phase 1: §4.2 defines Phase 1 as the behaviorally inert lifecycle-only bundle based on `149ebcf`, while §4.3 is the rollout of `main` and its behavioral changes. +6. **Bulk PITR Automation and Private Room Inventory Tooling Gap** + - **Specification**: Build operator CLI tooling (e.g. under `packages/cli/` or `scripts/`) to automate mass PITR iteration over candidate rooms from D1, and maintain an authenticated audit stream of private room creations (`POST /_/private`) to enable private room discovery. + - **Cutover Blocker Decision**: **NON-BLOCKING FOLLOW-UP GAP**. Single-room PITR endpoint (`POST /_/:room/pitr-restore`) is fully functional. Operators facing mass incidents on public rooms can execute manual loops or write one-off scripts using `npx wrangler d1 execute` and `curl`. Private room recovery during an incident relies on edge access/audit log recovery of room names. + --- ## §9 Go / No-Go Checklist @@ -1050,3 +1061,4 @@ The following user-visible behavior changes take effect upon completing the upgr 4. **Form/App-Mode Tab Hydration**: Browser tabs in form/app mode open across cutover must perform a page reload (`packages/client/src/boot.ts:384-390`). 5. **Passkey Accounts & Private Sheets**: Passkeys and private room creation become available after Phase 3 (`packages/worker/src/routes/auth.ts:133-137`). 6. **WebSocket Message Rate Limits**: Exceeding 1500 messages per 10-second window closes the WebSocket with 1008 (`packages/worker/src/room.ts:1887-1890`). +7. **Private Room Exclusion from D1 & Operational Recovery Limit**: Private rooms (`meta:access === 'private'`) are write-time excluded from the D1 `rooms` index (`packages/worker/src/room.ts:2270-2271`). They are invisible to `GET /_rooms`, `GET /_roomtimes`, and direct D1 SQL queries (`SELECT room FROM rooms`). In the event of a broad incident, D1-driven mass PITR recovery covers only public indexed rooms; recovering corrupted private rooms requires an out-of-band inventory of private room names (e.g. from edge access or audit logs). From d3e189b4e9488ac6ce099337b6251f6ba0c82c8c Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:26:16 +0800 Subject: [PATCH 11/95] docs(migration): update D1 private room recovery & backup handling in upgrade plan --- docs/migration/PROD_UPGRADE_PLAN.md | 116 ++++++++++++++++------------ 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 9b745d11..626073f7 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -224,8 +224,8 @@ Prior to cutover, record explicit backups for every durable store and acknowledg | Store | Capture Command / Procedure | Backup Artifact Location | Recovery Limitation / Reality | | :---------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------- | -| **D1 Database** (`ethercalc_rooms`) | `npx wrangler d1 export ethercalc_rooms --remote --output=./backup_ethercalc_rooms_$(date +%Y%m%d_%H%M%S).sql`

Also verify D1 subsystem `version: "production"` (§0.2.1) and record current Time Travel bookmark:
`npx wrangler d1 time-travel info ethercalc_rooms` | Local SQL file `./backup_ethercalc_rooms_*.sql` and Cloudflare D1 Time Travel snapshot log. | Fully exportable and restorable via D1 SQL import or D1 Time Travel restore.

**Command Syntax**: `--output` flag is **REQUIRED** for `wrangler d1 export` (per [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/#export)).

**Retention Caveat**: D1 Time Travel retention is plan-dependent per [Cloudflare D1 Time Travel](https://developers.cloudflare.com/d1/reference/time-travel/): *"up to 30 days in the past (Workers Paid plan) or 7 days (Workers Free plan)"*. Operator MUST confirm which plan the account is on before relying on a 30-day retention window.

**Subsystem Precondition**: Requires `version: "production"` (§0.2.1). If database is `version: "alpha"`, Time Travel is unsupported and recovery relies strictly on `d1 export` SQL dumps.

**CLI Compatibility**: Requires Wrangler `>= v3.4.0` (satisfied by root `package.json` pinning `"wrangler": "^4.112.0"`). | -| **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**D1 Mirror Scope & Reality**: D1 `ethercalc_rooms` holds the cross-room index (`room`, `updated_at`, `cors_public` per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`) plus bounded per-room audit and chat log tails (`audit_log` and `chat_log` per `packages/worker/migrations/0003_audit_chat.sql:22-38` and `packages/worker/src/lib/seq-store.ts:39-41`). **NO authoritative SocialCalc sheet snapshots, cell state, or full command logs are stored in D1.**

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (requires `ETHERCALC_MIGRATE_TOKEN`, §0.1).
2. Cloudflare platform-level Durable Object storage replication and durability. | In-DO SQLite storage (`state.storage`) and per-room PITR history buffers. *(D1 export recovers room index metadata and bounded audit/chat log tails, NOT sheet cell state or snapshots).* | **NO OPERATOR BULK BACKUP**: RoomDO sheet content (snapshots, full command logs, cell values) cannot be exported in bulk. A `wrangler d1 export` backs up room index metadata and audit/chat log tails, NOT sheet cell state or snapshots. `AuthDO` credentials (passkeys, session revocation lists) and un-saved in-memory WebSocket operational state also CANNOT be exported out of Cloudflare. Recovery of sheet content relies entirely on per-room PITR or Cloudflare platform storage durability. | +| **D1 Database** (`ethercalc_rooms`) | `npx wrangler d1 export ethercalc_rooms --remote --output=./backup_ethercalc_rooms_$(date +%Y%m%d_%H%M%S).sql`

Also verify D1 subsystem `version: "production"` (§0.2.1) and record current Time Travel bookmark:
`npx wrangler d1 time-travel info ethercalc_rooms` | Local SQL file `./backup_ethercalc_rooms_*.sql` and Cloudflare D1 Time Travel snapshot log. | Fully exportable and restorable via D1 SQL import or D1 Time Travel restore.

**Command Syntax**: `--output` flag is **REQUIRED** for `wrangler d1 export` (per [Wrangler D1 CLI Reference](https://developers.cloudflare.com/workers/wrangler/commands/d1/#export)).

**Retention Caveat**: D1 Time Travel retention is plan-dependent per [Cloudflare D1 Time Travel documentation](https://developers.cloudflare.com/d1/reference/time-travel/).

**Sensitive Material Handling**: Exported `.sql` files contain unfiltered audit commands (SocialCalc code) and user chat messages from both public and private rooms (see §2.2) and MUST be handled as sensitive, access-restricted material. | +| **Durable Objects** (`RoomDO` / `AuthDO`) | **NO BULK EXPORT API EXISTS** in Cloudflare Workers for Durable Object SQLite storage.

**D1 Mirror Scope & Reality**: D1 `ethercalc_rooms` holds the public cross-room index (`room`, `updated_at`, `cors_public` per `packages/worker/migrations/0001_rooms.sql:17-21` and `packages/worker/src/lib/rooms-index.ts:46-49`) plus bounded per-room audit and chat log tails for **all** rooms (including private rooms; `audit_log` and `chat_log` per `packages/worker/migrations/0003_audit_chat.sql:22-38` and `packages/worker/src/lib/seq-store.ts:39-41`). **NO authoritative SocialCalc sheet snapshots, cell state, or full command logs are stored in D1.**

**Available Safeguards:**
1. Per-room PITR snapshot restoration via operator endpoint `POST /_/:room/pitr-restore` (`API.md:133-197`).
2. Active private room discovery via `SELECT DISTINCT room FROM audit_log UNION SELECT DISTINCT room FROM chat_log` (§2.4). | | **Static Assets** | Built from source code via `scripts/build-assets.ts`. | Source repository commits (`assets/` directory built during deploy). | 100% reproducible; re-deploys instantly from git source. | | **Secrets & Environment Variables** | Verified via `npx wrangler secret list --config=packages/worker/wrangler.toml --env=""`. | Cloudflare Workers Secret Manager. | Secret values CANNOT be retrieved/read back via CLI. Precondition checks in §0.1 ensure secrets are set prior to cutover. | @@ -234,9 +234,10 @@ Prior to cutover, record explicit backups for every durable store and acknowledg Because Cloudflare Workers provides no operator-side bulk export API for Durable Object SQLite storage, the operator MUST understand what each backup artifact covers: 1. **What `wrangler d1 export` Backs Up**: - - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`), `audit_log`, `chat_log`, and `cron_triggers` tables. - - Allows restoring the list of room names, last-modified timestamps, and bounded audit/chat log tails. + - Backs up the `rooms` index table (`room`, `updated_at`, `cors_public`), `audit_log`, `chat_log`, and `cron_triggers` tables across all rooms (including private rooms, since `#mirrorAudit` and `#mirrorChat` in `packages/worker/src/room.ts:2303-2309` do not filter on `access === 'private'`). + - Allows restoring the list of public room names, last-modified timestamps, bounded audit/chat log tails, and enumerating active private rooms (`SELECT DISTINCT room FROM audit_log UNION SELECT DISTINCT room FROM chat_log`). - **DOES NOT** back up or restore sheet cell data, authoritative SocialCalc command logs, or sheet snapshots. A D1 export cannot reconstruct RoomDO cell state. + - **Sensitive Material Handling**: Because `audit_log` and `chat_log` contain raw SocialCalc commands and chat messages from private rooms without write-time access filtering, `wrangler d1 export` SQL dumps contain private room data. Backup `.sql` artifacts MUST be handled as sensitive, access-restricted material. 2. **What Cloudflare D1 Time Travel Restores**: - Restores D1 tables (`rooms`, `audit_log`, `chat_log`, `cron_triggers`) to a prior point-in-time (§6.4). @@ -269,51 +270,64 @@ PITR is scoped to one Durable Object instance, not a namespace. EtherCalc has on ### 2.4 Mass RoomDO Recovery Procedure -D1 and Durable Object PITR supply complementary halves of broad recovery: the D1 `rooms` table supplies indexed room names and their last-write times, while per-room PITR restores the content of each corresponding `RoomDO`. `GET /_rooms` and `GET /_roomtimes` expose the same index, but a direct D1 query is safer for an operator procedure because it returns both fields in one captured result. - -1. **Freeze the recovery inputs.** Record the incident start and end in UTC and select one restore timestamp immediately before the first bad write. It MUST be within Cloudflare's 30-day Durable Object PITR window. Do not derive a different timestamp for each room unless the incident analysis explicitly requires it. -2. **Capture the candidate list before restoring anything.** `updated_at` is a JavaScript millisecond epoch (`Date.now()`), so query with millisecond boundaries. Replace the example values with the incident window and save the complete output: - - ```bash - npx wrangler d1 execute ethercalc_rooms \ - --remote \ - --config=packages/worker/wrangler.toml \ - --env="" \ - --command="SELECT room, updated_at FROM rooms WHERE updated_at BETWEEN 1786363200000 AND 1786366800000 ORDER BY updated_at ASC;" - ``` - - The equivalent HTTP sources are `GET /_rooms` for names and `GET /_roomtimes` for timestamps. Prefer the D1 query when filtering an incident window. -3. **Dry-run every candidate sequentially.** For each D1 row, URL-encode the room as one path segment and resolve the common pre-incident timestamp without scheduling a restore: - - ```bash - curl -sS --fail-with-body -w '\nHTTP %{http_code}\n' -X POST "https://ethercalc.net/_//pitr-restore" \ - -H "Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN" \ - -H "Content-Type: application/json" \ - --data '{"at":"2026-08-10T11:59:59.000Z","dryRun":true}' - ``` - - Record the room, D1 `updated_at`, requested timestamp, resolved `bookmark`, HTTP status, and response. Stop on any non-200 response; do not silently skip a room. -4. **Apply and verify one room at a time.** Submit the dry-run bookmark, retain the returned `undoBookmark`, then verify the room's snapshot/content before advancing: - - ```bash - curl -sS --fail-with-body -w '\nHTTP %{http_code}\n' -X POST "https://ethercalc.net/_//pitr-restore" \ - -H "Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN" \ - -H "Content-Type: application/json" \ - --data '{"bookmark":""}' - ``` - - A successful response reports `restored: true`; `exists: false` means the target predates creation of that room. If verification fails, submit the recorded `undoBookmark` as `bookmark` before investigating. Preserve the full candidate list and per-room responses as the recovery audit record. - -This is intentionally a **sequential, per-room** procedure. Its throughput and request rate are limited by the delay, retry, and concurrency controls the operator puts into an incident script; EtherCalc supplies none. PITR rewinds the entire RoomDO SQLite database to the selected timestamp, including later key-value state, so it cannot selectively remove one bad SocialCalc command while retaining later good commands. - -**Headline Enumeration Gap & Operational Severity:** The D1 `rooms` index is **NOT** a complete namespace inventory: - -1. **Private Room Write-Time Exclusion**: Tracing `POST /_/private` -> `/_do/init-private` (`packages/worker/src/room.ts:1310-1386`) shows that private room creation sets `STORAGE_KEYS.metaAccess = 'private'` directly in DO storage without calling `#mirrorIndex`. Furthermore, every subsequent write routes through `RoomDO.#mirrorIndex` (`packages/worker/src/room.ts:2266-2273`), which explicitly checks `const { access } = await this.#getAccessMeta(); if (access === 'private') return;` and returns before calling `mirrorRoomToD1`. In addition, `#postTouch` (`packages/worker/src/room.ts:620-622`) checks `access === 'private'` and calls `#deleteIndex` (`DELETE FROM rooms WHERE room = ?1`). Private rooms are **structurally excluded from D1 at write time**. They do NOT exist in the `rooms` table, nor in `GET /_rooms` or `GET /_roomtimes`. -2. **Form-Data Sibling Exclusion**: Rooms ending in `_formdata` (internal submitform storage) are filtered out of public listings by `isPublicRoomIndexEntry` (`packages/worker/src/lib/formdata-sibling.ts:18-19`). - -**Consequence for Incident Recovery**: The D1-driven mass recovery procedure above discovers and restores **only public indexed rooms**. Content in private rooms (which hold sensitive user-authenticated data) has **no platform enumeration source in D1**. If a broad incident corrupts private rooms, D1 query iteration will NOT find them. Restoring private rooms requires an out-of-band inventory of private room names (e.g. from edge access logs or application audit trails) and submitting per-room `POST /_/:room/pitr-restore` calls for each known private room name. Cloudflare provides no namespace-wide API to iterate Durable Object instances. - -**Tooling Gap Summary**: No bulk-PITR script or CLI exists under `packages/cli/`, `scripts/`, or `bin/`; only the single-room HTTP endpoint exists. An operator facing a broad incident must write and peer-review the enumeration/restore driver under pressure unless one is prepared in advance. + D1 and Durable Object PITR supply complementary halves of broad recovery: the D1 `rooms` table supplies indexed public room names and their last-write times, while `audit_log` and `chat_log` provide enumeration of active private rooms. Per-room PITR restores the content of each corresponding `RoomDO`. `GET /_rooms` and `GET /_roomtimes` expose the public index, but direct D1 SQL queries are safer for an operator procedure because they return exact fields in captured results. + + 1. **Freeze the recovery inputs.** Record the incident start and end in UTC and select one restore timestamp immediately before the first bad write. It MUST be within Cloudflare's 30-day Durable Object PITR window. Do not derive a different timestamp for each room unless the incident analysis explicitly requires it. + 2. **Capture the candidate list before restoring anything.** `updated_at` is a JavaScript millisecond epoch (`Date.now()`), so query with millisecond boundaries. Replace the example values with the incident window and save the complete output: + + ```bash + # 2a. Public indexed rooms (with last update timestamps): + npx wrangler d1 execute ethercalc_rooms \ + --remote \ + --config=packages/worker/wrangler.toml \ + --env="" \ + --command="SELECT room, updated_at FROM rooms WHERE updated_at BETWEEN 1786363200000 AND 1786366800000 ORDER BY updated_at ASC;" + + # 2b. Private active rooms (enumerated via unfiltered D1 audit and chat logs): + npx wrangler d1 execute ethercalc_rooms \ + --remote \ + --config=packages/worker/wrangler.toml \ + --env="" \ + --command="SELECT DISTINCT room FROM audit_log UNION SELECT DISTINCT room FROM chat_log;" + ``` + + The equivalent HTTP sources for public rooms are `GET /_rooms` for names and `GET /_roomtimes` for timestamps. Prefer the D1 queries when filtering an incident window. + 3. **Dry-run every candidate sequentially.** For each D1 candidate room (from public `rooms` or private `audit_log`/`chat_log`), URL-encode the room as one path segment and resolve the common pre-incident timestamp without scheduling a restore: + + ```bash + curl -sS --fail-with-body -w '\nHTTP %{http_code}\n' -X POST "https://ethercalc.net/_//pitr-restore" \ + -H "Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"at":"2026-08-10T11:59:59.000Z","dryRun":true}' + ``` + + Record the room, D1 `updated_at` (or `[private-log]`), requested timestamp, resolved `bookmark`, HTTP status, and response. Stop on any non-200 response; do not silently skip a room. + 4. **Apply and verify one room at a time.** Submit the dry-run bookmark, retain the returned `undoBookmark`, then verify the room's snapshot/content before advancing: + + ```bash + curl -sS --fail-with-body -w '\nHTTP %{http_code}\n' -X POST "https://ethercalc.net/_//pitr-restore" \ + -H "Authorization: Bearer $ETHERCALC_MIGRATE_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"bookmark":""}' + ``` + + A successful response reports `restored: true`; `exists: false` means the target predates creation of that room. If verification fails, submit the recorded `undoBookmark` as `bookmark` before investigating. Preserve the full candidate list and per-room responses as the recovery audit record. + + This is intentionally a **sequential, per-room** procedure. Its throughput and request rate are limited by the delay, retry, and concurrency controls the operator puts into an incident script; EtherCalc supplies none. PITR rewinds the entire RoomDO SQLite database to the selected timestamp, including later key-value state, so it cannot selectively remove one bad SocialCalc command while retaining later good commands. + + **Headline Enumeration Structure & Asymmetry:** + + 1. **Private Room Index Exclusion**: Tracing `POST /_/private` -> `/_do/init-private` (`packages/worker/src/room.ts:1310-1386`) shows that private room creation sets `STORAGE_KEYS.metaAccess = 'private'` directly in DO storage without calling `#mirrorIndex`. Furthermore, every subsequent write routes through `RoomDO.#mirrorIndex` (`packages/worker/src/room.ts:2266-2273`), which explicitly checks `const { access } = await this.#getAccessMeta(); if (access === 'private') return;` (lines 2270-2271) and returns before calling `mirrorRoomToD1`. In addition, `#postTouch` (`packages/worker/src/room.ts:620-622`) checks `access === 'private'` and calls `#deleteIndex` (`DELETE FROM rooms WHERE room = ?1`). Private rooms are **structurally excluded from the public D1 `rooms` index at write time**. They are invisible to `GET /_rooms`, `GET /_roomtimes`, and `SELECT room FROM rooms`. + 2. **Unfiltered D1 Audit/Chat Mirroring**: In contrast to `#mirrorIndex`, tracing `#applyCommandAndMirror` (`packages/worker/src/room.ts:726-750`) and `appendChat` (`packages/worker/src/room.ts:2330-2348`) reveals that audit entries (`#mirrorAudit` -> `appendAuditRows` at lines 2303-2304) and chat messages (`#mirrorChat` -> `appendChatRows` at lines 2308-2309) contain **no access check whatsoever**. When a command or chat message is executed in a private room, its audit row (containing the raw SocialCalc command) or chat row (containing user text) is mirrored to D1 `audit_log` or `chat_log`. + 3. **Form-Data Sibling Exclusion**: Rooms ending in `_formdata` (internal submitform storage) are filtered out of public listings by `isPublicRoomIndexEntry` (`packages/worker/src/lib/formdata-sibling.ts:18-19`). + + **Consequence for Incident Recovery & Data Handling**: + + - **Recovery Upside**: `SELECT DISTINCT room FROM audit_log UNION SELECT DISTINCT room FROM chat_log` acts as a platform-native D1 enumeration source for private rooms, materially softening the "no mass-recovery path" gap for private rooms. + - **Enumeration Caveats & Limits**: The D1 audit/chat query only discovers private rooms that have had at least one command or chat message mirrored. Completely idle/empty private rooms (created with no edits or chat) or rooms whose D1 records were explicitly deleted via `DELETE /_do/all` (`#deleteAuditChatFromD1` at lines 2313-2318) will not appear in D1. In addition, D1 log tails are bounded (`AUDIT_HISTORY_KEEP = 1024`, `CHAT_HISTORY_KEEP = 500` per `packages/worker/src/lib/seq-store.ts:40-41`). D1 log queries serve as an **enumeration aid**, NOT a complete content backup. For zero-activity private rooms, edge access logs or application audit trails remain necessary. + - **Privacy & Artifact Handling**: While the design intentionally keeps private room IDs out of the public `rooms` index (`/_rooms` and `/_roomtimes`), private room names, SocialCalc commands, and chat messages sit unfiltered in D1 `audit_log` and `chat_log`. D1 is an internal store and treated as sensitive, but operators MUST recognize that `wrangler d1 export` SQL dumps contain private-room material and handle exported `.sql` files as sensitive, access-restricted data. + + **Tooling Gap Summary**: No bulk-PITR script or CLI exists under `packages/cli/`, `scripts/`, or `bin/`; only the single-room HTTP endpoint exists. An operator facing a broad incident must write and peer-review the enumeration/restore driver under pressure unless one is prepared in advance. The D1 `audit_log`/`chat_log` distinct-room query provides native enumeration for active private rooms, restricting the out-of-band edge log search requirement to zero-activity private rooms only. --- @@ -1031,7 +1045,7 @@ The following five items have been evaluated and categorized: 6. **Bulk PITR Automation and Private Room Inventory Tooling Gap** - **Specification**: Build operator CLI tooling (e.g. under `packages/cli/` or `scripts/`) to automate mass PITR iteration over candidate rooms from D1, and maintain an authenticated audit stream of private room creations (`POST /_/private`) to enable private room discovery. - - **Cutover Blocker Decision**: **NON-BLOCKING FOLLOW-UP GAP**. Single-room PITR endpoint (`POST /_/:room/pitr-restore`) is fully functional. Operators facing mass incidents on public rooms can execute manual loops or write one-off scripts using `npx wrangler d1 execute` and `curl`. Private room recovery during an incident relies on edge access/audit log recovery of room names. + - **Cutover Blocker Decision**: **NON-BLOCKING FOLLOW-UP GAP**. Single-room PITR endpoint (`POST /_/:room/pitr-restore`) is fully functional. Operators facing mass incidents on public rooms can execute manual loops or write one-off scripts using `npx wrangler d1 execute` and `curl`. Active private room enumeration during an incident is supported via D1 queries (`SELECT DISTINCT room FROM audit_log UNION SELECT DISTINCT room FROM chat_log`), while zero-activity private rooms rely on edge access logs. --- @@ -1061,4 +1075,4 @@ The following user-visible behavior changes take effect upon completing the upgr 4. **Form/App-Mode Tab Hydration**: Browser tabs in form/app mode open across cutover must perform a page reload (`packages/client/src/boot.ts:384-390`). 5. **Passkey Accounts & Private Sheets**: Passkeys and private room creation become available after Phase 3 (`packages/worker/src/routes/auth.ts:133-137`). 6. **WebSocket Message Rate Limits**: Exceeding 1500 messages per 10-second window closes the WebSocket with 1008 (`packages/worker/src/room.ts:1887-1890`). -7. **Private Room Exclusion from D1 & Operational Recovery Limit**: Private rooms (`meta:access === 'private'`) are write-time excluded from the D1 `rooms` index (`packages/worker/src/room.ts:2270-2271`). They are invisible to `GET /_rooms`, `GET /_roomtimes`, and direct D1 SQL queries (`SELECT room FROM rooms`). In the event of a broad incident, D1-driven mass PITR recovery covers only public indexed rooms; recovering corrupted private rooms requires an out-of-band inventory of private room names (e.g. from edge access or audit logs). +7. **Private Room D1 Asymmetry & Recovery Mechanics**: Private rooms (`meta:access === 'private'`) are write-time excluded from the public D1 `rooms` index (`packages/worker/src/room.ts:2270-2271`). They are invisible to `GET /_rooms`, `GET /_roomtimes`, and direct D1 queries (`SELECT room FROM rooms`). However, `#applyCommandAndMirror` and `appendChat` mirror command audit entries and chat messages to D1 `audit_log` and `chat_log` without access checks (`packages/worker/src/room.ts:2303-2309`). Consequently: (1) `SELECT DISTINCT room FROM audit_log UNION SELECT DISTINCT room FROM chat_log` provides a platform-native D1 enumeration source for active private rooms during mass PITR recovery; (2) `wrangler d1 export` SQL backups contain private-room SocialCalc commands and chat messages, requiring exported `.sql` files to be handled as sensitive, access-restricted material. From 8c84f3309c002aedce24f40797bcaf993221cd75 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:29:28 +0800 Subject: [PATCH 12/95] =?UTF-8?q?docs:=20reconcile=20=C2=A73.2=20staging?= =?UTF-8?q?=20checklist=20with=20=C2=A75=20empirically=20verified=20contra?= =?UTF-8?q?cts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/migration/PROD_UPGRADE_PLAN.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 626073f7..58642f83 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -375,14 +375,14 @@ Execute the following verification checklist against `https://ethercalc-staging. - Verify that the new client bundle (`boot.ts:384-390`) sends `ask.log` for the main room in addition to `_formdata`, successfully hydrating the sheet grid. 4. **Large Command Batch & Limit Enforcement**: - Submit a command batch exceeding `MAX_SHEET_CELLS = 200_000` (e.g. `POST /_/staging-limit-test` expanding dimensions to A1:Z10000 = 260,000 cells). - - Verify the HTTP response is `413` with the exact lowercase body `command exceeds sheet limits` (`packages/worker/src/room.ts:703`), while the WebSocket closes with `1008` and the distinct capitalized reason `Command exceeds sheet limits` (`packages/worker/src/room.ts:1805`). + - Verify the HTTP response is `413 Payload Too Large` with `Content-Type: text/plain; charset=utf-8` and the exact lowercase body `command exceeds sheet limits` (`packages/worker/src/room.ts:703`, matching local Worker baseline in §5 Probe 6), while the WebSocket closes with `1008` and the distinct capitalized reason `Command exceeds sheet limits` (`packages/worker/src/room.ts:1805`). 5. **Legacy `/socket.io/*` Transport Shim**: - Request `curl -fsS "https://ethercalc-staging.audreyt.workers.dev/socket.io/1/?t=$(date +%s)"`. - - Verify response returns HTTP 200 with session handshake parameters. + - Verify response returns HTTP 200, `Content-Type: text/plain; charset=utf-8`, and body matching `<32-hex-session-id>:60:60:websocket,xhr-polling` (matching local Worker baseline in §5 Probe 8). 6. **XLSX Import and Export**: - - Upload a test `.xlsx` file via `POST /_/staging-xlsx-test`. - - Export sheet via `GET /_/staging-xlsx-test.xlsx`. - - Verify downloaded file parses correctly and preserves cell data. + - Upload a test `.xlsx` file via `POST /_/staging-xlsx-test` with header `Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet` (which decodes binary workbook data into `loadclipboard` + `paste A1 all` commands via the `xlsx-deferred` handler in `packages/worker/src/routes/rooms.ts:745`). + - Export sheet via `GET /staging-xlsx-test.xlsx` (or the alternative valid export spelling `GET /_/staging-xlsx-test/xlsx`). Verify response returns `200 OK` with `Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`, binary body starting with ZIP signature bytes `PK\x03\x04`, and `Content-Disposition: attachment; filename="staging-xlsx-test.xlsx"`. Confirm downloaded file parses correctly and preserves cell data. + - **CRITICAL WARNING**: The `GET /_/staging-xlsx-test.xlsx` spelling is **NOT** an export route. It targets a raw room named `staging-xlsx-test.xlsx` and returns `404 Not Found` with an empty `text/plain` body (matching local Worker baseline in §5 Probe 10a). An operator following the incorrect `/_/.xlsx` spelling during staging rehearsal would see a 404, erroneously conclude XLSX export is broken, and could halt an otherwise healthy cutover. 7. **Phase 2 Passkey Disabled Check (`ETHERCALC_AUTH="0"`)**: - Query `curl -fsS https://ethercalc-staging.audreyt.workers.dev/_auth/whoami`. - Verify body returns `{"uid":null,"enabled":false}` and `POST /_/private` returns `401 Unauthorized`. @@ -394,12 +394,11 @@ Execute the following verification checklist against `https://ethercalc-staging. - **CRITICAL NOTE**: Passkeys registered on staging are bound to WebAuthn RP ID `ethercalc-staging.audreyt.workers.dev` (`packages/worker/wrangler.toml:84`). They are **NOT portable** to production (`ethercalc.net`) because WebAuthn RP IDs strictly enforce exact domain matching. 9. **Asset Purge & Freshness Check**: - Request `curl -fsSI https://ethercalc-staging.audreyt.workers.dev/static/player.js`. - - Confirm asset loads cleanly with `200 OK` and `Content-Type: text/javascript; charset=utf-8` (matching local Worker baseline in §5 Probe 3). + - Confirm asset loads cleanly with `200 OK` and `Content-Type: text/javascript; charset=utf-8` (matching local Worker baseline in §5 Probe 4). - **Note**: Production Cloudflare Workers Assets edge may emit `Content-Type: application/javascript` or `text/javascript` depending on global edge MIME tables; operators should treat a MIME subtype mismatch as informational as long as HTTP status is `200 OK`. 10. **Cross-Room Index Gating (`/_rooms*`)**: - -- Request `curl -sS -i https://ethercalc-staging.audreyt.workers.dev/_rooms`. -- On staging (`ETHERCALC_CORS="1"`), verify endpoint returns `403 Forbidden` (`"_rooms not available with CORS"`). + - Request `curl -sS -i https://ethercalc-staging.audreyt.workers.dev/_rooms`. + - On staging (`ETHERCALC_CORS="1"`), verify endpoint returns HTTP `403 Forbidden`, `Content-Type: text/plain; charset=utf-8`, and body matching exactly `_rooms not available with CORS` (30 bytes, no trailing newline, matching local Worker baseline in §5 Probe 9). --- From b8e50d29c7b0d05f68ddd787f43193f8a2014bda Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:37:12 +0800 Subject: [PATCH 13/95] docs: source-verify section 7 self-host upgrade plan and record pinned workerd version --- docs/migration/PROD_UPGRADE_PLAN.md | 32 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 58642f83..663a4d35 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -900,23 +900,28 @@ For self-hosted deployments running EtherCalc via Docker Compose or Kubernetes/H ### 7.1 Container Image Bump & Workerd Compatibility Lockstep 1. Update image tag from `ethercalc:0.20260717.0` to `ethercalc:main` (or newest tag). -2. **Compatibility Date Lockstep Check**: +2. **Base Image & Workerd Bundle Verification (`[SOURCE-VERIFIED]`)**: + - `Dockerfile:22` pins base image `FROM oven/bun:1.3.14`. + - `Dockerfile:60` builds the standalone ES module bundle via `scripts/build-workerd-bundle.sh`, which runs `bunx wrangler deploy --dry-run --outdir=packages/worker/workerd/worker/` to produce `packages/worker/workerd/worker/index.js` alongside checked-in `packages/worker/workerd/config.capnp`. +3. **Compatibility Date Lockstep & Pinned Version Check (`[SOURCE-VERIFIED]`)**: - `packages/worker/wrangler.toml:3` specifies `compatibility_date = "2026-07-21"`. - `packages/worker/workerd/config.capnp:61` specifies `compatibilityDate = "2026-07-14"`. - - Standalone `workerd` binary releases enforce that `compatibilityDate` must be `<= workerd release date`. Operator MUST verify host `workerd` binary version is dated `>= 2026-07-14`. + - Standalone `workerd` binary releases enforce that `compatibilityDate` must be `<= workerd release date`. + - **Lockfile & Binary Resolution Verification**: `bun.lock` resolves root/wrangler dependency `workerd@1.20260714.1` (and platform binaries `@cloudflare/workerd-*-*@1.20260714.1`) via `wrangler@4.112.0`. The version string `1.20260714.1` decodes to release date `2026-07-14`, which matches `compatibilityDate = "2026-07-14"` exactly. + - **Entrypoint Binary Resolution**: `bin/workerd-entrypoint.sh:48-55` explicitly targets wrangler's pinned dependency `$APP_ROOT/node_modules/wrangler/../workerd/bin/workerd` (`1.20260714.1`) to ensure runtime execution does not pick up older nested copies (such as `@cloudflare/vitest-pool-workers`'s `1.20260701.1`). Operator host `workerd` binary MUST be dated `>= 2026-07-14` (pinned release: `1.20260714.1`). ### 7.2 Durable Object On-Disk Storage Backup Before restarting containers, back up the local DO storage directory. -- **Docker Compose Track**: +- **Docker Compose Track (`[SOURCE-VERIFIED]`)**: - The shipped compose file mounts DO storage at `./ethercalc-data:/data` (`docker-compose.yml:37`). - - Container entrypoint (`bin/workerd-entrypoint.sh:25-27`) writes SQLite files under `$DATA_DIR/do` (`/data/do`). + - Container entrypoint (`bin/workerd-entrypoint.sh:25-27`) writes SQLite files under `$DATA_DIR/do` (`/data/do`), where `workerd serve` stores per-DO SQLite databases under `/data/do//`. - **Host Backup Command**: ```bash tar -czvf ethercalc-do-backup-$(date +%Y%m%d_%H%M%S).tar.gz ./ethercalc-data ``` -- **Kubernetes / Helm Track**: +- **Kubernetes / Helm Track (`[SOURCE-VERIFIED]`)**: - `helm/values.yaml:80-90` configures `persistence.enabled: true` (`accessMode: ReadWriteOnce`, `size: 10Gi`). - `helm/templates/deployment.yaml:117-129` binds the PVC `persistentVolumeClaim: claimName: {{ include "ethercalc.pvcName" . }}` to `mountPath: /data`. - **Cluster Backup Procedure**: Take a volume snapshot of the backing PersistentVolumeClaim (`ethercalc-data`), or stream an archive out via `kubectl exec`: @@ -926,8 +931,7 @@ Before restarting containers, back up the local DO storage directory. ### 7.3 Mandatory Nginx Reverse Proxy Requirement -Internet-facing self-host installations MUST run the nginx proxy recipe (`deploy/nginx/ethercalc.conf`) for rate-limiting, CSP header injection, and client IP header sanitization (`CF-Connecting-IP` stripping). Validate with `./scripts/smoke-proxy.sh`. - +Internet-facing self-host installations MUST run the nginx proxy recipe (`deploy/nginx/ethercalc.conf`) for rate-limiting and client IP header sanitization (overwriting client-supplied `CF-Connecting-IP`, `X-Forwarded-For`, and `X-Forwarded-Proto` with `$remote_addr` and `$scheme`). Note: CSP headers are injected directly by the Worker application layer (`packages/worker/src/lib/csp.ts`), not by Nginx. Validate basic proxy syntax, routing, health, room-index gating, and WebSocket upgrades with `./scripts/smoke-proxy.sh`. ### 7.4 Security Environment Variables & Passkey Anchors Self-host operators MUST configure passkey environment variables to enable passkey features. @@ -1014,6 +1018,20 @@ To light up passkey authentication and private rooms, the operator MUST provide ./scripts/smoke-selfhost.sh ``` +### 7.5 Verification Status & Local Execution Gate Note + +- **Source-Verified Claims**: + - Dockerfile base image (`oven/bun:1.3.14` at `Dockerfile:22`). + - Standalone ES module bundle creation (`scripts/build-workerd-bundle.sh` producing `packages/worker/workerd/worker/index.js`). + - Capnp compatibility date lockstep (`compatibilityDate = "2026-07-14"` in `config.capnp:61` satisfied by resolved `workerd@1.20260714.1` in `bun.lock`). + - Entrypoint workerd binary targeting (`bin/workerd-entrypoint.sh:48-55` picking wrangler's pinned dependency). + - Volume mount target `./ethercalc-data:/data` mapping to DO SQLite storage path `/data/do` (`docker-compose.yml:37`, `bin/workerd-entrypoint.sh:25-27`, `Dockerfile:63-72`). + - Security env defaults (`ETHERCALC_DISABLE_ROOM_INDEX=1` default ON in `Dockerfile:90`, `docker-compose.yml:46`, `helm/values.yaml:98`). + - Nginx reverse proxy configuration (`deploy/nginx/ethercalc.conf`, which applies `limit_req`/`limit_conn` and replaces `CF-Connecting-IP`/`X-Forwarded-*` headers) and validator script (`scripts/smoke-proxy.sh`). +- **Unexercised / Scope Limits**: + - Docker smoke scripts (`./scripts/smoke-selfhost.sh` and `./scripts/smoke-proxy.sh`) could NOT be executed in this local environment due to missing `docker compose` CLI subcommand. + - `./scripts/smoke-proxy.sh` validates `nginx -t` syntax, `/_health`, `GET /`, `/_rooms` HTTP 403, and `/_ws/` HTTP 101 WebSocket upgrade forwarding; it does not exercise rate-limiting thresholds or IP header substitution details. + - End-to-end container boot, workerd execution, and reverse proxy routing for the self-host path remain unexercised locally and rely on CI execution (`.github/workflows/ci.yml` `build:selfhost` job). --- ## §8 Pre-Cutover PRs & Preparation Bundles From 9eac6ad9f0b76b4a0973735c3958233eccbe63aa Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:38:43 +0800 Subject: [PATCH 14/95] docs: call out zero-margin workerd lockstep and test skew --- docs/migration/PROD_UPGRADE_PLAN.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 663a4d35..331acdd4 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -907,8 +907,11 @@ For self-hosted deployments running EtherCalc via Docker Compose or Kubernetes/H - `packages/worker/wrangler.toml:3` specifies `compatibility_date = "2026-07-21"`. - `packages/worker/workerd/config.capnp:61` specifies `compatibilityDate = "2026-07-14"`. - Standalone `workerd` binary releases enforce that `compatibilityDate` must be `<= workerd release date`. - - **Lockfile & Binary Resolution Verification**: `bun.lock` resolves root/wrangler dependency `workerd@1.20260714.1` (and platform binaries `@cloudflare/workerd-*-*@1.20260714.1`) via `wrangler@4.112.0`. The version string `1.20260714.1` decodes to release date `2026-07-14`, which matches `compatibilityDate = "2026-07-14"` exactly. - - **Entrypoint Binary Resolution**: `bin/workerd-entrypoint.sh:48-55` explicitly targets wrangler's pinned dependency `$APP_ROOT/node_modules/wrangler/../workerd/bin/workerd` (`1.20260714.1`) to ensure runtime execution does not pick up older nested copies (such as `@cloudflare/vitest-pool-workers`'s `1.20260701.1`). Operator host `workerd` binary MUST be dated `>= 2026-07-14` (pinned release: `1.20260714.1`). + - **Lockfile & Dual Workerd Package Version Resolution**: `bun.lock` resolves two distinct version sets for `workerd` and platform binaries (`@cloudflare/workerd-darwin-64`, `-darwin-arm64`, `-linux-64`, `-linux-arm64`, `-windows-64`): + - **Hoisted / Production Dependency (`wrangler@4.112.0`)**: `workerd@1.20260714.1` (and all 5 platform packages at `1.20260714.1`), which decodes to release date `2026-07-14`. + - **Nested Test Dependency (`@cloudflare/vitest-pool-workers@0.18.0`)**: `workerd@1.20260701.1` under `miniflare/workerd` and `wrangler/workerd` (and all 5 platform packages at `1.20260701.1`), which decodes to release date `2026-07-01` (`2026-07-01 < 2026-07-14`). + - **Zero-Margin Lockstep Invariant Callout**: The lockstep condition `2026-07-14 <= 2026-07-14` holds with **zero margin of slack**. Bumping `config.capnp`'s `compatibilityDate` by even a single day without simultaneously updating the pinned `workerd` dependency breaks standalone self-host (`workerd serve` hard-rejects configs whose date exceeds its build date). Conversely, updating `workerd` alone is safe (widens the margin). Note the critical deployment asymmetry: `wrangler.toml`'s `compatibility_date = "2026-07-21"` is 7 days ahead and safe because Wrangler clamps with a warning; syncing `config.capnp` to `"2026-07-21"` without a workerd bump would break self-host containers while Cloudflare edge deploys continue passing cleanly. + - **Entrypoint Binary Targeting & Test-Runtime Skew**: `bin/workerd-entrypoint.sh:48-55` explicitly targets wrangler's pinned dependency `$APP_ROOT/node_modules/wrangler/../workerd/bin/workerd` (`1.20260714.1`) to ensure runtime execution does not pick up the older nested `1.20260701.1` binary (which would refuse the config and crash-loop). Additionally, note that `@cloudflare/vitest-pool-workers` executes unit/workers-pool tests against the older `1.20260701.1` binary (a 13-day skew from the `1.20260714.1` deployment runtime); while `config.capnp` is not loaded in vitest, test behavior depending on workerd engine changes between 2026-07-01 and 2026-07-14 is not exercised by workers-pool tests. Operator host `workerd` binary MUST be dated `>= 2026-07-14` (pinned release: `1.20260714.1`). ### 7.2 Durable Object On-Disk Storage Backup @@ -1023,8 +1026,9 @@ To light up passkey authentication and private rooms, the operator MUST provide - **Source-Verified Claims**: - Dockerfile base image (`oven/bun:1.3.14` at `Dockerfile:22`). - Standalone ES module bundle creation (`scripts/build-workerd-bundle.sh` producing `packages/worker/workerd/worker/index.js`). - - Capnp compatibility date lockstep (`compatibilityDate = "2026-07-14"` in `config.capnp:61` satisfied by resolved `workerd@1.20260714.1` in `bun.lock`). - - Entrypoint workerd binary targeting (`bin/workerd-entrypoint.sh:48-55` picking wrangler's pinned dependency). + - Capnp compatibility date lockstep (`compatibilityDate = "2026-07-14"` in `config.capnp:61` satisfied with zero margin by hoisted `workerd@1.20260714.1` in `bun.lock`). + - Dual workerd dependency mapping in `bun.lock` (hoisted `1.20260714.1` vs nested `vitest-pool-workers` `1.20260701.1`) and test-runtime skew documentation. + - Entrypoint workerd binary targeting (`bin/workerd-entrypoint.sh:48-55` picking wrangler's hoisted dependency over older nested copies). - Volume mount target `./ethercalc-data:/data` mapping to DO SQLite storage path `/data/do` (`docker-compose.yml:37`, `bin/workerd-entrypoint.sh:25-27`, `Dockerfile:63-72`). - Security env defaults (`ETHERCALC_DISABLE_ROOM_INDEX=1` default ON in `Dockerfile:90`, `docker-compose.yml:46`, `helm/values.yaml:98`). - Nginx reverse proxy configuration (`deploy/nginx/ethercalc.conf`, which applies `limit_req`/`limit_conn` and replaces `CF-Connecting-IP`/`X-Forwarded-*` headers) and validator script (`scripts/smoke-proxy.sh`). From 6e33aa3c9d98c6b09eeac8a24000900d832947d1 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:40:46 +0800 Subject: [PATCH 15/95] docs: note CI guard for workerd date lockstep --- docs/migration/PROD_UPGRADE_PLAN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 331acdd4..6c497e7e 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -910,8 +910,8 @@ For self-hosted deployments running EtherCalc via Docker Compose or Kubernetes/H - **Lockfile & Dual Workerd Package Version Resolution**: `bun.lock` resolves two distinct version sets for `workerd` and platform binaries (`@cloudflare/workerd-darwin-64`, `-darwin-arm64`, `-linux-64`, `-linux-arm64`, `-windows-64`): - **Hoisted / Production Dependency (`wrangler@4.112.0`)**: `workerd@1.20260714.1` (and all 5 platform packages at `1.20260714.1`), which decodes to release date `2026-07-14`. - **Nested Test Dependency (`@cloudflare/vitest-pool-workers@0.18.0`)**: `workerd@1.20260701.1` under `miniflare/workerd` and `wrangler/workerd` (and all 5 platform packages at `1.20260701.1`), which decodes to release date `2026-07-01` (`2026-07-01 < 2026-07-14`). - - **Zero-Margin Lockstep Invariant Callout**: The lockstep condition `2026-07-14 <= 2026-07-14` holds with **zero margin of slack**. Bumping `config.capnp`'s `compatibilityDate` by even a single day without simultaneously updating the pinned `workerd` dependency breaks standalone self-host (`workerd serve` hard-rejects configs whose date exceeds its build date). Conversely, updating `workerd` alone is safe (widens the margin). Note the critical deployment asymmetry: `wrangler.toml`'s `compatibility_date = "2026-07-21"` is 7 days ahead and safe because Wrangler clamps with a warning; syncing `config.capnp` to `"2026-07-21"` without a workerd bump would break self-host containers while Cloudflare edge deploys continue passing cleanly. - - **Entrypoint Binary Targeting & Test-Runtime Skew**: `bin/workerd-entrypoint.sh:48-55` explicitly targets wrangler's pinned dependency `$APP_ROOT/node_modules/wrangler/../workerd/bin/workerd` (`1.20260714.1`) to ensure runtime execution does not pick up the older nested `1.20260701.1` binary (which would refuse the config and crash-loop). Additionally, note that `@cloudflare/vitest-pool-workers` executes unit/workers-pool tests against the older `1.20260701.1` binary (a 13-day skew from the `1.20260714.1` deployment runtime); while `config.capnp` is not loaded in vitest, test behavior depending on workerd engine changes between 2026-07-01 and 2026-07-14 is not exercised by workers-pool tests. Operator host `workerd` binary MUST be dated `>= 2026-07-14` (pinned release: `1.20260714.1`). + - **Zero-Margin Lockstep Invariant — CI-Guarded**: The lockstep condition `2026-07-14 <= 2026-07-14` holds with **zero margin of slack**. Bumping `config.capnp`'s `compatibilityDate` by even a single day without simultaneously updating the pinned `workerd` dependency would make standalone `workerd serve` hard-reject the config. This mismatch is guarded by the root test `standalone compatibility date does not exceed the pinned workerd release` (`scripts/vite-workflow.test.ts:440-456`): after extracting the hoisted workerd release date from `bun.lock` and the configured date from `config.capnp`, `scripts/vite-workflow.test.ts:455` asserts `expect(Number(configuredDate)).toBeLessThanOrEqual(Number(workerdDate));`. The test runs through the root `test` script (`package.json:83`) under preflight Gate 3; the recorded Gate 3 run passed all 36 root tests (`docs/migration/PREFLIGHT_RESULTS.md:94-101`). Therefore, a mismatched date fails fast in CI/preflight rather than becoming a mystery production crash-loop, provided that gate is honored. Conversely, updating `workerd` alone is safe (it widens the margin). The deployment asymmetry remains critical: `wrangler.toml`'s `compatibility_date = "2026-07-21"` is 7 days ahead and safe because Wrangler clamps with a warning, but syncing `config.capnp` to `"2026-07-21"` without a workerd bump would fail the named test and break standalone self-host while Cloudflare edge deploys remain viable. + - **Entrypoint Binary Targeting & Test-Runtime Skew**: `bin/workerd-entrypoint.sh:40-55` explicitly targets wrangler's pinned dependency `$APP_ROOT/node_modules/wrangler/../workerd/bin/workerd` (`1.20260714.1`) to ensure runtime execution does not pick up the older nested `1.20260701.1` binary (which would refuse the config and crash-loop). This is a historical failure guard, not hypothetical defensive complexity: the entrypoint comment records that a first-match `find` can boot whichever workerd the filesystem lists first. Maintainers MUST NOT simplify the preferred-binary block back to first-match discovery. Additionally, note that `@cloudflare/vitest-pool-workers` executes workers-pool tests against the older `1.20260701.1` binary (a 13-day skew from the `1.20260714.1` deployment runtime); while `config.capnp` is not loaded in vitest, test behavior depending on workerd engine changes between 2026-07-01 and 2026-07-14 is not exercised by the 196 workers-pool tests. Operator host `workerd` binary MUST be dated `>= 2026-07-14` (pinned release: `1.20260714.1`). ### 7.2 Durable Object On-Disk Storage Backup From 523c9807695a62503fcc3caa3292b9fdc08b0450 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 15:55:49 +0800 Subject: [PATCH 16/95] docs: cross-reference upgrade automation guards --- docs/migration/PROD_UPGRADE_PLAN.md | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 6c497e7e..315ae94b 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -344,6 +344,7 @@ Note on configuration variables & staging deployment safety: - The Phase 1 branch (`.worktrees/phase1-lifecycle/packages/worker/wrangler.toml`) contains no `ETHERCALC_AUTH` in `[env.staging.vars]`, so Phase 1 on staging is passkey-off by construction. - Conversely, Phase 2 for production requires a release branch (`release/phase2-rollout`) whose `packages/worker/wrangler.toml` sets `ETHERCALC_AUTH = "0"` in both `[vars]` and `[env.staging.vars]` (see §4.3). Staging rehearsal MUST deploy from this release branch. - **CRITICAL WARNING [EMPIRICALLY VERIFIED]**: When `.wrangler/deploy/config.json` exists, running `wrangler deploy --env staging` without `--config wrangler.toml` does **NOT** throw an error — Wrangler silently falls back to the top-level production configuration in `dist/ethercalc/wrangler.json` (which drops `[env.staging]`), binding directly to the **production D1 database (`ethercalc_rooms`)** and **production WebAuthn RP ID (`ethercalc.net`)**. Operators MUST ALWAYS pass `--config wrangler.toml --env=staging` to guarantee staging database and domain isolation. + > **PARTIALLY AUTOMATED** — Root test `nightly staging validation bypasses the generated production config` (`scripts/vite-workflow.test.ts:458-469`; root `vp run test`) pins the nightly command to `--config wrangler.toml --env staging` and rejects its unqualified form. It does not protect an operator's ad hoc CLI command; the explicit `--config` live-deploy check remains load-bearing. - **WARNING**: An ambient exported shell variable (e.g., `ETHERCALC_AUTH="0" wrangler deploy`) has no effect on deployed Worker vars because Wrangler resolves Worker variables from `wrangler.toml`, not the ambient process environment. - **[OPERATOR-VERIFY]**: An operator MAY use a `wrangler deploy --var ETHERCALC_AUTH:0` override as an alternative, provided they run `wrangler deploy --help` first to verify that `--var` exists in the pinned Wrangler CLI version (it is not confirmed in published Wrangler CLI documentation). @@ -367,38 +368,48 @@ Execute the following verification checklist against `https://ethercalc-staging. 1. **Legacy Un-migrated Room Handling**: - Access a room created prior to the upgrade (or a newly generated room without `meta:access`/`meta:acl` DO storage keys). - Verify `authorize()` in `packages/worker/src/lib/authorize.ts:20-22` evaluates `access == null` as `'public'`, returning HTTP 200 and allowing full read/write access (proven by `packages/worker/test/room.test.ts:257-353`). + > **PARTIALLY AUTOMATED** — `legacy room without meta:access or meta:acl storage keys remains fully accessible to anonymous users` (`packages/worker/test/room.test.ts:258-353`; worker `test:workers`, CI `test`) proves anonymous HTTP, WebSocket, and write access without either key. Staging still proves that the deployed artifact can read an actual pre-upgrade DO. 2. **Legacy `?auth=` Query Parameter Link**: - Request `https://ethercalc-staging.audreyt.workers.dev/testroom?auth=legacysecret`. - Verify room opens without 500 error or authentication breakdown (`auth=0` remains view-only). + > **PARTIALLY AUTOMATED** — The same workers-pool test (`room.test.ts:314-353`; worker `test:workers`, CI `test`) proves the legacy `auth=0` WebSocket path is view-only, while `packages/e2e/tests/redirects.spec.ts:52-62` (CI `e2e`) preserves generated `?auth=` links. Neither assertion opens this arbitrary legacy-secret URL against the deployed staging route. 3. **Form/App-Mode Room Hydration**: - Open a form-mode sheet (`ss.formDataViewer` active). - Verify that the new client bundle (`boot.ts:384-390`) sends `ask.log` for the main room in addition to `_formdata`, successfully hydrating the sheet grid. + > **PARTIALLY AUTOMATED** — `requests formdata then main-room log when a formDataViewer is present` (`packages/client/test/boot.test.ts:183-200`; client `test:coverage`, CI `test`) asserts both `ask.log` calls in order. The browser check adds proof that the built client shipped and hydrates through the live staging Worker. 4. **Large Command Batch & Limit Enforcement**: - Submit a command batch exceeding `MAX_SHEET_CELLS = 200_000` (e.g. `POST /_/staging-limit-test` expanding dimensions to A1:Z10000 = 260,000 cells). - Verify the HTTP response is `413 Payload Too Large` with `Content-Type: text/plain; charset=utf-8` and the exact lowercase body `command exceeds sheet limits` (`packages/worker/src/room.ts:703`, matching local Worker baseline in §5 Probe 6), while the WebSocket closes with `1008` and the distinct capitalized reason `Command exceeds sheet limits` (`packages/worker/src/room.ts:1805`). + > **PARTIALLY AUTOMATED** — `routes-rooms.node.test.ts:810-825` asserts the route's exact HTTP 413, MIME, and lowercase body; `room.node.test.ts:3118-3139` asserts WebSocket close 1008 with the capitalized reason (worker `test:coverage`, CI `test`). Staging adds end-to-end proof for the deployed artifact and binding path. 5. **Legacy `/socket.io/*` Transport Shim**: - Request `curl -fsS "https://ethercalc-staging.audreyt.workers.dev/socket.io/1/?t=$(date +%s)"`. - Verify response returns HTTP 200, `Content-Type: text/plain; charset=utf-8`, and body matching `<32-hex-session-id>:60:60:websocket,xhr-polling` (matching local Worker baseline in §5 Probe 8). + > **PARTIALLY AUTOMATED** — `GET /socket.io/1/ returns a colon-delimited handshake body` (`packages/worker/test/legacy-socketio.test.ts:35-46`; worker `test:workers`, CI `test`) asserts 200, text/plain, a 32-hex SID, and WebSocket transport. It only requires positive timeout fields, not the runbook's exact `60:60`, and does not exercise staging routing. 6. **XLSX Import and Export**: - Upload a test `.xlsx` file via `POST /_/staging-xlsx-test` with header `Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet` (which decodes binary workbook data into `loadclipboard` + `paste A1 all` commands via the `xlsx-deferred` handler in `packages/worker/src/routes/rooms.ts:745`). - Export sheet via `GET /staging-xlsx-test.xlsx` (or the alternative valid export spelling `GET /_/staging-xlsx-test/xlsx`). Verify response returns `200 OK` with `Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`, binary body starting with ZIP signature bytes `PK\x03\x04`, and `Content-Disposition: attachment; filename="staging-xlsx-test.xlsx"`. Confirm downloaded file parses correctly and preserves cell data. - **CRITICAL WARNING**: The `GET /_/staging-xlsx-test.xlsx` spelling is **NOT** an export route. It targets a raw room named `staging-xlsx-test.xlsx` and returns `404 Not Found` with an empty `text/plain` body (matching local Worker baseline in §5 Probe 10a). An operator following the incorrect `/_/.xlsx` spelling during staging rehearsal would see a 404, erroneously conclude XLSX export is broken, and could halt an otherwise healthy cutover. + > **PARTIALLY AUTOMATED** — Worker tests assert XLSX import decoding (`routes-rooms-post.node.test.ts:167-218`) and export MIME, disposition, ZIP signature, and both valid routes (`exports.test.ts:151-172`); Playwright additionally imports real XLSX files (`packages/e2e/tests/landing-import.spec.ts:98-159`). Gates: worker `test:coverage`/`test:workers` and CI `e2e`. They do not assert the misleading `/_/.xlsx` 404 or the staging artifact. 7. **Phase 2 Passkey Disabled Check (`ETHERCALC_AUTH="0"`)**: - Query `curl -fsS https://ethercalc-staging.audreyt.workers.dev/_auth/whoami`. - Verify body returns `{"uid":null,"enabled":false}` and `POST /_/private` returns `401 Unauthorized`. - This single probe proves the kill switch actually shipped to staging with passkeys disabled and would have caught deploying staging with `ETHERCALC_AUTH="1"`. Local proof of kill-switch lockout behavior is in `packages/worker/test/routes-rooms.node.test.ts` (test named `keeps an existing private room locked when ETHERCALC_AUTH is off`). + > **PARTIALLY AUTOMATED** — `whoami reports enabled:false when ETHERCALC_AUTH is "0" or null` (`packages/worker/test/routes-auth.node.test.ts:232-251`) and private-room route tests (`routes-rooms.node.test.ts:925-1008`) prove the kill switch locally under worker `test:coverage` (CI `test`). This live probe adds the distinct, essential assertion that `ETHERCALC_AUTH="0"` actually shipped to staging. 8. **Phase 3 Passkey Enabled Check (`ETHERCALC_AUTH="1"`)**: - Query `curl -fsS https://ethercalc-staging.audreyt.workers.dev/_auth/whoami`. - Verify body returns `{"uid":null,"enabled":true}`. - Initiate passkey registration ceremony at `POST /_auth/register-init` and complete at `POST /_auth/register-complete`. - **CRITICAL NOTE**: Passkeys registered on staging are bound to WebAuthn RP ID `ethercalc-staging.audreyt.workers.dev` (`packages/worker/wrangler.toml:84`). They are **NOT portable** to production (`ethercalc.net`) because WebAuthn RP IDs strictly enforce exact domain matching. + > **PARTIALLY AUTOMATED** — `whoami reports the verified principal, anonymity, and availability` (`routes-auth.node.test.ts:201-229`; worker `test:coverage`) covers the enabled response, while `packages/e2e/tests/passkey-webauthn-real.spec.ts:194-407` (CI `e2e`) completes real virtual-authenticator registration and private-room authorization against a localhost RP. The manual ceremony proves the staging RP/origin and deployed credentials, which those local gates cannot. 9. **Asset Purge & Freshness Check**: - Request `curl -fsSI https://ethercalc-staging.audreyt.workers.dev/static/player.js`. - Confirm asset loads cleanly with `200 OK` and `Content-Type: text/javascript; charset=utf-8` (matching local Worker baseline in §5 Probe 4). - **Note**: Production Cloudflare Workers Assets edge may emit `Content-Type: application/javascript` or `text/javascript` depending on global edge MIME tables; operators should treat a MIME subtype mismatch as informational as long as HTTP status is `200 OK`. + > **PARTIALLY AUTOMATED** — `packages/worker/test/assets.test.ts:358-365` (worker `test:workers`, CI `test`) asserts that `/static/player.js` is served with 200, and `packages/e2e/tests/client-single-smoke.spec.ts:37-125` (CI `e2e`) proves the bundle boots and persists an edit. Neither checks staging edge-cache freshness or pins the MIME subtype. 10. **Cross-Room Index Gating (`/_rooms*`)**: - Request `curl -sS -i https://ethercalc-staging.audreyt.workers.dev/_rooms`. - On staging (`ETHERCALC_CORS="1"`), verify endpoint returns HTTP `403 Forbidden`, `Content-Type: text/plain; charset=utf-8`, and body matching exactly `_rooms not available with CORS` (30 bytes, no trailing newline, matching local Worker baseline in §5 Probe 9). + > **PARTIALLY AUTOMATED** — `GET /_rooms returns 403 when room index is disabled` (`packages/worker/test/routes-rooms.node.test.ts:606-631`) asserts both `ETHERCALC_DISABLE_ROOM_INDEX` and legacy `ETHERCALC_CORS="1"` paths plus the exact body; `room-index-access.node.test.ts:29-87` covers precedence (worker `test:coverage`, CI `test`). Staging still proves the intended vars shipped. --- @@ -429,6 +440,7 @@ Before running `vp exec wrangler deploy --env=""` or `npx wrangler deploy` in `p `Using redirected Wrangler configuration.` 2. If a redirect exists locally, ALWAYS pass `--config wrangler.toml` explicitly (e.g. `npx wrangler deploy --config wrangler.toml --env=""`) to force Wrangler to read `wrangler.toml` directly and upload from `assets/`. 3. For staging deploys, ALWAYS pass `--config wrangler.toml --env staging` (e.g. `npx wrangler deploy --config wrangler.toml --env staging`) so Wrangler reads `wrangler.toml` directly and applies `[env.staging]` overlay bindings. +> **PARTIALLY AUTOMATED** — Root test `nightly staging validation bypasses the generated production config` (`scripts/vite-workflow.test.ts:458-469`; root `vp run test`) guarantees the checked-in nightly staging gate uses explicit source config. It does not inspect an operator workspace, production workflow command, or ad hoc deploy, so the redirect-banner check above still protects the artifact actually being uploaded. ### 4.1 Step 1: Pre-Deploy D1 Database Migrations @@ -612,6 +624,7 @@ npx wrangler versions deploy @100% --env="" # 6. MANDATORY EDGE CACHE PURGE: # Cloudflare Dashboard -> Caching -> Configuration -> Purge Everything ``` +> **PARTIALLY AUTOMATED** — The root nightly-config test (`scripts/vite-workflow.test.ts:458-469`; root `vp run test`) protects the checked-in staging dry run, worker auth tests (`routes-auth.node.test.ts:232-251`) protect the disabled response, and `build:assets` is a preflight gate. They do not upload this version, prove the flag/config selected by Wrangler, ramp live traffic, observe analytics, or purge the edge cache; those operator actions remain load-bearing. **ROLLBACK TARGET FOR PHASE 2**: Phase 2 contains ZERO DO lifecycle changes. **Phase 2 can be rolled back to Phase 1 instantly at any time via `npx wrangler versions deploy @100% --env=""`**. Because `ETHERCALC_AUTH = "0"`, no private rooms or passkeys can be created during Phase 2, making rollback completely safe and hazard-free. @@ -683,10 +696,12 @@ custom-domain redirects. curl -fsS -i https://ethercalc.net/_health # Expected [VERIFIED LOCALLY]: HTTP 200, Content-Type: application/json, body # `{"status":"ok","version":"0.0.0","now":""}` +# **PARTIALLY AUTOMATED** — `health.test.ts:6-19` (worker `test:workers`, CI `test`) and Playwright `health.spec.ts:9-24` (CI `e2e`) assert this status, MIME, and body shape locally. This production probe adds deployed-runtime and live-binding evidence; the tests do not identify the deployed SHA because `version` remains `0.0.0`. # Probe 2: Phase 2 Behavioral Probe (Verifies new code running with Passkeys OFF) curl -fsS -i https://ethercalc.net/_auth/whoami # Expected [VERIFIED LOCALLY]: HTTP 200, body `{"uid":null,"enabled":false}` +# **PARTIALLY AUTOMATED** — `routes-auth.node.test.ts:232-251` asserts this exact disabled response under worker `test:coverage` (CI `test`). The production response proves the Phase 2 flag shipped to the version serving traffic. # Probe 3: Anonymous Private Room Creation Check curl -sS -i -X POST https://ethercalc.net/_/private @@ -694,6 +709,7 @@ curl -sS -i -X POST https://ethercalc.net/_/private # (no trailing newline). This proves an anonymous caller cannot create a # private room. It does not by itself prove the flag is off: the same anonymous # request also returned 401 with ETHERCALC_AUTH="1"; Probe 2 proves the flag. +# **PARTIALLY AUTOMATED** — `routes-rooms.node.test.ts:925-956` asserts exact 401 `Unauthorized` both anonymously and with a session while auth is off (worker `test:coverage`, CI `test`). The live probe confirms the deployed route, but—as noted above—Probe 2 is what identifies the flag state. # Probe 4: Fresh Client Asset Probe (Verifies Cache Purge) curl -fsSI https://ethercalc.net/static/player.js @@ -701,6 +717,7 @@ curl -fsSI https://ethercalc.net/static/player.js # `Content-Type: text/javascript; charset=utf-8`. The local Workers Assets # binding also returned `Cache-Control: public, max-age=0, must-revalidate`; # production edge-cache freshness must still be checked after the purge. +# **PARTIALLY AUTOMATED** — `assets.test.ts:358-365` asserts a 200 asset response (worker `test:workers`, CI `test`), and Playwright `client-single-smoke.spec.ts:37-125` proves the shipped asset set boots locally (CI `e2e`). Only this live check can validate the post-purge production edge cache and its MIME table. # Probe 5: Public Sheet Read/Write Probe curl -fsS -i https://ethercalc.net/testprodcutover @@ -709,6 +726,7 @@ curl -fsS -i https://ethercalc.net/testprodcutover curl -sS -i -X POST https://ethercalc.net/_/testprodcutover -d "page-size: A4" # Expected [VERIFIED LOCALLY]: HTTP 202, Content-Type: # application/json; charset=utf-8, body `{"command":"page-size: A4"}` +# **PARTIALLY AUTOMATED** — Playwright `client-single-smoke.spec.ts:37-125` and `room-crud-export.spec.ts:4-204` cover a real local Worker public-sheet boot, edit, persistence, reload, and command API (CI `e2e`). This probe confirms the production route and Durable Object deployment. # Probe 6: Sheet Limit Enforcement Probe # Use a fresh room so a rejected command is observable as an absent snapshot. @@ -726,6 +744,7 @@ curl -sS -i -X POST "https://ethercalc.net/_/${LIMIT_ROOM}" \ curl -sS -i "https://ethercalc.net/_/${LIMIT_ROOM}" # Expected: HTTP 404 with an empty body, independently proving the rejected # mutation did not create a snapshot in the fresh room. +# **PARTIALLY AUTOMATED** — `routes-rooms.node.test.ts:810-838` asserts 413 propagation and the 202 positive control; `room.node.test.ts:1110-1122` asserts rejection leaves storage empty (worker `test:coverage`, CI `test`). The fresh production-room sequence adds proof that the deployed route and RoomDO agree and that no production snapshot was created. # Probe 7: WebSocket Upgrade Probe curl --http1.1 --max-time 2 -sS -D - -o /dev/null \ @@ -739,11 +758,13 @@ curl --http1.1 --max-time 2 -sS -D - -o /dev/null \ # `Sec-WebSocket-Accept`. curl then exits 28 when --max-time closes the still # open socket; that timeout after the 101 is expected. This raw probe verifies # the upgrade, not the contents of a decoded snapshot frame. +# **PARTIALLY AUTOMATED** — `packages/worker/test/ws.test.ts:24-40` asserts 426 without Upgrade and 101 with a WebSocket handle (worker `test:workers`, CI `test`); Playwright `realtime-collab.spec.ts:75-152` proves real local WebSocket edits converge (CI `e2e`). The curl probe adds production proxy/edge upgrade evidence and an explicit `Sec-WebSocket-Accept` check. # Probe 8: Legacy socket.io Compatibility Probe curl -fsS -i "https://ethercalc.net/socket.io/1/?t=$(date +%s)" # Expected [VERIFIED LOCALLY]: HTTP 200, Content-Type: text/plain; charset=utf-8, # body `<32-hex-session-id>:60:60:websocket,xhr-polling` +# **PARTIALLY AUTOMATED** — `legacy-socketio.test.ts:35-46` asserts 200, text/plain, 32-hex SID, and WebSocket transport (worker `test:workers`, CI `test`), but only checks that both timeout fields are positive rather than exactly `60:60`. Production routing and the stricter response remain live checks. # Probe 9: Cross-Room Index Gate curl -sS -i https://ethercalc.net/_rooms @@ -751,6 +772,7 @@ curl -sS -i https://ethercalc.net/_rooms # text/plain; charset=utf-8, body exactly `_rooms not available with CORS` # (30 bytes, no trailing newline) when ETHERCALC_CORS="1" and # ETHERCALC_DISABLE_ROOM_INDEX is unset. +# **PARTIALLY AUTOMATED** — `routes-rooms.node.test.ts:606-631` asserts the two flag paths and exact 403 body, with precedence covered by `room-index-access.node.test.ts:29-87` (worker `test:coverage`, CI `test`). Production still confirms the intended vars are active. # Probe 10a: Documented XLSX Path curl -sS -i https://ethercalc.net/_/testprodcutover.xlsx @@ -766,12 +788,14 @@ curl -fsS -D - -o /dev/null https://ethercalc.net/testprodcutover.xlsx # body beginning with ZIP signature bytes `PK\x03\x04`, and # `Content-Disposition: attachment; filename="testprodcutover.xlsx"`. # The other valid export spelling is `/_/testprodcutover/xlsx`. +# **PARTIALLY AUTOMATED** — `exports.test.ts:151-172` asserts 200, XLSX MIME, exact disposition, ZIP signature, and both valid route forms (worker `test:workers`, CI `test`); Playwright `room-crud-export.spec.ts:157-186` parses the ZIP structure (CI `e2e`). The manual probe confirms that export behavior is present in production. # ─── PHASE 3 PROBES (ETHERCALC_AUTH = "1") ─────────────────────────── # Probe 11: Phase 3 Behavioral Probe (Verifies Passkeys ON) curl -fsS -i https://ethercalc.net/_auth/whoami # Expected [VERIFIED LOCALLY]: HTTP 200, body `{"uid":null,"enabled":true}` +# **PARTIALLY AUTOMATED** — `routes-auth.node.test.ts:201-229` asserts the enabled anonymous response (worker `test:coverage`, CI `test`), and `passkey-webauthn-real.spec.ts:194-407` performs a real localhost-RP ceremony (CI `e2e`). This live response proves the Phase 3 flag and bindings reached production; it does not replace a production passkey ceremony. # Probe 12: WebAuthn www Alias Redirect Probe curl -fsSI https://www.ethercalc.net/_auth/register-init @@ -903,6 +927,7 @@ For self-hosted deployments running EtherCalc via Docker Compose or Kubernetes/H 2. **Base Image & Workerd Bundle Verification (`[SOURCE-VERIFIED]`)**: - `Dockerfile:22` pins base image `FROM oven/bun:1.3.14`. - `Dockerfile:60` builds the standalone ES module bundle via `scripts/build-workerd-bundle.sh`, which runs `bunx wrangler deploy --dry-run --outdir=packages/worker/workerd/worker/` to produce `packages/worker/workerd/worker/index.js` alongside checked-in `packages/worker/workerd/config.capnp`. + > **PARTIALLY AUTOMATED** — CI `build:selfhost` runs `scripts/smoke-selfhost.sh`, which builds this Dockerfile, rejects leaked `.dev.vars`/generated Wrangler config, boots the resulting workerd container, and exercises health plus persistent CRUD (`scripts/smoke-selfhost.sh:30-141`). That proves the checked-in image path builds and runs, not that the operator selected or pulled the intended release image. 3. **Compatibility Date Lockstep & Pinned Version Check (`[SOURCE-VERIFIED]`)**: - `packages/worker/wrangler.toml:3` specifies `compatibility_date = "2026-07-21"`. - `packages/worker/workerd/config.capnp:61` specifies `compatibilityDate = "2026-07-14"`. @@ -911,6 +936,7 @@ For self-hosted deployments running EtherCalc via Docker Compose or Kubernetes/H - **Hoisted / Production Dependency (`wrangler@4.112.0`)**: `workerd@1.20260714.1` (and all 5 platform packages at `1.20260714.1`), which decodes to release date `2026-07-14`. - **Nested Test Dependency (`@cloudflare/vitest-pool-workers@0.18.0`)**: `workerd@1.20260701.1` under `miniflare/workerd` and `wrangler/workerd` (and all 5 platform packages at `1.20260701.1`), which decodes to release date `2026-07-01` (`2026-07-01 < 2026-07-14`). - **Zero-Margin Lockstep Invariant — CI-Guarded**: The lockstep condition `2026-07-14 <= 2026-07-14` holds with **zero margin of slack**. Bumping `config.capnp`'s `compatibilityDate` by even a single day without simultaneously updating the pinned `workerd` dependency would make standalone `workerd serve` hard-reject the config. This mismatch is guarded by the root test `standalone compatibility date does not exceed the pinned workerd release` (`scripts/vite-workflow.test.ts:440-456`): after extracting the hoisted workerd release date from `bun.lock` and the configured date from `config.capnp`, `scripts/vite-workflow.test.ts:455` asserts `expect(Number(configuredDate)).toBeLessThanOrEqual(Number(workerdDate));`. The test runs through the root `test` script (`package.json:83`) under preflight Gate 3; the recorded Gate 3 run passed all 36 root tests (`docs/migration/PREFLIGHT_RESULTS.md:94-101`). Therefore, a mismatched date fails fast in CI/preflight rather than becoming a mystery production crash-loop, provided that gate is honored. Conversely, updating `workerd` alone is safe (it widens the margin). The deployment asymmetry remains critical: `wrangler.toml`'s `compatibility_date = "2026-07-21"` is 7 days ahead and safe because Wrangler clamps with a warning, but syncing `config.capnp` to `"2026-07-21"` without a workerd bump would fail the named test and break standalone self-host while Cloudflare edge deploys remain viable. + > **ALREADY AUTOMATED — skim after a green root test gate.** `standalone compatibility date does not exceed the pinned workerd release` (`scripts/vite-workflow.test.ts:440-456`; root `vp run test`) parses the real `bun.lock` and `config.capnp` values and asserts `configuredDate <= workerdDate`. Recomputing that inequality manually adds no live-deployment evidence. - **Entrypoint Binary Targeting & Test-Runtime Skew**: `bin/workerd-entrypoint.sh:40-55` explicitly targets wrangler's pinned dependency `$APP_ROOT/node_modules/wrangler/../workerd/bin/workerd` (`1.20260714.1`) to ensure runtime execution does not pick up the older nested `1.20260701.1` binary (which would refuse the config and crash-loop). This is a historical failure guard, not hypothetical defensive complexity: the entrypoint comment records that a first-match `find` can boot whichever workerd the filesystem lists first. Maintainers MUST NOT simplify the preferred-binary block back to first-match discovery. Additionally, note that `@cloudflare/vitest-pool-workers` executes workers-pool tests against the older `1.20260701.1` binary (a 13-day skew from the `1.20260714.1` deployment runtime); while `config.capnp` is not loaded in vitest, test behavior depending on workerd engine changes between 2026-07-01 and 2026-07-14 is not exercised by the 196 workers-pool tests. Operator host `workerd` binary MUST be dated `>= 2026-07-14` (pinned release: `1.20260714.1`). ### 7.2 Durable Object On-Disk Storage Backup @@ -935,6 +961,7 @@ Before restarting containers, back up the local DO storage directory. ### 7.3 Mandatory Nginx Reverse Proxy Requirement Internet-facing self-host installations MUST run the nginx proxy recipe (`deploy/nginx/ethercalc.conf`) for rate-limiting and client IP header sanitization (overwriting client-supplied `CF-Connecting-IP`, `X-Forwarded-For`, and `X-Forwarded-Proto` with `$remote_addr` and `$scheme`). Note: CSP headers are injected directly by the Worker application layer (`packages/worker/src/lib/csp.ts`), not by Nginx. Validate basic proxy syntax, routing, health, room-index gating, and WebSocket upgrades with `./scripts/smoke-proxy.sh`. +> **PARTIALLY AUTOMATED** — CI `build:selfhost` runs `scripts/smoke-proxy.sh`, which asserts `nginx -t`, proxied health/root success, `/_rooms` 403, and a 101 WebSocket upgrade (`scripts/smoke-proxy.sh:29-85`). A self-host operator still must prove that this proxy is the public ingress; the script does not exercise rate-limit thresholds, header replacement semantics, TLS, or the operator's deployed stack. ### 7.4 Security Environment Variables & Passkey Anchors Self-host operators MUST configure passkey environment variables to enable passkey features. @@ -989,6 +1016,7 @@ Self-host operators MUST configure passkey environment variables to enable passk #### 2. What an Operator Must Set to Enable Passkeys To light up passkey authentication and private rooms, the operator MUST provide all four WebAuthn trust anchors matching their exact public HTTPS origin: +> **PARTIALLY AUTOMATED** — `check-helm-hardening.sh:41-61` (CI `helm-lint`) asserts all Helm trust anchors render together and incomplete enablement fails closed; worker `routes-auth.node.test.ts:160-251` (CI `test`) asserts disabled/enabled route behavior. Automation cannot decide whether an operator's RP ID and origin match the real public HTTPS host. - **Docker Compose Operator Overrides (`.env` or shell export)**: @@ -998,6 +1026,7 @@ To light up passkey authentication and private rooms, the operator MUST provide ETHERCALC_RP_NAME="Example Sheets" ETHERCALC_ORIGIN="https://sheets.example.com" ``` + > **PARTIALLY AUTOMATED** — Worker auth tests cover the resulting flag/binding semantics, but no Docker smoke enables passkeys or validates these Compose-supplied values. The operator must confirm the concrete environment and public origin. - **Kubernetes / Helm Operator Overrides (`values.yaml` or `--set`)**: @@ -1012,6 +1041,7 @@ To light up passkey authentication and private rooms, the operator MUST provide secrets: key: "openssl-rand-hex-32-value" ``` + > **PARTIALLY AUTOMATED** — `check-helm-hardening.sh:41-61` (CI `helm-lint`) proves these Helm values render all four Worker variables and rejects incomplete anchors. It does not apply the chart to a cluster or validate the chosen RP/origin and secret there. - **Consequence of Leaving Unset**: `authEnabled()` in `packages/worker/src/routes/auth.ts:34-43` checks `flagEnabled(env.ETHERCALC_AUTH) && env.AUTH !== undefined && env.ETHERCALC_RP_ID && env.ETHERCALC_ORIGIN`. If passkey variables are left unset, `authEnabled()` returns `false` (fails closed), returning HTTP 404 for `/_auth/*` endpoints and HTTP 401 for private room creation. - **Room Index Access Fallback**: `shouldDisableRoomIndex()` in `packages/worker/src/lib/room-index-access.ts:40-46` checks `ETHERCALC_DISABLE_ROOM_INDEX` first; if absent, it falls back to legacy `ETHERCALC_CORS`. Defaults to `1` (gated) in self-host manifests. @@ -1020,6 +1050,7 @@ To light up passkey authentication and private rooms, the operator MUST provide bash scripts/check-helm-hardening.sh ./scripts/smoke-selfhost.sh ``` + > **ALREADY AUTOMATED — skim manual reruns when the required CI jobs are green.** CI `helm-lint` runs `check-helm-hardening.sh` (`.github/workflows/ci.yml:138-171`), and CI `build:selfhost` runs both `smoke-selfhost.sh` and `smoke-proxy.sh` (`ci.yml:119-136`). Re-running the same scripts against the same tree adds no live-cluster evidence; validating the operator's deployed stack is a separate manual responsibility. ### 7.5 Verification Status & Local Execution Gate Note @@ -1073,14 +1104,29 @@ The following five items have been evaluated and categorized: ## §9 Go / No-Go Checklist This checklist has nine conditions and spans the full cutover. Before executing Phase 1 (`npx wrangler deploy --env=""`), items 1–7 MUST be satisfied; items 5 and 6 are already complete and therefore intentionally checked below. Item 8 gates the Phase 2 upload, and item 9 gates Phase 3. +### 9.1 Manual-Attention Calibration + +The audit unit is one `[OPERATOR-VERIFY]` site, numbered §3.2 check, §5 probe +(`10a` and `10b` counted separately), actionable §7 check, or §9 condition: +**47 checks total — 5 ALREADY AUTOMATED, 28 PARTIALLY AUTOMATED, and 14 +GENUINELY MANUAL**. Thus **33/47 are automation-backed**, but 28 of those still +add a distinct live-artifact, live-binding, edge, or deployment assertion. +`ALREADY AUTOMATED` items may be skimmed after their named gate is green; +`PARTIALLY AUTOMATED` items must retain the stated live delta; the remaining +14 demand credentials, backups, operational judgment, or soak observation. + - [ ] **1. Baseline Capture & Subsystem Verification**: `wrangler deployments list`, `wrangler versions list`, and `wrangler d1 info ethercalc_rooms --json` executed and recorded, confirming D1 storage subsystem `version: "production"` for Time Travel availability (§0.2, §0.2.1). - [ ] **2. Preflight Gates Green Against Final Tree (9/9 Runnable Gates Verified; 2 Docker Smokes Pending CI)**: All 9 locally-runnable preflight gates (`vp run typecheck`, `vp lint`, `vp run test`, worker `test:node` & `test:workers`, worker 100% coverage gate `test:coverage`, `build:assets` + `e2e#test`, `build:dry`, `check-helm-hardening.sh`, and `ratchet-verify.sh`) passed 100% green against the final tree state including the `rooms.ts` command-rejection status propagation fix (§1.2, `docs/migration/PREFLIGHT_RESULTS.md`). The 2 Docker smoke gates (`./scripts/smoke-selfhost.sh`, `./scripts/smoke-proxy.sh`) remain unverified locally due to missing local `docker compose` CLI subcommand and require CI execution before final cutover. + > **ALREADY AUTOMATED — inspect gate status rather than re-performing its assertions.** The root/worker/client gates run the named test and coverage commands; CI `test`, `e2e`, and `helm-lint` execute their corresponding suites, while CI `build:selfhost` runs both Docker smokes (`.github/workflows/ci.yml:17-235`). Once those required jobs are green on the final tree, repeating the same local assertions adds no deployment evidence. - [ ] **3. Secrets Provisioned in Production**: `wrangler secret list` confirms `ETHERCALC_KEY` and `ETHERCALC_MIGRATE_TOKEN` are active in Cloudflare Secrets (§0.1). - [ ] **4. D1 Database Export & Time Travel Bookmark Recorded**: `npx wrangler d1 export ethercalc_rooms --remote --output=...` executed (using required `--output` flag), account plan confirmed (30d Paid / 7d Free retention), and Time Travel bookmark timestamp captured (§2.1, §6.4). - [x] **5. Phase 1 Branch (Forward-Fix Artifact) Prepared**: `release/phase1-lifecycle` branch built, typechecks, and dry-runs cleanly (§4.2, §6.1, and §8 item 1; verified via `vp run @ethercalc/worker#typecheck` and `vp run @ethercalc/worker#build:dry`). + > **ALREADY AUTOMATED — this preparation condition adds no live check.** The named worker `typecheck` and Wrangler `build:dry` gates compile and bundle the Phase 1 tree; their recorded green outputs are the condition itself. Deployment and soak are intentionally separate items 8 and 9. - [x] **6. PR 4 Command-Rejection Propagation Landed and Verified for Phase 2**: `POST /_/:room` returns the RoomDO status and body for every non-2xx verdict, matching `PUT /_/:room` at `packages/worker/src/routes/rooms.ts:355-369`. Verified via route contract tests `POST /_/:room command mutations propagate a DO 413 sheet-limit verdict` and `POST /_/:room returns 202 command echo on successful DO dispatch` in `packages/worker/test/routes-rooms.node.test.ts` (52 node files / 1520 tests; 13 workers-pool files / 196 tests; §8 item 5; §10 item 3). + > **ALREADY AUTOMATED — skim after the worker coverage gate.** `routes-rooms.node.test.ts:810-838` asserts exact 413 propagation plus the 202 success path under worker `test:coverage` (CI `test`), while `room.node.test.ts:1110-1122` asserts RoomDO rejection and no write. This checklist line contains no additional live assertion. - [ ] **7. Staging Rehearsal Passed**: Phase 1, Phase 2, and Phase 3 deployed and verified on `https://ethercalc-staging.audreyt.workers.dev` (§§3.1–3.2). + > **PARTIALLY AUTOMATED** — Every §3.2 behavior has a named local guard, and root test `nightly staging validation bypasses the generated production config` pins the nightly dry-run command. Only the rehearsal proves all three deployed artifacts, staging bindings/RP, existing DO state, and edge behavior together; use the per-item §3.2 deltas rather than treating local green as a substitute. - [ ] **8. Phase 1 Deployed & Soaked Before Phase 2 Upload**: Phase 1 (`npx wrangler deploy`) executed and verified stable before uploading the Phase 2 gradual release (§4.2 and §4.3). - [ ] **9. Phase 2 Soaked with `ETHERCALC_AUTH="0"` Before Phase 3**: Phase 2 fully rolled out to 100% and soaked before activating passkeys in Phase 3 (§4.3 and §4.4). From b71874d8bdb03f1dc7df52b766fddb30433fe09b Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 16:00:50 +0800 Subject: [PATCH 17/95] test(worker): pin player.js MIME and socket.io handshake wire shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Module scripts require an exact JS Content-Type, and Socket.IO 0.x clients parse handshake fields positionally — pin both contracts that real clients depend on. --- packages/worker/test/assets.test.ts | 5 ++++- packages/worker/test/legacy-socketio.test.ts | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/worker/test/assets.test.ts b/packages/worker/test/assets.test.ts index ff9a3a1c..55f75e67 100644 --- a/packages/worker/test/assets.test.ts +++ b/packages/worker/test/assets.test.ts @@ -111,7 +111,7 @@ function buildFullStub(): { ["/manifest.json", { body: "{}", contentType: "application/json" }], ["/manifest.appcache", { body: "CACHE MANIFEST\n", contentType: "text/cache-manifest" }], ["/static/socialcalc.js", { body: "/* SocialCalc */", contentType: "application/javascript" }], - ["/static/player.js", { body: "/* player */", contentType: "application/javascript" }], + ["/static/player.js", { body: "/* player */", contentType: "application/octet-stream" }], ["/formbuilder.js", { body: "/* form builder */", contentType: "application/javascript" }], ["/l10n/en.json", { body: '{"lang":"en"}', contentType: "application/json" }], ["/l10n/de.json", { body: '{"lang":"de"}', contentType: "application/json" }], @@ -360,6 +360,9 @@ describe("GET /static/player.js", () => { const stub = buildFullStub(); const res = await call("/static/player.js", { ASSETS: stub.ASSETS }); expect(res.status).toBe(200); + // DiskDirectory/standalone workerd emits octet-stream; serveAsset/mimeForPath + // rewrites .js to application/javascript so module-script MIME checks pass. + expect(res.headers.get("Content-Type")).toBe("application/javascript; charset=utf-8"); expect(await res.text()).toBe("/* player */"); expect(stub.calls.at(-1)?.pathname).toBe("/static/player.js"); }); diff --git a/packages/worker/test/legacy-socketio.test.ts b/packages/worker/test/legacy-socketio.test.ts index 6003118b..bc4fd45f 100644 --- a/packages/worker/test/legacy-socketio.test.ts +++ b/packages/worker/test/legacy-socketio.test.ts @@ -38,12 +38,14 @@ describe('legacy socket.io shim — routes', () => { expect(res.headers.get('content-type')).toMatch(/text\/plain/); const body = await res.text(); // `:::` + // Defaults from createSocketIoShim (60/60) + DEFAULT_TRANSPORTS. + // Socket.IO 0.x clients parse these fields positionally — pin exact shape. const parts = body.split(':'); expect(parts).toHaveLength(4); - expect(parts[0]!.length).toBe(32); // 32-char hex sid - expect(Number(parts[1])).toBeGreaterThan(0); - expect(Number(parts[2])).toBeGreaterThan(0); - expect(parts[3]!).toContain('websocket'); + expect(parts[0]).toMatch(/^[0-9a-f]{32}$/); + expect(parts[1]).toBe('60'); + expect(parts[2]).toBe('60'); + expect(parts[3]).toBe('websocket,xhr-polling'); }); it('GET /socket.io/1 (no trailing slash) also handshakes', async () => { From 1824aa454f3fd92958e36b5801fa049f2161ce18 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Mon, 10 Aug 2026 16:03:02 +0800 Subject: [PATCH 18/95] test(worker): pin both serveAsset MIME branches for player.js Hosted Cloudflare Assets pass text/javascript through; standalone workerd rewrites octet-stream via mimeForPath. Document the split in the upgrade runbook and refresh the now-pinned socket.io notes. --- docs/migration/PROD_UPGRADE_PLAN.md | 36 +++++++++++++++++++++-------- packages/worker/test/assets.test.ts | 26 ++++++++++++++++++--- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/docs/migration/PROD_UPGRADE_PLAN.md b/docs/migration/PROD_UPGRADE_PLAN.md index 315ae94b..8624553f 100644 --- a/docs/migration/PROD_UPGRADE_PLAN.md +++ b/docs/migration/PROD_UPGRADE_PLAN.md @@ -384,7 +384,7 @@ Execute the following verification checklist against `https://ethercalc-staging. 5. **Legacy `/socket.io/*` Transport Shim**: - Request `curl -fsS "https://ethercalc-staging.audreyt.workers.dev/socket.io/1/?t=$(date +%s)"`. - Verify response returns HTTP 200, `Content-Type: text/plain; charset=utf-8`, and body matching `<32-hex-session-id>:60:60:websocket,xhr-polling` (matching local Worker baseline in §5 Probe 8). - > **PARTIALLY AUTOMATED** — `GET /socket.io/1/ returns a colon-delimited handshake body` (`packages/worker/test/legacy-socketio.test.ts:35-46`; worker `test:workers`, CI `test`) asserts 200, text/plain, a 32-hex SID, and WebSocket transport. It only requires positive timeout fields, not the runbook's exact `60:60`, and does not exercise staging routing. + > **PARTIALLY AUTOMATED** — `GET /socket.io/1/ returns a colon-delimited handshake body` (`packages/worker/test/legacy-socketio.test.ts`; worker `test:workers`, CI `test`) asserts 200, text/plain, a `/^[0-9a-f]{32}$/` SID, exact `60:60` timeouts, and transports `websocket,xhr-polling`. It does not exercise staging routing. 6. **XLSX Import and Export**: - Upload a test `.xlsx` file via `POST /_/staging-xlsx-test` with header `Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet` (which decodes binary workbook data into `loadclipboard` + `paste A1 all` commands via the `xlsx-deferred` handler in `packages/worker/src/routes/rooms.ts:745`). - Export sheet via `GET /staging-xlsx-test.xlsx` (or the alternative valid export spelling `GET /_/staging-xlsx-test/xlsx`). Verify response returns `200 OK` with `Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`, binary body starting with ZIP signature bytes `PK\x03\x04`, and `Content-Disposition: attachment; filename="staging-xlsx-test.xlsx"`. Confirm downloaded file parses correctly and preserves cell data. @@ -403,9 +403,12 @@ Execute the following verification checklist against `https://ethercalc-staging. > **PARTIALLY AUTOMATED** — `whoami reports the verified principal, anonymity, and availability` (`routes-auth.node.test.ts:201-229`; worker `test:coverage`) covers the enabled response, while `packages/e2e/tests/passkey-webauthn-real.spec.ts:194-407` (CI `e2e`) completes real virtual-authenticator registration and private-room authorization against a localhost RP. The manual ceremony proves the staging RP/origin and deployed credentials, which those local gates cannot. 9. **Asset Purge & Freshness Check**: - Request `curl -fsSI https://ethercalc-staging.audreyt.workers.dev/static/player.js`. - - Confirm asset loads cleanly with `200 OK` and `Content-Type: text/javascript; charset=utf-8` (matching local Worker baseline in §5 Probe 4). - - **Note**: Production Cloudflare Workers Assets edge may emit `Content-Type: application/javascript` or `text/javascript` depending on global edge MIME tables; operators should treat a MIME subtype mismatch as informational as long as HTTP status is `200 OK`. - > **PARTIALLY AUTOMATED** — `packages/worker/test/assets.test.ts:358-365` (worker `test:workers`, CI `test`) asserts that `/static/player.js` is served with 200, and `packages/e2e/tests/client-single-smoke.spec.ts:37-125` (CI `e2e`) proves the bundle boots and persists an edit. Neither checks staging edge-cache freshness or pins the MIME subtype. + - Confirm asset loads cleanly with `200 OK` and a JavaScript `Content-Type` that browsers accept for ` -``` - -(`assets/index.html:27`) +## 6. Root HTML, normal cache, and AppCache skew -- DevMode can serve a dynamic always-dirty stub - (`routes/assets.ts` `/manifest.appcache`); production serves the static - file via ASSETS. +### 6.1 Actual layout delta -**Modern Chromium has removed AppCache.** This file does **not** force -old tabs to reload in current browsers. It is not a cutover control plane. +At `d2afa90`, `index.html`: -### 5.2 Cache headers +- declares ``; +- contains inline room/bootstrap, localization, resize, and chat-submit logic; +- loads the already-deployed passkey UI and `static/player.js`. -Global middleware (`packages/worker/src/index.ts:115-117`): +At `HEAD`, the manifest attribute is gone and page logic is extracted to five +tracked files introduced at `b7d8840`: -```ts -if (hasSession || isAuthRoute || hasAuthorization || isOperatorRoute) { - c.header('Cache-Control', 'private, no-store'); -} -``` +- `static/index-bootstrap.js`; +- `static/index-l10n.js`; +- `static/panels.js`; +- `static/start-bootstrap.js`; +- `static/start-page.js`. -That is **only** for session/auth/operator traffic — **not** anonymous -HTML or `/static/*`. +The target still ships `manifest.appcache` and all assets referenced by the old +root. The manifest blob itself is unchanged from `d2afa90`. -`serveAsset` (`routes/assets.ts:92-135`) rewrites `Content-Type` when -needed and **does not set `Cache-Control`**. HTML and `static/player.js` -therefore inherit **Cloudflare Workers Assets default caching** on the -hosted path. +### 6.2 Cached old root HTML against the new Worker -**MIME split for `/static/player.js`** (runbook §3.2 item 9 / §5 Probe 4; -`mimeForPath` / `MIME_BY_EXT` in `routes/assets.ts:54-78,122-130`): +A cached `d2afa90` root normally **does not fail with asset 404s** against the +new Worker. Its inline bootstraps need no extracted files, and the target still +serves `jquery.js`, `socialcalc.js`, passkey assets, `player.js`, localization, +styles, and images. Instead it prolongs a hybrid window: -- **Hosted Cloudflare Workers Assets** (production/staging / `wrangler dev`): - upstream already supplies a real type — empirically - `text/javascript; charset=utf-8`. `serveAsset` passes non- - `application/octet-stream` types through untouched. -- **Standalone workerd / Sandstorm `DiskDirectory`:** upstream is - `application/octet-stream` for every file; `serveAsset` rewrites `.js` - via `mimeForPath` to `application/javascript; charset=utf-8`. +- the DOM/page bootstrap remains old; +- unhashed external files may be old or new depending on browser/edge cache; +- the old `player.js` can continue speaking to the new stricter Worker; +- a reload is not proof that every cached resource changed unless cache is + purged or bypassed. -Either JavaScript subtype is fine for `