refactor(hub): run each local allocation in a sidecar process of its own

The local provisioner goes back to stock's model, as its own harness runs
it: ensure starts a sidecar process for the allocation and destroy stops it
and removes its files, so cleanup is always confirmed and no process keeps
the memory of deployments it no longer hosts. Each sidecar advertises one
active deployment. A late ensure for a destroyed identity is fenced in
memory, as the hub mints a fresh identity for every ensure after a restart.

Boot never respawns: it retires every sidecar an earlier boot listed, in
this layout or the single shared sidecar's, and removes their files. A
sidecar that exits on its own is retired at once. The provisioner refuses a
start past the configured number of sidecars, or while the box lacks the
memory for one beside those still connecting, as retryable capacity. The
stock provisioner conformance suite runs against it again.

Author: TheGreatAxios

apps/hub/src/server.ts

@@ -46,7 +46,7 @@ import { connectRailwaySidecars, sweepRailwaySidecarsOf, type RailwaySidecarProv
 import { createHostCapacity } from "./sidecar/host-capacity.js";
 import { createLocalSidecarManifestStore } from "./sidecar/manifest-store.js";
 import { createPlacementChooser, localSidecarUsage } from "./sidecar/placement.js";
-import { restoreLocalSidecars } from "./sidecar/restore.js";
+import { bootLocalSidecars } from "./sidecar/boot.js";
 import { createSpawnLocalSidecar } from "./sidecar/spawn.js";
 import { SIDECAR_ADAPTER_MANIFEST } from "./sidecar-config.js";
 import { createPortalHandler, isPortalRequest, withPortalCors } from "./portal.js";
@@ -163,17 +163,12 @@ const local = createLocalProcessSidecarProvisioner({
   spawnSidecar: createSpawnLocalSidecar({
     interchangeDir: V,
     credentialEncryptionKey: env.SIDECAR_CREDENTIAL_ENCRYPTION_KEY,
-    maxActiveDeployments: localSidecarCap,
     processEnv: childProcessEnv(env),
   }),
-  manifests: createLocalSidecarManifestStore({
-    dataRoot: localSidecarDataRoot,
-    cipher: credentialCipher,
-    log: logJson,
-  }),
+  manifests: createLocalSidecarManifestStore({ dataRoot: localSidecarDataRoot, log: logJson }),
   capacity: createHostCapacity({ cgroupRoot: "/sys/fs/cgroup", meminfo: "/proc/meminfo" }),
   adapterManifest: SIDECAR_ADAPTER_MANIFEST,
-  onAccepted: notePlaced,
+  maxSidecars: localSidecarCap,
   log: logJson,
 });
 const docker = await connectDockerSidecars({ docker: dockerSettings(env), database, adapterManifest: SIDECAR_ADAPTER_MANIFEST });
@@ -247,6 +242,7 @@ const composition = await createInterchangeHub({
 });
 const stock = composition.server;
 composition.sidecarRouter.events.on("sidecar.allocated.connected", notePlaced);
+composition.sidecarRouter.events.on("sidecar.allocated.connected", local.connected);
 
 /** The hub's credentials, and the push that takes back from sidecars an App key a read moved to the hub. */
 const credentialDeps = {
@@ -261,7 +257,7 @@ const movedAppKeys: string[] = [];
 for (const tenantId of await githubTenants(composition.db)) {
   if (await moveAppKey(credentialDeps, tenantId)) movedAppKeys.push(tenantId);
 }
-await restoreLocalSidecars(composition.db, local);
+await bootLocalSidecars(composition.db, local);
 if (dockerUp) logJson({ level: "info", msg: "docker_sidecars_swept", removed: await sweepDockerSidecarsOf(composition.db, dockerUp) });
 async function tendE2b(sidecars: E2bSidecarProvisioner): Promise<void> {
   try {

apps/hub/src/sidecar/boot.ts

@@ -0,0 +1,20 @@
+import { schema, type DB } from "@intx/db";
+import { and, inArray, notInArray } from "drizzle-orm";
+import type { LocalProcessSidecarProvisioner } from "./local-provisioner.js";
+
+export async function bootLocalSidecars(db: DB["db"], local: LocalProcessSidecarProvisioner): Promise<void> {
+  async function stillOpen(allocationIds: readonly string[]): Promise<ReadonlySet<string>> {
+    if (allocationIds.length === 0) return new Set();
+    const ids = [...allocationIds];
+    const allocations = await db
+      .select({ id: schema.sidecarAllocation.id })
+      .from(schema.sidecarAllocation)
+      .where(and(inArray(schema.sidecarAllocation.id, ids), notInArray(schema.sidecarAllocation.status, ["released", "failed"])));
+    const probes = await db
+      .select({ id: schema.workflowProbe.id })
+      .from(schema.workflowProbe)
+      .where(and(inArray(schema.workflowProbe.id, ids), notInArray(schema.workflowProbe.status, ["succeeded", "failed"])));
+    return new Set([...allocations, ...probes].map((row) => row.id));
+  }
+  await local.boot({ stillOpen });
+}

apps/hub/src/sidecar/host-capacity.test.ts

@@ -52,8 +52,8 @@ test("without a cgroup limit MemAvailable decides, and a box reporting neither h
   expect(await createHostCapacity(await box({})).freeBytes()).toBeUndefined();
 });
 
-test("the sidecar needs one footprint free, and no figure never declines", () => {
-  expect(shortfall(SIDECAR_BYTES)).toBeUndefined();
-  expect(shortfall(SIDECAR_BYTES / 2)).toBe(`${SIDECAR_BYTES / 2 / MiB} MiB free, ${SIDECAR_BYTES / MiB} MiB needed`);
-  expect(shortfall(undefined)).toBeUndefined();
+test("each sidecar still starting holds one footprint, and no figure never declines", () => {
+  expect(shortfall(2 * SIDECAR_BYTES, 1)).toBeUndefined();
+  expect(shortfall(2 * SIDECAR_BYTES, 2)).toBe(`${(2 * SIDECAR_BYTES) / MiB} MiB free, ${(3 * SIDECAR_BYTES) / MiB} MiB needed`);
+  expect(shortfall(undefined, 100)).toBeUndefined();
 });

apps/hub/src/sidecar/host-capacity.ts

@@ -1,7 +1,9 @@
-// Whether this box has the memory to start the local sidecar: free memory in
+// Whether this box has the memory for one more local sidecar: free memory in
 // the container's cgroup (v2, else v1) where a limit is set, else Linux's
-// MemAvailable. A box with neither, such as a macOS dev machine, never
-// declines: os.freemem there leaves out memory the system would reclaim.
+// MemAvailable, less one footprint for each sidecar started that has not
+// connected yet, as its memory is not in use yet. A box with neither, such as
+// a macOS dev machine, never declines: os.freemem there leaves out memory the
+// system would reclaim.
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -67,8 +69,9 @@ export function createHostCapacity({ cgroupRoot, meminfo }: HostCapacityOpts): H
   };
 }
 
-/** Why there is no room to start the sidecar, or undefined when there is. */
-export function shortfall(free: number | undefined): string | undefined {
-  if (free === undefined || free >= SIDECAR_BYTES) return undefined;
-  return `${Math.floor(free / MiB)} MiB free, ${SIDECAR_BYTES / MiB} MiB needed`;
+/** Why there is no room for one more sidecar beside `starting` ones not connected yet, or undefined when there is. */
+export function shortfall(free: number | undefined, starting: number): string | undefined {
+  const needed = SIDECAR_BYTES * (starting + 1);
+  if (free === undefined || free >= needed) return undefined;
+  return `${Math.floor(free / MiB)} MiB free, ${needed / MiB} MiB needed`;
 }

apps/hub/src/sidecar/local-provisioner.conformance.test.ts

@@ -0,0 +1,61 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describeProvisionerConformance } from "@corbits/provisioner-test";
+import { createLocalProcessSidecarProvisioner } from "./local-provisioner.js";
+import { createLocalSidecarManifestStore } from "./manifest-store.js";
+
+type FakeProcess = {
+  readonly sidecarId: string;
+  running: boolean;
+};
+
+describeProvisionerConformance("local-process", async () => {
+  const dataRoot = await fs.mkdtemp(path.join(os.tmpdir(), "local-provisioner-conformance-"));
+  const processes: FakeProcess[] = [];
+  const manifests = createLocalSidecarManifestStore({ dataRoot, log: () => {} });
+
+  const local = createLocalProcessSidecarProvisioner({
+    dataRoot,
+    capacity: { freeBytes: async () => undefined },
+    adapterManifest: [],
+    maxSidecars: 4,
+    spawnSidecar: function spawnSidecar({ sidecarId }) {
+      const process: FakeProcess = { sidecarId, running: true };
+      processes.push(process);
+      const exit = Promise.withResolvers<{ code: number | null; signal: string | null }>();
+      return {
+        pid: processes.length,
+        exited: exit.promise,
+        kill(signal) {
+          process.running = false;
+          exit.resolve({ code: null, signal });
+        },
+      };
+    },
+    manifests,
+    log: () => {},
+  });
+  local.provisioner.attach?.({ sidecarStopped: async () => ({ kind: "retired" }) });
+  await local.boot({ stillOpen: async (ids) => new Set(ids) });
+
+  return {
+    provisioner: local.provisioner,
+    async live(allocationId) {
+      let manifest: { generation: number; sidecarId: string };
+      try {
+        manifest = JSON.parse(await fs.readFile(path.join(dataRoot, "manifests", `${allocationId}.json`), "utf8"));
+      } catch {
+        return [];
+      }
+      return processes
+        .filter((process) => process.running && process.sidecarId === manifest.sidecarId)
+        .map(({ sidecarId }) => ({ generation: manifest.generation, sidecarId }));
+    },
+    capabilityChecks: {},
+    async dispose() {
+      await local.shutdown();
+      await fs.rm(dataRoot, { recursive: true, force: true });
+    },
+  };
+});