Sandbox templates: content-keyed records, derived-image strategies, template forks, GC

.github/workflows/on-pull-request.yml

@@ -133,9 +133,6 @@ jobs:
       - name: Build container image
         run: docker build -t drukbox:validate .
 
-      - name: Verify Docker CLI runs in the image
-        run: docker run --rm drukbox:validate docker --version
-
   api-tests:
     name: Run API Tests (docker provider)
     runs-on: ubuntu-latest

AGENTS.md

@@ -10,6 +10,7 @@ a service, not a library.
 It owns:
 
 - Host records and lifecycle state in Postgres
+- Template records, async template builds, and their provider images
 - Inline provisioning in `POST /hosts`
 - Provider VM creation and deletion (exe.dev, AWS, Hetzner, Exoscale,
   local Docker, Docker Sandboxes)
@@ -18,8 +19,9 @@ It owns:
 - An SSH gateway for hosts of gateway providers (`python -m gateway.server`)
 - Account-bound exe.dev HTTP proxy resources
 
-Periodic maintenance runs as cron jobs: `python -m hosts.janitor` reaps
-expired hosts, `python -m hosts.pool` tops up the warm pool.
+Periodic maintenance runs as cron jobs: `python -m janitor` reaps expired
+hosts and abandoned, failed, or unused templates, and `python -m hosts.pool`
+tops up the warm pool.
 
 No backwards compatibility is required unless a caller contract is explicitly
 documented in this repo.
@@ -47,8 +49,10 @@ src/
   hosts/             # Host API, models, schemas, service, janitor, pool, auth
   gateway/           # SSH gateway for gateway-provider hosts
   http_proxies/      # HTTP proxy API, schemas, service, deps
+  janitor/           # Cron entry point that runs the host and template reapers
   providers/         # VM provider ABC, capabilities, registry, adapters
   networking/        # Network provider framework and Tailscale adapter
+  templates/         # Template API, models, service, and janitor
   conftest.py        # Test env defaults and database reset fixture
 alembic/             # Database migrations
 api-tests/           # Playwright black-box API tests

Dockerfile

@@ -1,7 +1,5 @@
 # syntax=docker/dockerfile:1.7
 
-FROM docker:29.6.2-cli AS docker-cli
-
 FROM python:3.11-slim AS python-runtime
 
 WORKDIR /app
@@ -31,8 +29,6 @@ RUN uv sync --frozen --no-dev --all-extras
 
 RUN useradd --system --no-create-home --uid 1001 appuser
 
-COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
-
 USER appuser
 
 EXPOSE 8780

alembic/env.py

@@ -6,6 +6,7 @@
 from core.database import Base
 from core.settings import get_settings
 from hosts import models  # noqa: F401
+from templates import models as template_models  # noqa: F401
 
 config = context.config
 

alembic/versions/0004_templates.py

@@ -0,0 +1,48 @@
+"""reusable provider templates
+
+Revision ID: 0004_templates
+Revises: 0003_host_public_key
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0004_templates"
+down_revision: str | None = "0003_host_public_key"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "templates",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("provider", sa.String(length=20), nullable=False),
+        sa.Column("base_image", sa.Text(), nullable=False),
+        sa.Column("setup_script_hash", sa.String(length=64), nullable=False),
+        sa.Column("setup_script", sa.Text(), nullable=False),
+        sa.Column("label", sa.Text(), nullable=False),
+        sa.Column("image", sa.Text(), nullable=False),
+        sa.Column("status", sa.String(length=32), nullable=False),
+        sa.Column("last_error", sa.Text(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_templates_provider_base_image_setup_script_hash",
+        "templates",
+        ["provider", "base_image", "setup_script_hash"],
+        unique=True,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_templates_provider_base_image_setup_script_hash",
+        table_name="templates",
+    )
+    op.drop_table("templates")

api-tests/tests/full-api.spec.js

@@ -10,13 +10,17 @@ const EXPECTED_OPENAPI_OPERATIONS = [
   "DELETE /http-proxies/{name}",
   "DELETE /http-proxies/{name}/hosts/{host_id}",
   "DELETE /hosts/{host_id}",
+  "DELETE /templates/{template_id}",
   "GET /doctor",
   "GET /hosts",
   "GET /hosts/{host_id}",
+  "GET /templates",
+  "GET /templates/{template_id}",
   "POST /http-proxies",
   "POST /http-proxies/{name}/hosts/{host_id}",
   "POST /hosts",
   "POST /hosts/{host_id}/renew",
+  "POST /templates",
 ];
 
 const HOST_KEYS = [
@@ -56,6 +60,7 @@ test.describe("Drukbox API", () => {
   let publicApi;
   let createdHost;
   let createdProxyName;
+  let createdTemplate;
   let config;
 
   test.beforeAll(async () => {
@@ -78,6 +83,12 @@ test.describe("Drukbox API", () => {
       } catch {}
     }
 
+    if (api && createdTemplate?.id) {
+      try {
+        await api.delete(`/templates/${createdTemplate.id}`);
+      } catch {}
+    }
+
     await api?.dispose();
     await badTokenApi?.dispose();
     await publicApi?.dispose();
@@ -272,6 +283,49 @@ test.describe("Drukbox API", () => {
     const missing = await expectJson(await api.get(`/hosts/${createdHost.id}`), 404);
     expect(missing.detail).toBe("host not found");
   });
+
+  test("template lifecycle: build, fork a host, delete", async () => {
+    test.setTimeout(config.hostActiveTimeoutMs * 2);
+
+    createdTemplate = await expectJson(
+      await api.post("/templates", {
+        data: { setup_script: "printf drukbox > /drukbox-template-marker\n" },
+      }),
+      202,
+    );
+    expect(createdTemplate.id).toMatch(UUID_PATTERN);
+    expect(createdTemplate.status).toBe("building");
+    expect(createdTemplate.image).toBe("");
+    expect(createdTemplate).not.toHaveProperty("setup_script");
+
+    const built = await pollUntil(
+      async () => {
+        const template = await expectJson(
+          await api.get(`/templates/${createdTemplate.id}`),
+          200,
+        );
+        return template.status === "building" ? null : template;
+      },
+      { timeoutMs: config.hostActiveTimeoutMs, message: "template build did not finish" },
+    );
+    expect(built.status, built.last_error).toBe("available");
+    expect(built.image).not.toBe("");
+
+    const forked = await expectJson(
+      await api.post("/hosts", {
+        data: { template: createdTemplate.id },
+        timeout: config.hostActiveTimeoutMs,
+      }),
+      201,
+    );
+    expect(forked.image).toBe(built.image);
+    expect(forked.status).toBe("active");
+    await expectStatus(await api.delete(`/hosts/${forked.id}`), 204);
+
+    await expectStatus(await api.delete(`/templates/${createdTemplate.id}`), 204);
+    await expectStatus(await api.get(`/templates/${createdTemplate.id}`), 404);
+    createdTemplate = null;
+  });
 });
 
 function expectHost(host) {

docs/api.md

@@ -13,6 +13,8 @@ Every endpoint except `GET /healthz` requires
 ## Endpoints
 
 - `POST /hosts` · `GET /hosts` · `GET /hosts/{id}` · `DELETE /hosts/{id}`
+- `POST /templates` · `GET /templates` · `GET /templates/{id}` ·
+  `DELETE /templates/{id}`
 - `POST /http-proxies` · `DELETE /http-proxies/{name}` ·
   `POST|DELETE /http-proxies/{name}/hosts/{host_id}`
 - `GET /doctor` — read-only dependency diagnostics

docs/architecture.md

@@ -29,13 +29,15 @@ that true:
 ```text
 hosts.api          HTTP request/response concerns only
 hosts.service      host lifecycle behavior (HostService)
+templates.api      template request/response concerns only
+templates.service  template build and delete behavior (TemplateService)
 providers/<name>   one package per VM provider
 networking/        network provider framework + Tailscale adapter
 core/              settings, database, exception base
 diagnostics/       /doctor orchestration
 ```
 
-Provider-specific logic never lives in route handlers; HTTP decisions
+Provider-specific logic never lives in route imagers; HTTP decisions
 never live in service methods. Provider exceptions (`Exe*Error`,
 `Aws*Error`, `Hetzner*Error`, `Tailscale*Error`) are translated at the
 package boundary into neutral exceptions from `providers.exceptions` and
@@ -71,14 +73,15 @@ the core settings knowing any provider exists.
 
 Not every provider supports every feature, and the host contract must
 not grow provider-shaped warts. Optional features are capability
-mix-ins: `HttpProxyCapability` declares the http-proxy surface and the
-exe provider implements it. `resolve_capability` narrows a specific
-provider instance to a capability — the default provider for
-account-bound operations, the host's own provider for host-bound ones
-— and raises the shared `CapabilityUnsupportedError` when that
-provider doesn't implement it, which the routes surface as a clear
-error. New provider-specific features should follow this pattern
-rather than widening `VMProvider` or the host schema.
+mix-ins: `HttpProxyCapability` declares the http-proxy surface, and
+`TemplateCapability` declares the template create and delete surface.
+`resolve_capability` narrows a specific provider instance
+to a capability — the default provider for account-bound operations,
+the host's own provider for host-bound ones — and raises the shared
+`CapabilityUnsupportedError` when that provider does not implement it,
+which the routes surface as a clear error. New provider-specific
+features must follow this pattern rather than widening `VMProvider`
+or the host schema.
 
 The review question that guards the whole design: *does this change
 leak a provider into the contract?*
@@ -96,6 +99,19 @@ successful key returns the original host instead of a duplicate.
 Caller `env` is stored for provisioning and never returned by the API;
 keys in `hosts.schemas.RESERVED_HOST_ENV_KEYS` are rejected.
 
+A template is a persistent provider image keyed by provider, base image,
+and setup-script hash. `POST /templates` creates a `building` record and
+returns `202 Accepted`. Callers poll until the template becomes
+`available` or `failed`. Templates outlive hosts. Each provider builds
+and deletes its own templates behind `TemplateCapability`.
+
+A host request can name an available template by its ID — the ID that
+the create returned. The template's image becomes the host image. An
+explicit `image`
+wins over the template, and the template wins over the provider default.
+Host creation never builds a missing or unavailable template. It returns
+a client error, and the caller decides when to build.
+
 Every host is a renewable lease. A create without `expires_at` gets
 `now + LEASE_DEFAULT_TTL`, so a host whose owner disappears lapses and
 self-reaps instead of leaking VM cost; an explicit `expires_at: null`
@@ -106,13 +122,18 @@ hosts renew — unclaimed warm-pool members belong to pool maintenance
 and refuse with `409`.
 
 Two maintenance commands run as cron jobs from the same image:
-`hosts.janitor` reaps expired and orphaned hosts, `hosts.pool` keeps a
-warm pool of pre-provisioned hosts per provider (`POOL_SIZES`, with
-`POOL_SIZE` as the default provider's target) to hide provider cold
-starts. Pool members are warmed with the provider's default image and
-size, so a request that customizes its host — `image`, `env`,
-`instance_type`, or `disk_gb` — always provisions fresh instead of
-claiming a warm host.
+
+- `janitor` reaps expired and orphaned hosts, marks abandoned template
+  builds `failed`, and deletes failed or unused templates.
+- `hosts.pool` keeps a warm pool of pre-provisioned hosts per provider
+  (`POOL_SIZES`, with `POOL_SIZE` as the default provider's target) to
+  hide provider cold starts.
+
+When you edit a template setup script, the hash changes. The old
+template ages out after its last lease. Pool members
+are warmed with the provider's default image and size, so a request that
+customizes its host — `image`, `env`, `template`, `instance_type`, or
+`disk_gb` — always provisions fresh instead of claiming a warm host.
 
 ## Diagnostics
 

docs/deploy.md

@@ -20,15 +20,17 @@ docker run --rm -p 8780:8780 --env-file drukbox.env "$IMAGE"
 docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/alembic upgrade head
 
 # Maintenance (cron, e.g. every 10-15 min)
-docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/python -m hosts.janitor
+docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/python -m janitor
 docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/python -m hosts.pool
 ```
 
-The janitor reaps expired and orphaned hosts. The pool maintainer
+The janitor reaps expired and orphaned hosts, marks abandoned template
+builds failed, keeps failed builds for diagnosis, and deletes failed or
+unused templates. The pool maintainer
 pre-provisions warm hosts per provider and only does anything when at
 least one provider has a warm target (`POOL_SIZES` / `POOL_SIZE`).
-Schedule both under your cron infrastructure (k8s `CronJob`, systemd
-timer) from the same image and env file.
+Schedule both under your cron infrastructure (k8s `CronJob`,
+systemd timer) from the same image and env file.
 
 Use Postgres in production (`postgresql+psycopg://...`). SQLite
 (`sqlite+aiosqlite:///./drukbox.db`) is for single-process demos and
@@ -100,9 +102,8 @@ socket is host-root-equivalent. Do not expose a docker-backed drukbox to
 untrusted callers.
 
 Janitor and pool one-off containers using the Docker provider need the
-same socket mount and socket-GID supplemental group. `DOCKER_HOST` remains
-available when the daemon is remote or rootless instead of exposed through
-`/var/run/docker.sock`.
+same socket mount and socket-GID supplemental group. `DOCKER_HOST` remains available when the daemon is remote or
+rootless instead of exposed through `/var/run/docker.sock`.
 
 ## Local microVMs with Docker Sandboxes
 
@@ -294,6 +295,9 @@ Core, optional:
 | `SERVICE_LABEL` | `drukbox` | Label stamped onto provider resources (VM tags, SG tags). |
 | `UVICORN_HOST` | `0.0.0.0` | API bind address. Set `127.0.0.1` to restrict to loopback. |
 | `PROVISIONING_GRACE_SECONDS` | `600` | Safety TTL on in-flight hosts so the janitor reaps row + VM if the client disconnects mid-provision. Must exceed the worst-case provision duration. |
+| `TEMPLATE_BUILD_TIMEOUT` | `3600` | Max age in seconds of an unfinished template build before the janitor marks it failed. |
+| `TEMPLATE_FAILED_RETENTION` | `86400` | Seconds that failed template records and diagnostics remain before the janitor deletes them. |
+| `TEMPLATE_UNUSED_TTL` | `1209600` | Seconds that an available template remains after its last use, or creation when never used. |
 | `LEASE_DEFAULT_TTL` | `86400` | Lease TTL in seconds for hosts created without an explicit `expires_at`, and the extension applied by an empty `POST /hosts/{id}/renew`. An explicit `expires_at: null` at create time opts out of expiry entirely. |
 | `IDEMPOTENCY_KEY_TTL_HOURS` | `24` | Retention period for successful `Idempotency-Key` mappings. |
 | `POOL_SIZES` | `{}` | Warm hosts to keep ready per provider, as JSON (e.g. `{"exe": 2, "hetzner": 1}`). Overrides `POOL_SIZE` for the providers it names. |
@@ -319,6 +323,9 @@ exe.dev provider:
 | --- | --- | --- |
 | `EXE_API_TOKEN` | — (required) | Bearer token for the exe.dev exec API. |
 | `EXE_DEFAULT_IMAGE` | — (required) | Image used when the caller omits `image`. |
+| `EXE_IMAGE_REGISTRY` | — | Repository prefix for derived template images. A VM created from this registry gets `--registry-auth` so exe.dev can pull a private image. |
+| `EXE_REGISTRY_USERNAME` | — | Username for the derived-template image registry. |
+| `EXE_REGISTRY_PASSWORD` | — | Password or token for the derived-template image registry. |
 | `EXE_API_URL` | `https://exe.dev` | API base URL. |
 | `EXE_API_TIMEOUT` | `30.0` | Timeout for exe.dev API calls. |
 | `EXE_BOOTSTRAP_SSH_TIMEOUT_SECONDS` | `30.0` | ssh-keyscan retry budget for a fresh exe.dev sandbox. |

docs/security.md

@@ -76,9 +76,10 @@ covered in [Networking](networking.md). The security-relevant summary:
 
 ## Secrets and in-VM metadata
 
-Provider tokens (`EXE_API_TOKEN`, `HETZNER_API_TOKEN`, Tailscale OAuth)
-and AWS credentials are read from the environment / the AWS SDK default
-chain and never written to the database or returned by the API. Caller
+Provider tokens (`EXE_API_TOKEN`, `EXE_REGISTRY_PASSWORD`,
+`HETZNER_API_TOKEN`, Tailscale OAuth) and AWS credentials are read from
+the environment / the AWS SDK default chain and never written to the
+database or returned by the API. Caller
 `env` is write-only: it is delivered to the VM but never echoed in any
 response, and reserved keys (`TAILSCALE_AUTHKEY`) are rejected at the
 schema.

pyproject.toml

@@ -10,8 +10,10 @@ packages = [
     "src/gateway",
     "src/hosts",
     "src/http_proxies",
+    "src/janitor",
     "src/networking",
     "src/providers",
+    "src/templates",
 ]
 exclude = ["**/tests", "**/tests/**"]
 
@@ -61,6 +63,7 @@ dependencies = [
     "sqlalchemy[asyncio]>=2.0",
     "uuid6>=2024.7.10",
     "uvicorn[standard]>=0.34",
+    "aiodocker>=0.27.0",
 ]
 
 [project.optional-dependencies]