From b9ac7972bc5d5470ae8622fab501d44c09763a7b Mon Sep 17 00:00:00 2001 From: Samuel Laferriere <9342524+samlaf@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:03:19 -0400 Subject: [PATCH] chore(seismic)!: replace nginx and certbot with an unprivileged Caddy The public proxy ran as root: the nginx master, certbot (a root Python process that talks to the network and rewrites nginx's config) and the nginx-ssl-setup script. In-guest root can read root_key out of the custodian's memory, so the most exposed process on the node should hold no privilege it does not need. Caddy takes and renews its certificate in-process, as User=caddy with only CAP_NET_BIND_SERVICE, from a static Caddyfile in the measured image. - certificates: Let's Encrypt, falling back to ZeroSSL, over TLS-ALPN-01 only, so nothing listens on :80; no admin endpoint, HTTP/3 off - routes match their exact path (trailing slash allowed) and forward to /, since every backend serves one endpoint whatever the path; any other path is a 404, and WebSocket upgrades reach /ws only - caddy.service: ordered after tdx-init and persistent-luks-setup and required by nothing, with NoNewPrivileges, ProtectSystem=strict, PrivateTmp, PrivateDevices and RestrictAddressFamilies; all Caddy state lives in /persistent/caddy, which replaces /persistent/nginx - caddy v2.11.7 built from source in mkosi.build, with a pinned Go (go1.26.9) fetched through GOTOOLCHAIN, since Debian's is too old; static and reproducible - enclave pin to 74a6185: Caddy pastes {$DOMAIN_NAME} into its config as raw text, so tdx-init's domain validation is what keeps an operator's domain from injecting Caddyfile syntax - python3, nginx and certbot leave the image - CI validates the Caddyfile with the image's own caddy and expects four source pins; image.json and the release notes record the caddy commit - README: the endpoint table describes Caddy, and /summit is read-only JSON-RPC (getDepositSignature is on summit's localhost admin listener) BREAKING CHANGE: existing nodes take a fresh certificate on their first boot on this image, which counts against Let's Encrypt's 5-a-week limit for an exact name set. Nothing answers on :80. Refs: SEI-629 --- .github/workflows/seismic.yml | 31 +++--- README.md | 18 ++-- modules/seismic/mkosi.build | 22 ++++ modules/seismic/mkosi.conf | 18 +--- .../seismic/mkosi.extra/etc/caddy/Caddyfile | 96 +++++++++++++++++ .../mkosi.extra/etc/nginx/node-template.conf | 99 ----------------- .../etc/seismic/tmpfiles-persistent.conf | 6 +- .../etc/systemd/system/caddy.service | 43 ++++++++ .../etc/systemd/system/certbot-renew.service | 13 --- .../etc/systemd/system/certbot-renew.timer | 10 -- .../systemd/system/nginx-ssl-setup.service | 14 --- .../mkosi.extra/usr/bin/nginx-ssl-setup | 100 ------------------ modules/seismic/mkosi.postinst | 5 +- modules/seismic/readme.md | 54 +++++----- modules/seismic/sources.yaml | 5 +- scripts/seismic/image_json.sh | 3 +- 16 files changed, 231 insertions(+), 306 deletions(-) create mode 100644 modules/seismic/mkosi.extra/etc/caddy/Caddyfile delete mode 100644 modules/seismic/mkosi.extra/etc/nginx/node-template.conf create mode 100644 modules/seismic/mkosi.extra/etc/systemd/system/caddy.service delete mode 100644 modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.service delete mode 100644 modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.timer delete mode 100644 modules/seismic/mkosi.extra/etc/systemd/system/nginx-ssl-setup.service delete mode 100644 modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup diff --git a/.github/workflows/seismic.yml b/.github/workflows/seismic.yml index c6ff4b8..4e8373f 100644 --- a/.github/workflows/seismic.yml +++ b/.github/workflows/seismic.yml @@ -93,14 +93,12 @@ jobs: bash -n \ modules/seismic/mkosi.build \ modules/seismic/mkosi.postinst \ - modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup \ modules/seismic/mkosi.extra/usr/bin/persistent-luks-setup \ scripts/seismic/founding_inputs.sh \ scripts/seismic/image_json.sh shellcheck -x \ modules/seismic/mkosi.build \ modules/seismic/mkosi.postinst \ - modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup \ modules/seismic/mkosi.extra/usr/bin/persistent-luks-setup \ scripts/seismic/founding_inputs.sh \ scripts/seismic/image_json.sh @@ -113,8 +111,8 @@ jobs: source = pathlib.Path("modules/seismic/sources.yaml").read_text() pins = re.findall(r"^\s+git_reference:\s*([0-9a-f]+)\s*$", source, re.MULTILINE) - if len(pins) != 3 or any(len(pin) != 40 for pin in pins): - raise SystemExit("sources.yaml must contain three full 40-character git pins") + if len(pins) != 4 or any(len(pin) != 40 for pin in pins): + raise SystemExit("sources.yaml must contain four full 40-character git pins") build: name: build Seismic image @@ -169,11 +167,10 @@ jobs: exit 1 } done <<'EOF' + etc/caddy/Caddyfile etc/systemd/system/attestation.service - etc/systemd/system/certbot-renew.service - etc/systemd/system/certbot-renew.timer + etc/systemd/system/caddy.service etc/systemd/system/custodian.service - etc/systemd/system/nginx-ssl-setup.service etc/systemd/system/persistent-luks-setup.service etc/systemd/system/reth.service etc/systemd/system/summit-keygen.service @@ -182,14 +179,13 @@ jobs: etc/systemd/system/summit.target etc/systemd/system/tdx-init.service etc/systemd/system/minimal.target.wants/attestation.service - etc/systemd/system/minimal.target.wants/certbot-renew.timer + etc/systemd/system/minimal.target.wants/caddy.service etc/systemd/system/minimal.target.wants/custodian.service - etc/systemd/system/minimal.target.wants/nginx-ssl-setup.service etc/systemd/system/minimal.target.wants/persistent-luks-setup.service etc/systemd/system/minimal.target.wants/reth.service etc/systemd/system/minimal.target.wants/summit.target etc/systemd/system/minimal.target.wants/tdx-init.service - usr/bin/nginx-ssl-setup + usr/bin/caddy usr/bin/persistent-luks-setup usr/bin/seismic-attestation-service usr/bin/seismic-custodian-service @@ -252,6 +248,16 @@ jobs: && find . -type f -print0 | sort -z | xargs -0 sha256sum) \ > build/initrd-files.sha256 + # A node parses the Caddyfile only after the config POST, so a mistake + # in it would first show as a node without HTTPS. The image's own caddy + # checks it here, with placeholder values for what domain.env provides. + - name: Validate the Caddyfile + run: | + root="$RUNNER_TEMP/initrd-root" + XDG_DATA_HOME="$RUNNER_TEMP/caddy" XDG_CONFIG_HOME="$RUNNER_TEMP/caddy" \ + DOMAIN_NAME=node.example.com DOMAIN_EMAIL=ops@example.com \ + "$root/usr/bin/caddy" validate --config "$root/etc/caddy/Caddyfile" --adapter caddyfile + - name: Upload checksums for reproducibility comparison if: >- github.event_name == 'pull_request' || @@ -379,7 +385,7 @@ jobs: # The release notes: the same facts as image.json, for a human reading # the release — the blob URL and the measurements URL together, what - # each founding input is, and the three source commits (this + # each founding input is, and the four source commits (this # repository's own commit is the tag; the pins live in sources.yaml at # that commit, so no asset repeats them beyond image.json). - name: Write the release notes @@ -389,7 +395,7 @@ jobs: run: | fact() { jq -r "$1" build/image.json; } vhd_blob_url=$(fact '.targets["azure-tdx"].vhd_blob_url') - reth=$(fact .sources.seismic_reth); summit=$(fact .sources.summit); enclave=$(fact .sources.enclave) + reth=$(fact .sources.seismic_reth); summit=$(fact .sources.summit); enclave=$(fact .sources.enclave); caddy=$(fact .sources.caddy) cat > "$RUNNER_TEMP/notes.md" <", - "sources": {"seismic_reth": "", "summit": "", "enclave": ""}, + "sources": {"seismic_reth": "", "summit": "", "enclave": "", "caddy": ""}, "targets": { "azure-tdx": { "vhd_blob_url": "https://seismicimages.blob.core.windows.net/dev/seismic_2026-09-22.2ee71c.vhd", @@ -97,36 +97,36 @@ Where the image's bytes are and what they are, in one machine-readable file, so } ``` -`targets` is keyed by attestation type, one entry per cloud the image is published for — Azure only today; a GCP image adds an entry, not a schema. Per target: the artifact the nodes boot from, the storage account's ARM ID (Azure's managed-disk import refuses to read a blob from another subscription or resource group without it, and the URL names the account but neither of those; a SAS URL would avoid the requirement but expires, so it has no place in a release), the measurements asset for that target, and the sha256 of the artifact where it is reproducible — the `.efi`, since the VHD's wrapping is not. Once per file: the image name (the tag, and the stem of `measurement_id`), this repository's commit, and the three source pins from `sources.yaml`. The publish job writes it (`make image-json`, [`scripts/seismic/image_json.sh`](scripts/seismic/image_json.sh)) after the VHD is pushed, asking Azure for the account's ID rather than carrying it in this repo, and renders the release notes from it. It refuses a measurements file and a `SHA256SUMS` that name different UKIs, so the two halves of a release cannot come from different builds. It is the one asset `make release-assets` does not produce, since the ID is a fact about where the bytes were put, not about the build. +`targets` is keyed by attestation type, one entry per cloud the image is published for — Azure only today; a GCP image adds an entry, not a schema. Per target: the artifact the nodes boot from, the storage account's ARM ID (Azure's managed-disk import refuses to read a blob from another subscription or resource group without it, and the URL names the account but neither of those; a SAS URL would avoid the requirement but expires, so it has no place in a release), the measurements asset for that target, and the sha256 of the artifact where it is reproducible — the `.efi`, since the VHD's wrapping is not. Once per file: the image name (the tag, and the stem of `measurement_id`), this repository's commit, and the four source pins from `sources.yaml`. The publish job writes it (`make image-json`, [`scripts/seismic/image_json.sh`](scripts/seismic/image_json.sh)) after the VHD is pushed, asking Azure for the account's ID rather than carrying it in this repo, and renders the release notes from it. It refuses a measurements file and a `SHA256SUMS` that name different UKIs, so the two halves of a release cannot come from different builds. It is the one asset `make release-assets` does not produce, since the ID is a fact about where the bytes were put, not about the build. ## What's in the image The Seismic node stack: [`seismic-reth`](https://github.com/SeismicSystems/seismic-reth), [`summit`](https://github.com/SeismicSystems/summit), and from [`enclave`](https://github.com/SeismicSystems/enclave) `tdx-init`, the custodian and the -attestation service, built from the commits pinned in -[`sources.yaml`](modules/seismic/sources.yaml), plus nginx and certbot from Debian. +attestation service, and [Caddy](https://github.com/caddyserver/caddy) as the public HTTPS +proxy, all built from the commits pinned in [`sources.yaml`](modules/seismic/sources.yaml). [`modules/seismic/readme.md`](modules/seismic/readme.md) covers how they are assembled: units, users and groups, directories. ## Exposed HTTPS endpoints -nginx terminates TLS (Let's Encrypt) and reverse-proxies the following paths to in-TEE services. See [`modules/seismic/mkosi.extra/etc/nginx/node-template.conf`](modules/seismic/mkosi.extra/etc/nginx/node-template.conf). +Caddy terminates TLS on `:443` with a certificate it takes and renews itself (Let's Encrypt, or ZeroSSL when that fails), and reverse-proxies the following paths to in-TEE services. Each backend serves one endpoint whatever the request path, so a route matches its path exactly, with or without a trailing slash, and forwards to `/`; anything else (`/rpc/foo`, `/rpcfoo`) is a 404. Only `/ws` accepts WebSocket upgrades. See [`modules/seismic/mkosi.extra/etc/caddy/Caddyfile`](modules/seismic/mkosi.extra/etc/caddy/Caddyfile). | Route | Backend | Purpose | Public? | | ----------------- | -------------------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------- | | `/rpc` | `reth` `:8545` | Ethereum JSON-RPC (shielded tx support via TxSeismic) | ✅ intended public | | `/ws` | `reth` `:8546` | Ethereum WebSocket RPC | ✅ intended public | -| `/summit` | `summit` `:3030` | Consensus REST API, incl. `/summit/get_deposit_signature/...` used by the staking UI | ⚠️ **overly broad** — see warning below | +| `/summit` | `summit` `:3030` | Consensus JSON-RPC: read-only queries (health, checkpoints, headers, validator balances, deposits, state proofs) | ⚠️ no rate limiting — see below | | `/attestation` | `attestation-service` `:7878` | Attestation API (tx-io attestation evidence, health/LUKS status, admission-chain status, peer root-key bootstrap) | ✅ same surface as `:7878`, which is open to anyone | | `/metrics/reth` | `reth` `:9001` | Prometheus metrics | ⚠️ unauthenticated | | `/metrics/summit` | `summit` `:9002` | Prometheus metrics | ⚠️ unauthenticated | ### ⚠️ Known sharp edges -Everything above listens on the single public `:443`. The intended long-term fix is to split nginx into two tiers — a public server block with only the endpoints that should reach the open internet (`/rpc`, `/ws`, a narrowed `/summit/get_deposit_signature`, `/attestation`), and an internal server block on a separate port with the rest (`/metrics/*`, the full `/summit/*` query surface). The deploy tooling would then configure cloud firewall rules (Azure NSG / GCP firewall) to allow the public port from `0.0.0.0/0` and restrict the internal port to the VPC CIDR. Until that split lands, the concrete issues are: +Everything above listens on the single public `:443`. The intended long-term fix is to split the proxy into two tiers — a public site with only the endpoints that should reach the open internet (`/rpc`, `/ws`, `/attestation`, and whatever of `/summit` the staking UI needs), and an internal site on a separate port with the rest (`/metrics/*`, the rest of `/summit`). The deploy tooling would then configure cloud firewall rules (Azure NSG / GCP firewall) to allow the public port from `0.0.0.0/0` and restrict the internal port to the VPC CIDR. Until that split lands, the concrete issues are: -- **`/summit/*` is a blanket proxy.** Summit exposes a JSON-RPC surface (via `jsonrpsee`) with ~20 methods: mostly read-only state queries (`getCheckpoint`, `getValidatorBalance`, `getDeposit`, etc. — analogous to `eth_*` reads and safe to expose), plus `getDepositSignature` which causes a BLS signature in the enclave, plus `sendGenesis` on the genesis-setup API which must *never* be public at runtime. Narrowing requires JSON-RPC-method-level filtering (all calls are `POST /`, so you can't gate by URL path alone). `getDepositSignature` additionally has no rate limiting — trivially DoS-able — and should gain `limit_req` regardless of network controls. +- **`/summit` is a blanket proxy with no rate limiting.** Summit's public listener serves only read-only queries (`health`, `getCheckpoint`, `getValidatorBalance`, `getDeposit`, `getStateProof`, … — analogous to `eth_*` reads). Its signing method, `getDepositSignature`, is on a localhost-only admin listener, and `sendGenesis` on a separate genesis-setup server; the proxy reaches neither. Narrowing `/summit` further needs JSON-RPC-method-level filtering, since every call is `POST /`. - **`/metrics/*` is unauthenticated.** Operationally safe on a locked-down network (the cloud firewall rule above is the right fix), but anyone who can reach the port can scrape sync status, peer info, and resource usage. Do *not* expose the internal port to the open internet without adding basic auth or an IP allowlist on top. ## Our diff vs upstream diff --git a/modules/seismic/mkosi.build b/modules/seismic/mkosi.build index 961ec31..74a8e60 100755 --- a/modules/seismic/mkosi.build +++ b/modules/seismic/mkosi.build @@ -10,6 +10,7 @@ SOURCES_YAML="$SRCDIR/modules/seismic/sources.yaml" ENCLAVE_COMMIT=$(mkosi-chroot yq -r '.enclave.git_reference' < "$SOURCES_YAML") RETH_COMMIT=$(mkosi-chroot yq -r '.seismic_reth.git_reference' < "$SOURCES_YAML") SUMMIT_COMMIT=$(mkosi-chroot yq -r '.summit.git_reference' < "$SOURCES_YAML") +CADDY_COMMIT=$(mkosi-chroot yq -r '.caddy.git_reference' < "$SOURCES_YAML") # We manage our own rust toolchain instead of using the one provided by the Debian Snapshot. # This decouples us from Debian's release cycle and installs deterministically the version @@ -144,3 +145,24 @@ make_git_package \ "https://github.com/SeismicSystems/summit" \ "$SUMMIT_BUILD_CMD" \ "target/release/summit:/usr/bin/summit" + +# Build caddy. Debian's golang-go is too old for it and only bootstraps: +# GOTOOLCHAIN makes it fetch the pinned Go release, which the checksum +# database verifies, the way rustup pins Rust above. It must be at least the +# `go` line of caddy's go.mod. CGO_ENABLED=0 links caddy statically, so it +# does not depend on the image's libc. +CADDY_BUILD_CMD=" + git checkout $CADDY_COMMIT + export GOTOOLCHAIN='go1.26.9' + export GOPATH='/go' + export GOCACHE='/go/cache' + export CGO_ENABLED='0' + + go build -trimpath -ldflags '-s -w -buildid=' -o build/caddy ./cmd/caddy +" +make_git_package \ + "caddy" \ + "$CADDY_COMMIT" \ + "https://github.com/caddyserver/caddy" \ + "$CADDY_BUILD_CMD" \ + "build/caddy:/usr/bin/caddy" diff --git a/modules/seismic/mkosi.conf b/modules/seismic/mkosi.conf index 52c2899..17ea0ce 100644 --- a/modules/seismic/mkosi.conf +++ b/modules/seismic/mkosi.conf @@ -26,15 +26,7 @@ ImageId=seismic # 1. Signed-PCR-policy LUKS enrollment (the BLOCKER header in # persistent-luks-setup) must land first — else every release # rebases roothash → PCR 11 → bricks unlock on first update. -# 2. Audit and relocate /usr/ writes from runtime scripts. Main -# offender is certbot's /etc/letsencrypt/; converges with -# the nginx+certbot → RA-TLS TODO below. -# 3. Uniform `User=` across reth/summit/nginx (enclave already -# done). Verity's value is in narrowing in-guest privilege -# pivots; with services-as-root, an RCE in any of them is -# RCE-into-root-readable enclave memory, and verity buys -# little. With per-service UIDs, the standard pivots -# (overwrite /usr/bin/X, swap a .so) are what verity closes. +# 2. Audit and relocate /usr/ writes from runtime scripts. [Build] Environment=KERNEL_CONFIG_SNIPPETS_SEISMIC=modules/seismic/kernel/config.d @@ -56,12 +48,6 @@ Packages=cryptsetup # TPM2 libraries the attestation service links against libtss2-esys-3.0.2-0t64 libtss2-tctildr0t64 - # TODO(samlaf): replace nginx+certbot with RA-TLS (e.g. flashbots' - # cvm-reverse-proxy) which embeds the TDX attestation quote in the - # TLS cert, removing the need for Let's Encrypt and CAs entirely. - nginx - certbot - python3-certbot-nginx # TODO: document what lz4 is used for lz4 @@ -80,3 +66,5 @@ BuildPackages=build-essential pkg-config # yq is used by mkosi.build to read source pins from sources.yaml yq + # bootstraps the pinned Go toolchain mkosi.build builds caddy with + golang-go diff --git a/modules/seismic/mkosi.extra/etc/caddy/Caddyfile b/modules/seismic/mkosi.extra/etc/caddy/Caddyfile new file mode 100644 index 0000000..d189992 --- /dev/null +++ b/modules/seismic/mkosi.extra/etc/caddy/Caddyfile @@ -0,0 +1,96 @@ +# The node's public HTTPS proxy. Every confidentiality property comes from +# attested application-layer keys, so TLS here is hygiene: a valid +# certificate for browsers and RPC clients. +# +# Caddy substitutes {$VAR} as raw text before parsing this file. Both values +# come from domain.env, which tdx-init writes from the operator's config POST +# and admits only hostname and plain-email characters, so neither can inject +# config: +# https://github.com/SeismicSystems/enclave/blob/seismic/crates/tdx-init-config/src/domain.rs +{ + # The default admin endpoint, localhost:2019, would let any local process + # replace this config. + admin off + persist_config off + + # Caddy's default issuers, Let's Encrypt and then ZeroSSL, spelled out + # because each one disables the HTTP-01 challenge. Each repeated + # `cert_issuer` is tried in order. ZeroSSL needs `email`: Caddy registers + # it with ZeroSSL's API for the account credentials its ACME endpoint + # requires. For Let's Encrypt it is only the account contact. + # + # One challenge: TLS-ALPN-01, which Caddy answers in the TLS handshake on + # :443. HTTP-01 would need :80 open to the internet, and so would the + # HTTP->HTTPS redirect, so with both off nothing listens on :80 and the + # firewall can close it. The cost: :443 must reach Caddy directly, with no + # TLS-terminating load balancer in front, and a failed TLS-ALPN-01 has no + # other challenge to fall back to, only the other CA. + email {$DOMAIN_EMAIL} + cert_issuer acme { + disable_http_challenge + } + cert_issuer acme https://acme.zerossl.com/v2/DV90 { + disable_http_challenge + } + auto_https disable_redirects + + # No HTTP/3: the firewall does not open UDP 443. + servers { + protocols h1 h2 + } +} + +{$DOMAIN_NAME} { + # Only /ws may open a WebSocket, as an HTTP/1.1 Upgrade or an HTTP/2 + # extended CONNECT. + @websocket_off_ws { + not path /ws /ws/ + expression {header.Upgrade} != "" || {method} == "CONNECT" + } + handle @websocket_off_ws { + respond 403 + } + + # Every backend serves one endpoint whatever the request path, so each + # route matches its prefix exactly, with or without a trailing slash, and + # forwards to /. Anything under or beside it (/rpc/foo, /rpcfoo) is a 404. + @rpc path /rpc /rpc/ + handle @rpc { + rewrite / + reverse_proxy 127.0.0.1:8545 + } + + @ws path /ws /ws/ + handle @ws { + rewrite / + reverse_proxy 127.0.0.1:8546 + } + + @summit path /summit /summit/ + handle @summit { + rewrite / + reverse_proxy 127.0.0.1:3030 + } + + @attestation path /attestation /attestation/ + handle @attestation { + rewrite / + reverse_proxy 127.0.0.1:7878 + } + + @metrics_reth path /metrics/reth /metrics/reth/ + handle @metrics_reth { + rewrite / + reverse_proxy 127.0.0.1:9001 + } + + @metrics_summit path /metrics/summit /metrics/summit/ + handle @metrics_summit { + rewrite / + reverse_proxy 127.0.0.1:9002 + } + + handle { + respond 404 + } +} diff --git a/modules/seismic/mkosi.extra/etc/nginx/node-template.conf b/modules/seismic/mkosi.extra/etc/nginx/node-template.conf deleted file mode 100644 index 350b701..0000000 --- a/modules/seismic/mkosi.extra/etc/nginx/node-template.conf +++ /dev/null @@ -1,99 +0,0 @@ -# TODO: split into public (:443) + internal (:8443) server blocks so -# cloud firewall rules can restrict the internal port to the VPC CIDR. -# Public: /rpc, /ws, narrowed /summit/get_deposit_signature (+ limit_req), -# /attestation (the same surface as :7878, which is open to anyone). -# Internal-only: /metrics/*, full /summit/*. -# See README "Exposed HTTPS endpoints" for the full reasoning. -server { - listen 80; - - root /var/www/html; - index index.html index.htm index.nginx-debian.html; - - server_name DOMAIN_NAME_PLACEHOLDER; - - location / { - # First attempt to serve request as file, then - # as directory, then fall back to displaying a 404. - try_files $uri $uri/ =404; - } - - # HTTPS RPC API - location /rpc { - rewrite ^/rpc/?(.*)$ /$1 break; - proxy_pass http://localhost:8545; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; - } - - # WS RPC API - location /ws { - rewrite ^/ws/?(.*)$ /$1 break; - proxy_pass http://localhost:8546; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; - } - - # Summit REST API - location /summit { - rewrite ^/summit/?(.*)$ /$1 break; - proxy_pass http://localhost:3030; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; - } - - # Attestation service RPC API - location /attestation { - rewrite ^/attestation/?(.*)$ /$1 break; - proxy_pass http://localhost:7878; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; - } - - # Reth metrics (for external Prometheus scrape). - # TODO: add basic auth or IP allowlist if/when metrics are deemed sensitive. - location /metrics/reth { - rewrite ^/metrics/reth/?(.*)$ /$1 break; - proxy_pass http://localhost:9001; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # Summit metrics (for external Prometheus scrape). - location /metrics/summit { - rewrite ^/metrics/summit/?(.*)$ /$1 break; - proxy_pass http://localhost:9002; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} diff --git a/modules/seismic/mkosi.extra/etc/seismic/tmpfiles-persistent.conf b/modules/seismic/mkosi.extra/etc/seismic/tmpfiles-persistent.conf index f3352e9..b787b9a 100644 --- a/modules/seismic/mkosi.extra/etc/seismic/tmpfiles-persistent.conf +++ b/modules/seismic/mkosi.extra/etc/seismic/tmpfiles-persistent.conf @@ -8,10 +8,8 @@ # # Owners match the system users created in mkosi.postinst. -# The node's nginx site config and certbot state (account, certificate, TLS -# key). Root-only: nginx-ssl-setup and certbot-renew write it, nginx reads it. -d /persistent/nginx 0700 root root - - +# Caddy's ACME account, certificate and TLS key. +d /persistent/caddy 0700 caddy caddy - d /persistent/reth 0700 reth reth - diff --git a/modules/seismic/mkosi.extra/etc/systemd/system/caddy.service b/modules/seismic/mkosi.extra/etc/systemd/system/caddy.service new file mode 100644 index 0000000..2055574 --- /dev/null +++ b/modules/seismic/mkosi.extra/etc/systemd/system/caddy.service @@ -0,0 +1,43 @@ +[Unit] +Description=Caddy, the node's public HTTPS proxy +# The domain comes from tdx-init's domain.env, and the certificate lives on +# the LUKS volume. Nothing requires this unit: a node without a certificate +# has no public HTTPS, while reth and summit run regardless. +After=network-online.target tdx-init.service persistent-luks-setup.service +Wants=network-online.target +Requires=tdx-init.service persistent-luks-setup.service + +[Service] +Type=notify +User=caddy +Group=caddy + +# Read by systemd, not by caddy, so the caddy user needs no `conf` group. +# Provides {$DOMAIN_NAME} and {$DOMAIN_EMAIL} to the Caddyfile. +EnvironmentFile=/run/seismic/conf/domain.env +ExecStart=/usr/bin/caddy run --config /etc/caddy/Caddyfile --adapter caddyfile +# Caddy keeps all its state in $XDG_DATA_HOME/caddy: the ACME account, the +# certificate and its key. On the LUKS volume, so a reboot keeps the +# certificate. XDG_CONFIG_HOME only gives Caddy a config dir to resolve; with +# persist_config off it writes nothing there. +Environment=XDG_DATA_HOME=/persistent XDG_CONFIG_HOME=/persistent + +# Binding :443 is the one privilege it needs. +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=yes +ProtectSystem=strict +ReadWritePaths=/persistent/caddy +PrivateTmp=yes +PrivateDevices=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX + +# Caddy retries a failed issuance or renewal in-process; a restart is for +# the process itself dying. +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=minimal.target diff --git a/modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.service b/modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.service deleted file mode 100644 index c494b7f..0000000 --- a/modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Certbot SSL Certificate Renewal -After=network-online.target persistent-luks-setup.service -Requires=persistent-luks-setup.service -# The timer is enabled in every image, but only a node whose nginx-ssl-setup -# obtained a certificate has anything to renew. -ConditionPathExists=/persistent/nginx/.certbot-done - -[Service] -Type=oneshot -ExecStart=/usr/bin/certbot renew --quiet --nginx --config-dir /persistent/nginx/certbot --work-dir /persistent/nginx/certbot/work --logs-dir /persistent/nginx/certbot/logs - -# TODO: create lockfile when snapshots are running; don't run this until it's deleted diff --git a/modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.timer b/modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.timer deleted file mode 100644 index d70802f..0000000 --- a/modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Run certbot renewal once monthly - -[Timer] -OnCalendar=*-*-01 02:00:00 -RandomizedDelaySec=1h -Persistent=true - -[Install] -WantedBy=minimal.target diff --git a/modules/seismic/mkosi.extra/etc/systemd/system/nginx-ssl-setup.service b/modules/seismic/mkosi.extra/etc/systemd/system/nginx-ssl-setup.service deleted file mode 100644 index c0615eb..0000000 --- a/modules/seismic/mkosi.extra/etc/systemd/system/nginx-ssl-setup.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Setup Nginx with SSL via Certbot -After=network-online.target persistent-luks-setup.service tdx-init.service -Wants=network-online.target -Requires=persistent-luks-setup.service tdx-init.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/nginx-ssl-setup -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=minimal.target diff --git a/modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup b/modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup deleted file mode 100644 index e5c0d62..0000000 --- a/modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -set -euo pipefail - -CONFIG_FILE="/run/seismic/conf/domain.env" -NODE_TEMPLATE="/etc/nginx/node-template.conf" -NODE_NGINX_CONF="/persistent/nginx/node-nginx.conf" -NODE_NGINX_ENABLED="/etc/nginx/sites-enabled/node" -CERTBOT_DONE="/persistent/nginx/.certbot-done" - -# Certbot directories in /persistent/nginx/certbot/ -CERTBOT_CONFIG_DIR="/persistent/nginx/certbot" -CERTBOT_WORK_DIR="/persistent/nginx/certbot/work" -CERTBOT_LOGS_DIR="/persistent/nginx/certbot/logs" - -# Check if config exists -if [ ! -f "$CONFIG_FILE" ]; then - echo "Config file not found: $CONFIG_FILE" - echo "Run tdx-init first to create the config" - exit 1 -fi - -# tdx-init writes DOMAIN_NAME and DOMAIN_EMAIL into this env file. -# shellcheck source=/dev/null -source "$CONFIG_FILE" - -if [ -z "${DOMAIN_NAME:-}" ]; then - echo "No domain name found in config, skipping nginx/SSL setup" - exit 0 -fi - -echo "Setting up nginx for domain: $DOMAIN_NAME" - -# Create persistent directories -mkdir -p "$CERTBOT_CONFIG_DIR" "$CERTBOT_WORK_DIR" "$CERTBOT_LOGS_DIR" - -# Generate nginx config from template (only if certbot hasn't run yet) -if [ ! -f "$CERTBOT_DONE" ]; then - echo "Generating node nginx config from template" - sed "s/DOMAIN_NAME_PLACEHOLDER/$DOMAIN_NAME/g" "$NODE_TEMPLATE" > "$NODE_NGINX_CONF" -else - echo "Certbot already configured, preserving existing nginx config" -fi - -# Create symlinks to nginx sites-enabled -mkdir -p /etc/nginx/sites-enabled - -# Remove old default symlink if it exists -rm -f /etc/nginx/sites-enabled/default - -# Create node symlink -rm -f "$NODE_NGINX_ENABLED" -ln -s "$NODE_NGINX_CONF" "$NODE_NGINX_ENABLED" - -# Verify symlink was created -if [ ! -L "$NODE_NGINX_ENABLED" ]; then - echo "ERROR: Failed to create symlink at $NODE_NGINX_ENABLED" - exit 1 -fi -echo "Symlink created: $NODE_NGINX_ENABLED -> $NODE_NGINX_CONF" - -# Test nginx config -nginx -t - -# Start/reload nginx -systemctl enable nginx -systemctl restart nginx - -# Run certbot if not already done -if [ ! -f "$CERTBOT_DONE" ]; then - echo "Running certbot for $DOMAIN_NAME with email $DOMAIN_EMAIL" - - # Wait for nginx to be fully up - sleep 2 - - certbot --nginx \ - --non-interactive \ - --agree-tos \ - --email "$DOMAIN_EMAIL" \ - --domain "$DOMAIN_NAME" \ - --redirect \ - --config-dir "$CERTBOT_CONFIG_DIR" \ - --work-dir "$CERTBOT_WORK_DIR" \ - --logs-dir "$CERTBOT_LOGS_DIR" - - # Mark as done - touch "$CERTBOT_DONE" - - echo "SSL certificate obtained successfully" -else - echo "Certbot already run, skipping (remove $CERTBOT_DONE to re-run)" -fi - -echo "Nginx SSL setup complete" -echo "Endpoints available at:" -echo " https://$DOMAIN_NAME/rpc" -echo " https://$DOMAIN_NAME/ws" -echo " https://$DOMAIN_NAME/summit" -echo " https://$DOMAIN_NAME/attestation" -echo " https://$DOMAIN_NAME/metrics/reth" -echo " https://$DOMAIN_NAME/metrics/summit" diff --git a/modules/seismic/mkosi.postinst b/modules/seismic/mkosi.postinst index f0bbf24..a232f9d 100755 --- a/modules/seismic/mkosi.postinst +++ b/modules/seismic/mkosi.postinst @@ -26,9 +26,9 @@ mkosi-chroot useradd -r -s /bin/false -G conf,custodian-ipc,tpm attestation # reth:tx-io,rng). mkosi-chroot useradd -r -s /bin/false -G conf,engine-api,custodian-ipc reth mkosi-chroot useradd -r -s /bin/false -G conf,engine-api summit +mkosi-chroot useradd -r -s /bin/false caddy # Make scripts executable -chmod +x "$BUILDROOT/usr/bin/nginx-ssl-setup" chmod +x "$BUILDROOT/usr/bin/persistent-luks-setup" chmod +x "$BUILDROOT/usr/bin/summit-persist" @@ -39,8 +39,7 @@ chmod +x "$BUILDROOT/usr/bin/summit-persist" mkosi-chroot systemctl enable \ persistent-luks-setup.service \ tdx-init.service \ - nginx-ssl-setup.service \ - certbot-renew.timer \ + caddy.service \ reth.service \ custodian.service \ attestation.service \ diff --git a/modules/seismic/readme.md b/modules/seismic/readme.md index 0eb8d8f..8ab7a40 100644 --- a/modules/seismic/readme.md +++ b/modules/seismic/readme.md @@ -9,12 +9,12 @@ Top-level files at a glance: | File | Role | | -------------------------------------- | ----------------------------------------------------------------------------------------------- | -| [`mkosi.conf`](mkosi.conf) | Debian packages (`nginx`, `certbot`, `cryptsetup`, …) + build packages | -| [`mkosi.build`](mkosi.build) | Source-builds: pinned commits of `tdx-init`, `seismic-reth`, `seismic-attestation-service`, `seismic-custodian-service`, `summit` | +| [`mkosi.conf`](mkosi.conf) | Debian packages (`cryptsetup`, `jq`, …) + build packages | +| [`mkosi.build`](mkosi.build) | Source-builds: pinned commits of `tdx-init`, `seismic-reth`, `seismic-attestation-service`, `seismic-custodian-service`, `summit`, `caddy` | | [`sources.yaml`](sources.yaml) | Pinned git refs read by `mkosi.build` (structured manifest, Renovate/Dependabot-friendly) | | [`mkosi.postinst`](mkosi.postinst) | Creates users/groups, enables systemd services | | [`kernel/config.d/`](kernel/config.d/) | Seismic-specific kernel config snippets | -| [`mkosi.extra/`](mkosi.extra/) | Filesystem overlay — systemd units, nginx config, helper scripts | +| [`mkosi.extra/`](mkosi.extra/) | Filesystem overlay — systemd units, the Caddyfile, helper scripts | What the resulting boot measures into the vTPM on Azure — the register inventory, and the `(pcr4, pcr9, pcr11)` guest identity admission binds — is @@ -29,9 +29,9 @@ build-inputs only. The image rootfs is the union of four channels: | Channel | What it puts in the image | | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [`mkosi.extra/`](mkosi.extra/) | Copied wholesale at matching paths — see tree below. | -| `Packages=` in [`mkosi.conf`](mkosi.conf) | apt-installed Debian packages: `nginx`, `certbot`, `python3-certbot-nginx`, `cryptsetup`, `jq`, `libtss2-*`, `lz4` | -| [`mkosi.build`](mkosi.build) | Compiled binaries written to `$DESTDIR`: `tdx-init`, `seismic-reth`, `seismic-attestation-service`, `seismic-custodian-service`, `summit` → `/usr/bin/` | -| [`mkosi.postinst`](mkosi.postinst) | Image-fs mutations: system users + `engine-api`/`custodian-ipc`/`conf`/`tpm` groups in `/etc/{passwd,group}`, services symlinked into `/etc/systemd/system/minimal.target.wants/`, `/usr/bin` scripts (`nginx-ssl-setup`, `persistent-luks-setup`, `summit-persist`) made executable | +| `Packages=` in [`mkosi.conf`](mkosi.conf) | apt-installed Debian packages: `cryptsetup`, `jq`, `libtss2-*`, `lz4` | +| [`mkosi.build`](mkosi.build) | Compiled binaries written to `$DESTDIR`: `tdx-init`, `seismic-reth`, `seismic-attestation-service`, `seismic-custodian-service`, `summit`, `caddy` → `/usr/bin/` | +| [`mkosi.postinst`](mkosi.postinst) | Image-fs mutations: system users + `engine-api`/`custodian-ipc`/`conf`/`tpm` groups in `/etc/{passwd,group}`, services symlinked into `/etc/systemd/system/minimal.target.wants/`, `/usr/bin` scripts (`persistent-luks-setup`, `summit-persist`) made executable | `mkosi.extra/` lays out exactly what its name suggests — the same paths relative to the image root: @@ -39,14 +39,14 @@ relative to the image root: ``` mkosi.extra/ ├── etc/ -│ ├── nginx/node-template.conf → /etc/nginx/node-template.conf +│ ├── caddy/Caddyfile → /etc/caddy/Caddyfile │ ├── security/limits.d/nofile.conf → /etc/security/limits.d/nofile.conf │ ├── seismic/tmpfiles-persistent.conf → /etc/seismic/tmpfiles-persistent.conf │ ├── tmpfiles.d/seismic-runtime.conf → /etc/tmpfiles.d/seismic-runtime.conf -│ ├── systemd/system/*.{service,timer} → /etc/systemd/system/... +│ ├── systemd/system/*.{service,target} → /etc/systemd/system/... │ └── udev/rules.d/60-tpm-permissions.rules → /etc/udev/rules.d/... └── usr/ - └── bin/{nginx-ssl-setup,persistent-luks-setup,summit-persist} → /usr/bin/... + └── bin/{persistent-luks-setup,summit-persist} → /usr/bin/... ``` Module-root files that are **not** in the image: `mkosi.conf`, @@ -90,7 +90,8 @@ files, sockets or devices. | `attestation` | `conf`, `custodian-ipc`, `tpm` | `attestation.service` | | `reth` | `conf`, `engine-api`, `custodian-ipc` | `reth.service` (primary group `engine-api`) | | `summit` | `conf`, `engine-api` | `summit-keygen`, `summit-persist`, `summit` | -| root | | `persistent-luks-setup`, `nginx-ssl-setup`, `certbot-renew` | +| `caddy` | | `caddy.service` | +| root | | `persistent-luks-setup` | | Group | Grants | | --- | --- | @@ -115,7 +116,7 @@ on the LUKS volume, from | `/run/seismic/summit` | `summit:summit 0755` | `summit-keygen`, `summit-persist` | the attestation service; read-only to `summit.service` | | `/run/seismic/summit/keys` | `summit:summit 0700` | `summit-keygen` | `summit-persist`; inaccessible to `summit.service` | | `/run/reth-engine` | `reth:engine-api 0750` (`RuntimeDirectory=`) | reth's Engine API socket | summit | -| `/persistent/nginx` | `root:root 0700` | `nginx-ssl-setup`, `certbot-renew` | nginx | +| `/persistent/caddy` | `caddy:caddy 0700` | caddy | caddy | | `/persistent/reth` | `reth:reth 0700` | reth | reth | | `/persistent/summit/db` | `summit:summit 0700` | summit | summit | | `/persistent/summit/keys` | `summit:summit 0700` | `summit-persist` | summit, read-only | @@ -131,7 +132,7 @@ configuration, then writes one file per consumer into | File | Read by | | --- | --- | -| `domain.env` | `nginx-ssl-setup` | +| `domain.env` | systemd for `caddy.service` | | `attestation.env` | the attestation service, once the sentinel appears | | `network-manifest.json` | the attestation service, whose quote window it closes; the custodian, for its `founding_tx_io_pk` | | `reth-p2p.env` | systemd for `reth.service` | @@ -191,27 +192,30 @@ clouds override it with globs in `/etc/seismic-images/persistent-disk-glob`. TODO: have deploy tooling write the override at provisioning time. -### `nginx-ssl-setup.service` and `certbot-renew.timer` +### `caddy.service` -`nginx-ssl-setup` templates -[`node-template.conf`](mkosi.extra/etc/nginx/node-template.conf) with -the domain and obtains a Let's Encrypt certificate. Nothing depends on -it: a failed certificate leaves the node without public HTTPS, while -reth and summit run regardless. +The public HTTPS proxy on `:443`, as the unprivileged `caddy` user with +`CAP_NET_BIND_SERVICE` alone. The +[Caddyfile](mkosi.extra/etc/caddy/Caddyfile) is static and in the +image, so it is measured and the same on every node; the only per-node +input is the domain name and ACME contact from `domain.env`, which +systemd reads for it. -`certbot-renew.timer` is enabled in the image, since the rootfs is tmpfs -and an enablement made at runtime would not survive a reboot. Its -service is skipped until `nginx-ssl-setup` has obtained a certificate. -The timer renews monthly with `RandomizedDelaySec=1h`, so a fleet does -not hit Let's Encrypt in the same minute. Renewal can race a disk -snapshot (TODO in the service file). +Caddy takes and renews its certificate itself, from Let's Encrypt or, +when that fails, ZeroSSL, over TLS-ALPN-01 on `:443`, and retries a +failed attempt in-process. +Nothing listens on `:80`, there is no admin endpoint, and HTTP/3 is +off. The certificate, its key and the ACME account live in +`/persistent/caddy`, so a reboot keeps them. Nothing depends on the +unit: a node without a certificate has no public HTTPS, while reth and +summit run regardless. ### `reth.service` and `summit.service` | | reth | summit | | --- | --- | --- | | Public | devp2p `:30303` TCP+UDP (discv5 only) | P2P `:18551` | -| Loopback | RPC `:8545`, WS `:8546`, metrics `:9001` | REST `:3030` (nginx `/summit`), metrics `:9002` | +| Loopback | RPC `:8545`, WS `:8546`, metrics `:9001` | REST `:3030` (Caddy `/summit`), metrics `:9002` | reth fetches its purpose keys from the custodian socket at startup; `persistent-luks-setup` finishing implies the custodian holds diff --git a/modules/seismic/sources.yaml b/modules/seismic/sources.yaml index 930cb03..e5a4aca 100644 --- a/modules/seismic/sources.yaml +++ b/modules/seismic/sources.yaml @@ -12,8 +12,11 @@ enclave: # and tdx-init. We could consider using separate pins for each, but this is # simpler for now, plus they will likely need to be updated together in most # cases since they share libraries. - git_reference: 56d2a917cf4f5635651f229f59d958f64b1a7b73 + git_reference: 74a6185029895903ce408fca45332c486a049326 seismic_reth: git_reference: 39d04d158da383324b12c2e3397b0b28f3c3ee36 summit: git_reference: 532ad6fa5bf07f03faf349664000f13f36d6517a +caddy: + # v2.11.7. Plain upstream: no plugins, so the build needs no xcaddy. + git_reference: 72dd0fb067f6d7826c7f79907670ba4a713bfe37 diff --git a/scripts/seismic/image_json.sh b/scripts/seismic/image_json.sh index bf1f823..cc78153 100755 --- a/scripts/seismic/image_json.sh +++ b/scripts/seismic/image_json.sh @@ -91,6 +91,7 @@ jq -n \ --arg seismic_reth "$(pin seismic_reth)" \ --arg summit "$(pin summit)" \ --arg enclave "$(pin enclave)" \ + --arg caddy "$(pin caddy)" \ --arg vhd_blob_url "$vhd_blob_url" \ --arg storage_account_id "$account_id" \ --arg measurements "$(basename "$measurements")" \ @@ -98,7 +99,7 @@ jq -n \ '{ image: $image, commit: $commit, - sources: {seismic_reth: $seismic_reth, summit: $summit, enclave: $enclave}, + sources: {seismic_reth: $seismic_reth, summit: $summit, enclave: $enclave, caddy: $caddy}, targets: { "azure-tdx": { vhd_blob_url: $vhd_blob_url,