fix(seismic): certbot no longer gates consensus, renewal survives reboot

.github/workflows/seismic.yml

@@ -170,7 +170,10 @@ jobs:
             }
           done <<'EOF'
           etc/systemd/system/attestation.service
+          etc/systemd/system/certbot-renew.service
+          etc/systemd/system/certbot-renew.timer
           etc/systemd/system/custodian.service
+          etc/systemd/system/nginx-ssl-setup.service
           etc/systemd/system/persistent-luks-setup.service
           etc/systemd/system/reth.service
           etc/systemd/system/summit-keygen.service
@@ -179,7 +182,9 @@ jobs:
           etc/systemd/system/summit.target
           etc/systemd/system/tdx-init.service
           etc/systemd/system/minimal.target.wants/attestation.service
+          etc/systemd/system/minimal.target.wants/certbot-renew.timer
           etc/systemd/system/minimal.target.wants/custodian.service
+          etc/systemd/system/minimal.target.wants/nginx-ssl-setup.service
           etc/systemd/system/minimal.target.wants/persistent-luks-setup.service
           etc/systemd/system/minimal.target.wants/reth.service
           etc/systemd/system/minimal.target.wants/summit.target

modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.service

@@ -2,6 +2,9 @@
 Description=Certbot SSL Certificate Renewal
 After=network-online.target persistent-luks-setup.service
 Requires=persistent-luks-setup.service
+# The timer is enabled in every image, but only a node whose nginx-ssl-setup
+# obtained a certificate has anything to renew.
+ConditionPathExists=/persistent/nginx/.certbot-done
 
 [Service]
 Type=oneshot

modules/seismic/mkosi.extra/etc/systemd/system/certbot-renew.timer

@@ -7,4 +7,4 @@ RandomizedDelaySec=1h
 Persistent=true
 
 [Install]
-WantedBy=timers.target
+WantedBy=minimal.target

modules/seismic/mkosi.extra/etc/systemd/system/nginx-ssl-setup.service

@@ -11,4 +11,4 @@ StandardOutput=journal
 StandardError=journal
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=minimal.target

modules/seismic/mkosi.extra/etc/systemd/system/reth.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Seismic Reth Node
-After=network.target persistent-luks-setup.service nginx-ssl-setup.service attestation.service
-Requires=persistent-luks-setup.service nginx-ssl-setup.service attestation.service
+After=network.target persistent-luks-setup.service attestation.service
+Requires=persistent-luks-setup.service attestation.service
 
 [Service]
 Type=simple

modules/seismic/mkosi.extra/etc/systemd/system/summit.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Summit Consensus Server
-After=network.target persistent-luks-setup.service nginx-ssl-setup.service attestation.service reth.service summit-persist.service
-Requires=persistent-luks-setup.service nginx-ssl-setup.service attestation.service reth.service summit-persist.service
+After=network.target persistent-luks-setup.service attestation.service reth.service summit-persist.service
+Requires=persistent-luks-setup.service attestation.service reth.service summit-persist.service
 PartOf=summit.target
 
 [Service]

modules/seismic/mkosi.extra/usr/bin/nginx-ssl-setup

@@ -85,12 +85,7 @@ if [ ! -f "$CERTBOT_DONE" ]; then
     # Mark as done
     touch "$CERTBOT_DONE"
 
-    # Enable automatic renewal timer
-    systemctl enable certbot-renew.timer
-    systemctl start certbot-renew.timer
-
     echo "SSL certificate obtained successfully"
-    echo "Automatic renewal enabled"
 else
     echo "Certbot already run, skipping (remove $CERTBOT_DONE to re-run)"
 fi

modules/seismic/mkosi.postinst

@@ -32,19 +32,16 @@ chmod +x "$BUILDROOT/usr/bin/nginx-ssl-setup"
 chmod +x "$BUILDROOT/usr/bin/persistent-luks-setup"
 chmod +x "$BUILDROOT/usr/bin/summit-persist"
 
-# Enable services
-mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants"
-for service in \
+# Enable services: `systemctl enable` creates the symlink each unit's
+# [Install] WantedBy= names, here minimal.target.wants/<unit>, so the boot
+# target pulls it in. It must happen at build time: the rootfs is a tmpfs,
+# so a unit enabled on a running node is disabled again after a reboot.
+mkosi-chroot systemctl enable \
     persistent-luks-setup.service \
     tdx-init.service \
     nginx-ssl-setup.service \
+    certbot-renew.timer \
     reth.service \
     custodian.service \
     attestation.service \
     summit.target
-do
-    mkosi-chroot systemctl enable "$service"
-    ln -sf "/etc/systemd/system/$service" "$BUILDROOT/etc/systemd/system/minimal.target.wants/"
-done
-
-# Note: certbot-renew.timer is enabled by nginx-ssl-setup after initial certbot run

modules/seismic/readme.md

@@ -195,10 +195,13 @@ write the override at provisioning time.
 
 `nginx-ssl-setup` templates
 [`node-template.conf`](mkosi.extra/etc/nginx/node-template.conf) with
-the domain, obtains a Let's Encrypt certificate, and enables
-`certbot-renew.timer`. reth and summit both `Requires=` it, so a failed
-certificate keeps them down, though neither needs public HTTPS to run.
+the domain and obtains a Let's Encrypt certificate. Nothing depends on
+it: a failed certificate leaves the node without public HTTPS, while
+reth and summit run regardless.
 
+`certbot-renew.timer` is enabled in the image, since the rootfs is tmpfs
+and an enablement made at runtime would not survive a reboot. Its
+service is skipped until `nginx-ssl-setup` has obtained a certificate.
 The timer renews monthly with `RandomizedDelaySec=1h`, so a fleet does
 not hit Let's Encrypt in the same minute. Renewal can race a disk
 snapshot (TODO in the service file).