feat(worker): run the operator console as a service (#708)

.github/workflows/deploy-lint.yml

@@ -5,7 +5,7 @@
 # plist ships silently and only fails on the operator's box. This workflow is the cheap gate: it checks
 # STRUCTURE, not runtime behaviour.
 #
-#   - worker.service / receiver.service : `systemd-analyze verify` parses the units. The PLACEHOLDER
+#   - worker.service / receiver.service / console.service : `systemd-analyze verify` parses the units. The PLACEHOLDER
 #     paths (/opt/pi-dispatch, /usr/bin/node) do not exist on the runner, so path-existence warnings are
 #     EXPECTED and ignored; only genuine syntax errors (unknown directives, parse failures) fail.
 #   - com.pi-dispatch.worker.plist : `xmllint --noout` proves the plist is well-formed XML. plutil is
@@ -61,7 +61,7 @@ jobs:
           # verify exits non-zero for the missing PLACEHOLDER exec paths, which is expected -- so gate on
           # the CONTENT of the diagnostics, not the exit code. A real defect is an unknown directive or a
           # parse failure; a missing /opt/pi-dispatch or /usr/bin/node is not.
-          systemd-analyze verify --no-pager deploy/worker.service deploy/receiver.service > verify.log 2>&1 || true
+          systemd-analyze verify --no-pager deploy/worker.service deploy/receiver.service deploy/console.service > verify.log 2>&1 || true
           cat verify.log
           if grep -Eiq 'Unknown (key name|lvalue|section)|Failed to parse|Invalid setting|not a valid|expected ' verify.log; then
             echo "::error::systemd-analyze found a syntax error in a unit file. See the log above."
@@ -96,6 +96,34 @@ jobs:
           fi
           echo "OK: the rendered openshell unit is syntactically valid"
 
+      # Issue #708: the console's unit as `service install --console` renders it: the whole ExecStart replaced by the
+      # console package's entry with `serve --deployment`, which the template alone cannot show. The admin package is
+      # built first, since its dist/console.mjs is the entry the render resolves (and refuses without).
+      - name: a rendered console unit parses (issue #708)
+        run: |
+          node admin/build.mjs
+          d="$RUNNER_TEMP/render-708"
+          mkdir -p "$d"
+          : > "$d/.env"
+          (cd "$d" && node "$GITHUB_WORKSPACE/worker/src/cli.mjs" service render --console) > "$d/pi-dispatch-console.service"
+          cat "$d/pi-dispatch-console.service"
+          grep -Eq "^ExecStart=/[^ ]*node $GITHUB_WORKSPACE/admin/dist/console\.mjs serve --deployment $d\$" "$d/pi-dispatch-console.service" || { echo "::error::the rendered console unit does not run the console package's entry with serve --deployment"; exit 1; }
+          grep -qx 'RestartPreventExitStatus=2' "$d/pi-dispatch-console.service" || { echo "::error::the rendered console unit lost RestartPreventExitStatus=2"; exit 1; }
+          # And one folder holding characters a unit must quote or escape (a space, a quote, % and $), so the quoting
+          # (`systemdWord`, `%%` in WorkingDirectory=) is parsed by systemd itself, not only by the unit tests' reader.
+          q="$RUNNER_TEMP/it's 100% \$HOME"
+          mkdir -p "$q" "$d/quoted"
+          (cd "$q" && node "$GITHUB_WORKSPACE/worker/src/cli.mjs" service render --console) > "$d/quoted/pi-dispatch-console.service"
+          grep -q '^WorkingDirectory=.*100%% \$HOME$' "$d/quoted/pi-dispatch-console.service" || { echo "::error::the quoted folder's % is not doubled in WorkingDirectory="; exit 1; }
+          cat "$d/quoted/pi-dispatch-console.service"
+          systemd-analyze verify --no-pager "$d/pi-dispatch-console.service" "$d/quoted/pi-dispatch-console.service" > verify-708.log 2>&1 || true
+          cat verify-708.log
+          if grep -Eiq 'Unknown (key name|lvalue|section)|Failed to parse|Invalid setting|not a valid|expected ' verify-708.log; then
+            echo "::error::systemd-analyze found a syntax error in the rendered console unit. See the log above."
+            exit 1
+          fi
+          echo "OK: the rendered console unit is syntactically valid"
+
       - name: plist is well-formed XML
         run: |
           xmllint --noout deploy/com.pi-dispatch.worker.plist \

README.md

@@ -458,7 +458,8 @@ pi-dispatch service restart --drain   # pause, wait for running jobs, restart, r
 commands instead). A unit uses the folder you run `install` from as its working directory and reads that
 folder's `.env`. `--receiver` installs the receiver's unit instead of the worker's (the receiver package
 must be installed
-beside the worker), and `--env-setup <path>` wraps the worker in
+beside the worker), `--console` installs the operator console's (`pi-dispatch-console serve`, once the admin
+package is installed and a console token with the operator role exists: [`docs/console.md`](docs/console.md)), and `--env-setup <path>` wraps the worker in
 your secrets manager's setup script ([`docs/secrets.md`](docs/secrets.md)). On rootless Podman
 (`PI_BACKENDS=podman`) it also installs Valkey, the egress proxy and the proxy's network keeper as systemd
 units (Podman's Quadlet format), and `up` offers the same ([`docs/podman.md`](docs/podman.md)). Run one

deploy/console.service

@@ -0,0 +1,52 @@
+# UNTESTED EXAMPLE (DES-OPERATOR-CONSOLE) -- a starting point, not a shipped, verified unit. Adapt it.
+#
+# The operator console (`pi-dispatch-console serve`, issue #640) as a service, so it keeps running after the terminal
+# that started it closes (issue #708). `pi-dispatch service install --console` renders this file for THIS host: the
+# node, the console package's own entry (admin/dist/console.mjs of @edgehero/pi-dispatch-admin, wherever npm put it)
+# and the deployment folder are substituted, so prefer that to adapting it by hand.
+#
+# The console listens on PI_CONSOLE_LISTEN (default 127.0.0.1:4790, loopback only) and reads its tokens file
+# (PI_CONSOLE_TOKENS_FILE, default console-tokens.json in the deployment folder). It never needs Docker or Podman, so
+# this unit is ordered after nothing but the network and has no Quadlet companion. Reach it from another machine
+# with ssh -L, never by binding a public address: it speaks plain HTTP (docs/console.md).
+#
+# NO EnvironmentFile= on purpose: `serve --deployment` reads the deployment's `.env` itself, takes only its own
+# PI_CONSOLE_* keys and the Valkey keys it needs from it, and refuses a `.env` another account can change. Loading the
+# whole file into the process environment would hand it every worker secret (the provider key, the forge token) and
+# would make each of those keys read as this shell's, which wins over the file and skips that trust check.
+# WorkingDirectory / User / node path below are PLACEHOLDERS: set them to wherever your deployment folder is and
+# whoever owns it (the tokens file must be owned by that account, mode 0600, or serve refuses).
+
+[Unit]
+Description=pi-dispatch operator console (the web console over the queue, runs and spend; loopback by default)
+# For a system unit. In a --user unit these two are inert (the user manager has no network-online.target), which is
+# harmless: the console binds loopback or a socket, and its Valkey reads retry on their own.
+After=network-online.target
+Wants=network-online.target
+# Crash-loop bound, as the worker and receiver units carry it: at most StartLimitBurst restarts within
+# StartLimitIntervalSec, then systemd stops trying. Pairs with Restart=on-failure below.
+StartLimitIntervalSec=60
+StartLimitBurst=5
+
+[Service]
+Type=simple
+User=pi
+WorkingDirectory=/opt/pi-dispatch
+ExecStart=/usr/bin/node /usr/lib/node_modules/@edgehero/pi-dispatch-admin/dist/console.mjs serve --deployment /opt/pi-dispatch
+Restart=on-failure
+RestartSec=5
+# Exit 2 is the console's refusal: no tokens file, no operator token in it, an unusable PI_CONSOLE_LISTEN, or the
+# single-instance lock held by another console of this deployment. The next start reads the same files and fails
+# the same way, so it is never restarted; add a token (`pi-dispatch-console token add`) or fix the setting, then
+# start the unit again.
+# A crash leaves the lock (<PI_LOGS_DIR>/console-audit/.console.lock) behind holding a dead pid, which the next start
+# takes over. But if that pid has since been reused by another process (or by another account's, which reads as
+# alive), every start refuses with exit 2 and the unit stays stopped: when the pid the refusal names is not a console,
+# remove the lock file and start the unit again.
+RestartPreventExitStatus=2
+# serve handles SIGTERM: it ends the open streams, closes the listener and releases the lock, then exits.
+KillSignal=SIGTERM
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

deploy/worker-env-wrapper.cmd

@@ -54,6 +54,10 @@ REM number. So after the load the wrapper re-asserts its own: ENV_SETUP and PI_E
 REM %PI_ENV_SETUP%, which cmd expands when it READS this parenthesized block, before its first line runs
 REM (`set /?`, the reason delayed expansion exists), so the value is the unit's and no .env line reaches it;
 REM and ERRORLEVEL cleared just before the command runs. RC is assigned after the command, before it is read.
+REM PI_WRAPPER_NO_DOTENV=1 (issue #708): the command reads the deployment's .env ITSELF and takes only the keys it
+REM owns (the operator console, `serve --deployment`), so .env is neither loaded nor required here; see
+REM worker-env-wrapper.sh for why. Read before the load, from the service's own environment, and cleared after it.
+if "%PI_WRAPPER_NO_DOTENV%"=="1" goto wrapper_env_loaded
 if exist ".env" (
   for /f "usebackq eol=# tokens=1,* delims==" %%A in (".env") do set "%%A=%%B"
   set "ENV_SETUP=%PI_ENV_SETUP%"
@@ -65,6 +69,8 @@ if exist ".env" (
   )
   echo worker-env-wrapper: no .env in "%CD%" -- the environment comes from "%ENV_SETUP%" ^(PI_ENV_SETUP^) 1>&2
 )
+:wrapper_env_loaded
+set "PI_WRAPPER_NO_DOTENV="
 
 REM AFTER .env, deliberately: the manager is the newer source of truth, so a stale key left in the file
 REM loses instead of silently shadowing the managed one (mirrors the .sh twin). A missing or failing

deploy/worker-env-wrapper.sh

@@ -77,7 +77,15 @@ trap wrapper_stopped_early TERM INT
 # until the `shift` below.
 set -- "${PI_ENV_SETUP:-}" "$@"
 
-if [ -f ./.env ]; then
+# PI_WRAPPER_NO_DOTENV=1 (issue #708) is set by the unit for a command that reads the deployment's .env ITSELF and
+# takes only the keys it owns: the operator console (`serve --deployment`), which also refuses a .env another account
+# can change. Sourcing the whole file here would hand that process every worker secret and make each key read as its
+# own shell's, which wins over the file and skips that check. So ./.env is not sourced and not required. Tested
+# before the load, from the unit's environment, so no .env line can set it; PI_ENV_SETUP is honoured exactly as
+# without it (an operator who configured a script asked for it). Unset after, so the command never sees it.
+if [ "${PI_WRAPPER_NO_DOTENV:-}" = "1" ]; then
+	:
+elif [ -f ./.env ]; then
 	set -a; . ./.env; set +a
 elif [ -z "$1" ]; then
 	echo "worker-env-wrapper: .env not found in $PWD -- this wrapper must be started in the deployment folder (the unit's WorkingDirectory / nssm AppDirectory); it no longer guesses a location from its own path" >&2
@@ -90,6 +98,7 @@ fi
 # AFTER THE LOAD, every variable this wrapper reads is assigned, so no assignment line of ./.env
 # reaches one. The worker sees the unit's PI_ENV_SETUP too, never one the file set (the setup script,
 # which is unit configuration, may still set its own).
+unset PI_WRAPPER_NO_DOTENV
 env_setup=$1
 shift
 if [ -n "$env_setup" ]; then

docs/console.md

@@ -80,6 +80,43 @@ Host it likes, so against a job the token is what holds. A socket only you can o
 Docker Desktop, use it. It also stops another
 account from spending the login rate limit and slowing your own sign-in.
 
+## Keep it running as a service
+
+`serve` stops with its terminal. To keep the console running, install it as a service from the deployment folder:
+
+```bash
+pi-dispatch service install --console
+```
+
+That is a LaunchAgent on macOS, a `systemctl --user` unit on Linux (`--system` prints the sudo commands instead) and
+an nssm service on Windows, like the worker's ([`README.md`](../README.md#run-as-a-service)). The unit runs
+`pi-dispatch-console serve --deployment <this folder>`, so it reads this folder's `.env` and tokens file.
+`service status --console`, `start`, `stop`, `restart` and `uninstall` take the same flag.
+
+`install` refuses in two cases, before it writes anything:
+
+- the console is not installed where it can find it. Install the admin package where the worker is installed:
+  `npm install -g @edgehero/pi-dispatch-admin` beside a global worker, or `npm install @edgehero/pi-dispatch-admin` in
+  the folder whose `node_modules` holds the worker. A `pi-dispatch-console` on your PATH is used only when it is that
+  package's own file (a pnpm or Volta shim is not), and on Windows the PATH is not searched;
+- the tokens file holds no operator token: `serve` would refuse to start, so the service would never run. Add one
+  with `pi-dispatch-console token add --name <you> --role operator` first.
+
+When `serve` refuses (exit 2: no operator token, another console holding the lock, an unusable address), the service
+stays stopped instead of restarting. Fix the cause, then `pi-dispatch service start --console`.
+
+After a crash the lock (`console-audit/.console.lock` in your logs folder) still names the dead console's pid, and the
+next start takes it over. If another process has taken that pid since, every start refuses and names it: when that
+pid is not a console, remove the lock file and start the service again.
+
+The service does not load your `.env` into its environment: `serve --deployment` reads the file itself and takes only
+the console's own settings from it, so the worker's secrets never reach the console process.
+
+`pi-dispatch doctor` says whether a console service is installed for the deployment and whether the console answers
+on its address (one `GET /healthz` to the loopback address or the socket, never anywhere else). An installed service
+that does not answer is a failure; no service and nothing running is a warning. `pi-dispatch up` names the install
+command once an operator token exists.
+
 ## From another machine
 
 Forward a local port over SSH and open it in your browser. To a console on the default TCP port:
@@ -203,6 +240,9 @@ set). It then checks:
 - `PI_CONSOLE_LISTEN` is a value `serve` would accept;
 - `PI_CONSOLE_BIND_PUBLIC=1` (a warning, every time);
 - the audit chain verifies (a warning when it does not).
+- whether a console service is installed for this folder (`pi-dispatch service install --console`) and whether the
+  console answers on its address: a failure when the service is installed and nothing answers, a warning when there
+  is no service and nothing runs.
 
 ## The screens