fix(ci): make links_live check the whole live site without throttling

- Feed lychee all page URLs from the live sitemap (lychee does not
  spider; the previous run only ever checked the homepage's links)
- Throttle the crawl of www.rnpgp.org (concurrency 4 + 500ms interval):
  full-concurrency crawls were getting 503s from GitHub Pages
- Verified locally: 2,724 links over all 64 live pages, 0 errors

Author: ronaldtse

.github/workflows/links_live.yml

@@ -3,6 +3,7 @@ name: links_live
 # Post-deploy link check of the LIVE site: verifies that production URLs —
 # including the absolute canonical/OG URLs that pre-deploy checks must skip —
 # actually resolve. Runs after every successful deploy and weekly.
+# Page URLs come from the sitemap (lychee does not spider).
 
 on:
   workflow_run:
@@ -20,9 +21,18 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Build URL list from the live sitemap
+      run: |
+        curl -fsSL https://www.rnpgp.org/sitemap-index.xml \
+          | grep -oE 'https://www\.rnpgp\.org/sitemap-[0-9]+\.xml' \
+          | while read -r sm; do curl -fsSL "$sm"; done \
+          | grep -oE '<loc>[^<]+' \
+          | sed 's/<loc>//' > urls.txt
+        wc -l urls.txt
+
     - name: Link Checker (live site)
       id: lychee
       uses: lycheeverse/lychee-action@v2
       with:
-        args: --config lychee-live.toml --no-progress 'https://www.rnpgp.org'
+        args: --config lychee-live.toml --no-progress --files-from urls.txt
         fail: true

lychee-live.toml

@@ -2,10 +2,10 @@
 # Unlike lychee.toml (pre-deploy), this does NOT exclude www.rnpgp.org — the whole
 # point is verifying that production URLs, including canonical/OG ones, resolve.
 
-max_concurrency = 32
+max_concurrency = 16
 timeout = 30
-retry_wait_time = 2
-max_retries = 2
+retry_wait_time = 3
+max_retries = 3
 
 # Pages exist but rate-limit automated checkers (HN, git.gnupg.org, …)
 accept = ["200", "204", "429"]
@@ -17,3 +17,8 @@ exclude = [
   # Cirrus CI blocks non-browser TLS clients.
   '^https://cirrus-ci\.com/',
 ]
+
+# Be gentle with GitHub Pages — full-concurrency crawls get throttled (503).
+[hosts."www.rnpgp.org"]
+concurrency = 4
+request_interval = "500ms"