ci: fail a PR if any test gets more than 1.5x slower than master

.github/workflows/ci.yml

@@ -67,14 +67,26 @@ jobs:
       - name: Install Dependencies
         run: npm install
 
+      # Per-test durations from the last master run of this version group, so a PR
+      # can be failed if it makes a test more than 1.5x slower (test/common/compareDurations.js).
+      - name: Restore duration baseline
+        uses: actions/cache/restore@v4
+        with:
+          path: durations
+          key: durations-${{ strategy.job-index }}-${{ github.sha }}
+          restore-keys: durations-${{ strategy.job-index }}-
+      - run: |
+          mv durations baseline 2>/dev/null || mkdir -p baseline
+          mkdir -p durations
+
       - name: Start Tests
         run: |
           exit_code=0
           pids=""
           for v in ${{ matrix.versions }}; do
             # TRACE is cheap (<1% runtime) and only written by the harness bot,
             # so always record a packet trace for post-mortem analysis.
-            TRACE="trace-${v}.jsonl" npm run mocha_test -- --retries 3 -g "${v}v" > "test-${v}.log" 2>&1 &
+            TRACE="trace-${v}.jsonl" DURATIONS="durations/durations-${v}.json" npm run mocha_test -- --retries 3 -g "${v}v" > "test-${v}.log" 2>&1 &
             pids="$pids $!"
           done
           for pid in $pids; do
@@ -110,6 +122,16 @@ jobs:
           done
           exit $exit_code
 
+      - name: Compare test durations against master
+        run: node test/common/compareDurations.js baseline durations
+
+      - name: Save duration baseline
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        uses: actions/cache/save@v4
+        with:
+          path: durations
+          key: durations-${{ strategy.job-index }}-${{ github.sha }}
+
       - name: Upload failed versions' logs and traces
         if: failure()
         uses: actions/upload-artifact@v4

package.json

@@ -5,7 +5,7 @@
   "main": "index.js",
   "types": "index.d.ts",
   "scripts": {
-    "mocha_test": "mocha --reporter spec --exit",
+    "mocha_test": "mocha --reporter ./test/common/durationsReporter.js --exit",
     "test": "npm run mocha_test",
     "pretest": "npm run lint",
     "lint": "standard && standard-markdown",

test/common/compareDurations.js

@@ -0,0 +1,32 @@
+// Usage: node test/common/compareDurations.js <baselineDir> <currentDir>
+// Fails if any test that passed in both runs got more than 1.5x slower than master's baseline.
+const fs = require('fs')
+const path = require('path')
+
+const [baselineDir, currentDir] = process.argv.slice(2)
+const FACTOR = 1.5
+// Ignore tests too short for a 1.5x jump to mean anything (server/network jitter).
+const MIN_REGRESSION_MS = 5000
+
+let regressions = 0
+for (const file of fs.readdirSync(currentDir).filter(f => f.startsWith('durations-')).sort()) {
+  const baselineFile = path.join(baselineDir, file)
+  if (!fs.existsSync(baselineFile)) {
+    console.log(`${file}: no baseline yet, skipping`)
+    continue
+  }
+  const baseline = JSON.parse(fs.readFileSync(baselineFile))
+  const current = JSON.parse(fs.readFileSync(path.join(currentDir, file)))
+  console.log(`\n${file}`)
+  for (const [title, ms] of Object.entries(current)) {
+    const base = baseline[title]
+    if (base === undefined) continue
+    const regressed = ms > base * FACTOR && ms - base > MIN_REGRESSION_MS
+    if (regressed) regressions++
+    console.log(`  ${regressed ? 'SLOWER' : '      '} ${String(base).padStart(7)}ms -> ${String(ms).padStart(7)}ms  ${title}`)
+  }
+}
+if (regressions > 0) {
+  console.error(`\n${regressions} test(s) got more than ${FACTOR}x slower than master`)
+  process.exit(1)
+}

test/common/durationsReporter.js

@@ -0,0 +1,19 @@
+// The spec reporter, plus a JSON file of { "<test full title>": <ms> } for every
+// passing test, written to $DURATIONS. CI compares it against master's run.
+const fs = require('fs')
+const { reporters, Runner } = require('mocha')
+
+class DurationsReporter extends reporters.Spec {
+  constructor (runner, options) {
+    super(runner, options)
+    const durations = {}
+    runner.on(Runner.constants.EVENT_TEST_PASS, test => {
+      durations[test.fullTitle()] = test.duration
+    })
+    runner.once(Runner.constants.EVENT_RUN_END, () => {
+      if (process.env.DURATIONS) fs.writeFileSync(process.env.DURATIONS, JSON.stringify(durations, null, 2))
+    })
+  }
+}
+
+module.exports = DurationsReporter