From 40a86e4ce31df669529dcbd0e909fbe81fe3ea55 Mon Sep 17 00:00:00 2001 From: Andrei Radu Date: Sat, 29 Aug 2026 13:46:37 +0200 Subject: [PATCH 1/5] test: retarget DecodeBench assertions at the lab structured protocol The 'DecodeBench uses Showcase structural prompts' test still asserts pickShowcasePrompts("structural") and withFillToMaxInstruction, both of which 1.8.3 removed when DecodeBench moved onto the lab structured protocol (count 1 -> 200), and 1.8.4 replaced with the output-type picker. The test therefore fails on a clean main. Retargeted at the current API (pickDecodeBenchPrompts / decodeBenchPromptForType / normalizeDecodeBenchType) and added top_p: 1 plus a negative assertion on withFillToMaxInstruction, so the protocol is pinned in the direction it actually moved. The remaining assertions are unchanged and still pass. --- server/collectors/__tests__/showcasePrompts.test.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/collectors/__tests__/showcasePrompts.test.js b/server/collectors/__tests__/showcasePrompts.test.js index 57ce6ea2..5910d12e 100644 --- a/server/collectors/__tests__/showcasePrompts.test.js +++ b/server/collectors/__tests__/showcasePrompts.test.js @@ -91,12 +91,18 @@ test("catalog prompts exist for text, structural, and mixed pickers", () => { assert.ok(textCount >= 2); }); -test("DecodeBench uses Showcase structural prompts at temperature 0, thinking off", () => { - assert.match(benchSrc, /pickShowcasePrompts\("structural"/); - assert.match(benchSrc, /withFillToMaxInstruction/); +test("DecodeBench uses the lab structured protocol at temperature 0, thinking off", () => { + // 1.8.3 moved DecodeBench off the Showcase structural catalog + fill-to-max + // and onto the lab structured protocol (count 1 -> 200); 1.8.4 added the + // output-type picker. These assertions still referenced the removed API. + assert.match(benchSrc, /pickDecodeBenchPrompts\(/); + assert.match(benchSrc, /decodeBenchPromptForType\(/); + assert.match(benchSrc, /normalizeDecodeBenchType\(/); assert.match(benchSrc, /temperature:\s*0/); + assert.match(benchSrc, /top_p:\s*1/); assert.match(benchSrc, /applyThinkingFlags\(body,\s*modelId,\s*false\)/); assert.match(benchSrc, /min_tokens:\s*maxTokens/); + assert.doesNotMatch(benchSrc, /withFillToMaxInstruction/); assert.doesNotMatch(benchSrc, /uniquePrefillPrefix/); assert.doesNotMatch(benchSrc, /BENCH_PROMPTS/); }); From bd7eb601f1b133500737d517d9836c40c56a3e5b Mon Sep 17 00:00:00 2001 From: 0xdfi <259524292+0xdfi@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:24:18 -0400 Subject: [PATCH 2/5] fix: avoid recursive proc net fallback --- server/collectors/SystemCollector.js | 2 +- .../__tests__/SystemCollector.hostNet.test.js | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 server/collectors/__tests__/SystemCollector.hostNet.test.js diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index 9ac3a518..acc02233 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -1448,7 +1448,7 @@ export class SystemCollector { }); } } - return this._readHostFile(`/proc/net/${relPath}`); + return fs.readFileSync(`/proc/net/${relPath}`, "utf-8"); } /** Lightweight liveness for local Sparks. */ diff --git a/server/collectors/__tests__/SystemCollector.hostNet.test.js b/server/collectors/__tests__/SystemCollector.hostNet.test.js new file mode 100644 index 00000000..d1388378 --- /dev/null +++ b/server/collectors/__tests__/SystemCollector.hostNet.test.js @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import test from "node:test"; + +import { SystemCollector } from "../SystemCollector.js"; + +function localSpark() { + return { + id: "local-test", + name: "Local Test", + isLocal: true, + lanIp: "127.0.0.1", + }; +} + +test("host network file fallback reads container proc without recursing", async (t) => { + const collector = new SystemCollector(localSpark()); + collector._hasHostProc = () => false; + const reads = []; + t.mock.method(fs, "readFileSync", (filePath, encoding) => { + reads.push({ filePath, encoding }); + return "Inter-| Receive | Transmit\n"; + }); + + const contents = await collector._readHostNetFile("dev"); + + assert.match(contents, /Inter-\|/); + assert.deepEqual(reads, [{ filePath: "/proc/net/dev", encoding: "utf-8" }]); +}); From 60c7ff35e86fa13d5ff78eb04c67ef17f026679d Mon Sep 17 00:00:00 2001 From: 0xdfi <259524292+0xdfi@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:21:46 -0400 Subject: [PATCH 3/5] fix: include showcase fixture in production image --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 93ce2d02..8a19596a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,7 @@ COPY --from=builder /app/package.json ./package.json COPY --from=builder /app/package-lock.json ./package-lock.json COPY --from=builder /app/server ./server COPY --from=builder /app/src/shared ./src/shared +COPY --from=builder /app/src/components/ShowcasePage/showcasePrompts.ts ./src/components/ShowcasePage/showcasePrompts.ts COPY --from=builder /app/config ./config # Volume for persistent sparks.json From 41f6869946745d29a6bfee7d0f8d81af98c856c2 Mon Sep 17 00:00:00 2001 From: 0xdfi <259524292+0xdfi@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:05:42 -0400 Subject: [PATCH 4/5] test: make LLM rate timing deterministic --- .../__tests__/LlmProbe.exl3.test.js | 8 +++-- .../LlmProbe.vllm-llamacpp.regression.test.js | 34 ++++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/server/collectors/__tests__/LlmProbe.exl3.test.js b/server/collectors/__tests__/LlmProbe.exl3.test.js index f9839699..ae132b43 100644 --- a/server/collectors/__tests__/LlmProbe.exl3.test.js +++ b/server/collectors/__tests__/LlmProbe.exl3.test.js @@ -121,13 +121,15 @@ test("_applyExl3Health: counter diffs → tok/s; idle → 0", () => { assert.equal(probe.prefillTps, 40); }); -test("probe: exl3 path does not mislabel as vllm", async () => { +test("probe: exl3 path does not mislabel as vllm", async (t) => { + const now = 10_000; + t.mock.method(Date, "now", () => now); const probe = new LlmProbe({ lanIp: "127.0.0.1" }, 8888); probe.serverIsOpenAI = true; probe.backendType = "exl3"; probe.authOpen = true; - probe._lastDetectAt = Date.now(); - probe.lastProbeTime = Date.now() - 2000; + probe._lastDetectAt = now; + probe.lastProbeTime = now - 2000; probe.lastTokenCounts = { input: 100, output: 50 }; probe._fetch = async (url) => { const u = String(url); diff --git a/server/collectors/__tests__/LlmProbe.vllm-llamacpp.regression.test.js b/server/collectors/__tests__/LlmProbe.vllm-llamacpp.regression.test.js index cd2af56d..684ac1a0 100644 --- a/server/collectors/__tests__/LlmProbe.vllm-llamacpp.regression.test.js +++ b/server/collectors/__tests__/LlmProbe.vllm-llamacpp.regression.test.js @@ -36,6 +36,12 @@ function textRes(txt, status = 200) { }; } +function freezeProbeClock(t) { + const now = 10_000; + t.mock.method(Date, "now", () => now); + return now; +} + test("vLLM detect: /v1/models + vllm /metrics → vllm (not ds4/sglang)", async () => { const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8000); const hits = []; @@ -58,13 +64,14 @@ test("vLLM detect: /v1/models + vllm /metrics → vllm (not ds4/sglang)", async assert.ok(!hits.includes("/get_server_info") || hits.includes("/metrics")); }); -test("vLLM probe: counter diffs + tiles; skips get_server_info when known vllm", async () => { +test("vLLM probe: counter diffs + tiles; skips get_server_info when known vllm", async (t) => { + const now = freezeProbeClock(t); const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8000); probe.serverIsOpenAI = true; probe.backendType = "vllm"; probe.authOpen = true; - probe._lastDetectAt = Date.now(); - probe.lastProbeTime = Date.now() - 2000; + probe._lastDetectAt = now; + probe.lastProbeTime = now - 2000; probe.lastTokenCounts = { input: 1000, output: 500 }; const hits = []; probe._fetch = async (url) => { @@ -219,13 +226,14 @@ test("llama.cpp detect: /slots array wins over OpenAI paths", async () => { assert.equal(probe.backendType, "llama.cpp"); }); -test("llama.cpp probe: slot deltas → tok/s; props for model", async () => { +test("llama.cpp probe: slot deltas → tok/s; props for model", async (t) => { + const now = freezeProbeClock(t); const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8080); probe.serverIsOpenAI = false; probe.backendType = "llama.cpp"; probe.authOpen = true; - probe._lastDetectAt = Date.now(); - probe.lastProbeTime = Date.now() - 2000; + probe._lastDetectAt = now; + probe.lastProbeTime = now - 2000; probe.slotState.set(0, { decoded: 10, prompted: 5 }); probe._fetch = async (url) => { const u = String(url); @@ -268,13 +276,14 @@ test("llama.cpp probe: slot deltas → tok/s; props for model", async () => { assert.equal(snap.uncachedPrefillTps, null); }); -test("llama.cpp probe: n_prompt_tokens_cache → cached vs uncached prefill", async () => { +test("llama.cpp probe: n_prompt_tokens_cache → cached vs uncached prefill", async (t) => { + const now = freezeProbeClock(t); const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8080); probe.serverIsOpenAI = false; probe.backendType = "llama.cpp"; probe.authOpen = true; - probe._lastDetectAt = Date.now(); - probe.lastProbeTime = Date.now() - 2000; + probe._lastDetectAt = now; + probe.lastProbeTime = now - 2000; probe.slotState.set(0, { decoded: 10, prompted: 5 }); probe.lastPrefillKinds = { cached: 10, computed: 5 }; probe._fetch = async (url) => { @@ -299,13 +308,14 @@ test("llama.cpp probe: n_prompt_tokens_cache → cached vs uncached prefill", as assert.equal(snap.cachedPrefillTps, 15); // (40-10)/2 }); -test("llama.cpp: n_prompt_tokens_processed 0 is not treated as missing", async () => { +test("llama.cpp: n_prompt_tokens_processed 0 is not treated as missing", async (t) => { + const now = freezeProbeClock(t); const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8080); probe.serverIsOpenAI = false; probe.backendType = "llama.cpp"; probe.authOpen = true; - probe._lastDetectAt = Date.now(); - probe.lastProbeTime = Date.now() - 2000; + probe._lastDetectAt = now; + probe.lastProbeTime = now - 2000; probe.slotState.set(0, { decoded: 10, prompted: 0 }); probe.lastPrefillKinds = { cached: 10, computed: 0 }; probe._fetch = async (url) => { From 9e9b0968bf849cc4ab680e4acf6c669f613a296e Mon Sep 17 00:00:00 2001 From: Pi Agent Date: Mon, 17 Aug 2026 00:25:07 +0300 Subject: [PATCH 5/5] Show NVRM NV_ERR_NO_MEMORY on the GPU panel, polled once a minute. Kernel journal scan is cached off the 2s GPU loop. Count is since boot; the row is hidden at zero. No CPU panel. Co-authored-by: Cursor --- CHANGELOG.md | 1 + README.md | 1 + server/collectors/SystemCollector.js | 50 ++++++++++++++++++- .../__tests__/SystemCollector.nvErr.test.js | 16 ++++++ server/config.js | 3 ++ src/api/types.ts | 2 + src/components/SparkPage/GpuPanel.tsx | 12 +++++ 7 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 server/collectors/__tests__/SystemCollector.nvErr.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d52e9dd..3130d01c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ Format: version sections are listed newest first. ### Added - **EXL3 live tok/s** — detect ExLlamaV3 `tools/serve_openai.py` (`owned_by: exl3` or `/health` `{ok, busy}`) instead of mislabeling it as vLLM. Generation and prefill tok/s come from `/health` cumulative token counters (no Prometheus `/metrics`). - **Tailnet monitoring** — opt-in per unit (`tailscaleMonitoring`, default **off**); `tailscale status --json` on the host and a Tailnet card under Resources. Flags a unit that is healthy on the LAN but off its tailnet. ([#43](https://github.com/MiaAI-Lab/sparkDash/pull/43)) +- **NV_ERR_NO_MEMORY on the GPU panel** — count of NVRM `NV_ERR_NO_MEMORY` kernel log lines since boot (shown when > 0). Journal is scanned at most once a minute, not on the 2s poll. Replaces the approach in [#40](https://github.com/MiaAI-Lab/sparkDash/pull/40). ### Security - **`BIND_HOST` now defaults to `127.0.0.1` (loopback) instead of `0.0.0.0`** — the dashboard is unauthenticated and can SSH into and power off Sparks, so it is no longer reachable on the LAN by default. Set `BIND_HOST` to the host's LAN IP (or `0.0.0.0`) to opt in to remote access. **Migration:** if you access sparkDash from another machine via bare-metal `npm start`, set `BIND_HOST` explicitly. Production and dev Compose both set `BIND_HOST=0.0.0.0` (`network_mode: host`). Startup now also warns when bound to a non-loopback address. ([#35](https://github.com/MiaAI-Lab/sparkDash/pull/35)) diff --git a/README.md b/README.md index 9a37d93a..bef483eb 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,7 @@ Copy `.env.example` to `.env` if needed: | `POLL_INTERVAL_HERMES` | `600000` | Hermes Agent update check poll (ms) | | `POLL_INTERVAL_TAILSCALE` | `30000` | Tailnet probe poll (ms) | | `TAILSCALE_PROBE_TIMEOUT_MS` | `8000` | Timeout for `tailscale status --json` (ms) | +| `POLL_INTERVAL_NVERR` | `60000` | Kernel journal scan for NVRM `NV_ERR_NO_MEMORY` (ms) | | `HERMES_UPDATE_TIMEOUT_MS` | `600000` | Hard timeout for running `hermes update` over SSH (ms) | | `POLL_INTERVAL_LIVENESS` | `5000` | Online/SSH liveness check (ms) | | `SPARKDASH_SECRETS_KEY` | _(auto)_ | Passphrase or 64-char hex for secret encryption | diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index acc02233..2c2c9d05 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -1,9 +1,24 @@ import fs from "fs"; import path from "path"; -import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS } from "../config.js"; +import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS, POLL_INTERVAL_NVERR } from "../config.js"; import { normalizeMac, WOL_INTERFACE } from "../wol.js"; import { sshExec } from "./ssh.js"; +const NVERR_JOURNAL_CMD = + 'journalctl -k --no-pager -q --grep=NV_ERR_NO_MEMORY 2>/dev/null | grep -c NV_ERR_NO_MEMORY || true'; + +/** + * Parse `grep -c` stdout into a non-negative integer. Exported for tests. + * @param {unknown} raw + * @returns {number} + */ +export function parseNvErrNoMemoryCount(raw) { + const line = String(raw ?? "").trim().split("\n").pop() ?? ""; + const n = Number.parseInt(line, 10); + if (!Number.isFinite(n) || n < 0) return 0; + return n; +} + /** * SystemCollector — collects hardware metrics for a Spark. * In Phase 2, this is the LOCAL path only (no SSH). @@ -32,6 +47,8 @@ export class SystemCollector { // Cached hardware info this._hardwareInfo = null; + /** Cached NVRM NV_ERR_NO_MEMORY count (slow journal scan). */ + this._nvErrCache = { count: 0, at: 0 }; } /** Collect GPU metrics (temperature, usage, power, VRAM). */ @@ -157,6 +174,7 @@ export class SystemCollector { vram, processes, throttle: gpu.throttle, + nvErrNoMemory: await this._nvErrNoMemory(), }; } @@ -994,6 +1012,7 @@ export class SystemCollector { vram: { used: usedMB, total: totalMB, percentage, available: availableMB }, processes, throttle: gpu.throttle, + nvErrNoMemory: await this._nvErrNoMemory(), }; } catch (err) { console.error(`[SystemCollector] Remote GPU error for ${this.spark.id}:`, err.message); @@ -1499,6 +1518,34 @@ export class SystemCollector { return fs.promises.statfs(dir); } + /** + * Count NVRM `NV_ERR_NO_MEMORY` lines in the kernel journal since boot. + * Cached for POLL_INTERVAL_NVERR — never on the 2s GPU/memory loop uncached. + * @returns {Promise} + */ + async _nvErrNoMemory() { + const now = Date.now(); + if (this._nvErrCache.at > 0 && now - this._nvErrCache.at < POLL_INTERVAL_NVERR) { + return this._nvErrCache.count; + } + try { + let out; + if (this.spark.isLocal) { + out = this._hasHostProc() + ? await this._execOnHost(NVERR_JOURNAL_CMD) + : await this._exec(NVERR_JOURNAL_CMD); + } else { + out = await sshExec(this.spark, NVERR_JOURNAL_CMD, { timeoutMs: 8000 }); + } + const count = parseNvErrNoMemoryCount(out); + this._nvErrCache = { count, at: now }; + return count; + } catch { + this._nvErrCache.at = now; + return this._nvErrCache.count; + } + } + // ─── Default metrics ───────────────────────────────────── _defaultGpu() { return { @@ -1508,6 +1555,7 @@ export class SystemCollector { vram: { used: 0, total: 0, percentage: 0, available: 0 }, processes: [], throttle: this._defaultThrottle(), + nvErrNoMemory: 0, }; } diff --git a/server/collectors/__tests__/SystemCollector.nvErr.test.js b/server/collectors/__tests__/SystemCollector.nvErr.test.js new file mode 100644 index 00000000..5a4036db --- /dev/null +++ b/server/collectors/__tests__/SystemCollector.nvErr.test.js @@ -0,0 +1,16 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseNvErrNoMemoryCount } from "../SystemCollector.js"; + +test("parseNvErrNoMemoryCount reads grep -c output", () => { + assert.equal(parseNvErrNoMemoryCount("12"), 12); + assert.equal(parseNvErrNoMemoryCount("0"), 0); + assert.equal(parseNvErrNoMemoryCount(" 43\n"), 43); +}); + +test("parseNvErrNoMemoryCount defaults invalid input to 0", () => { + assert.equal(parseNvErrNoMemoryCount(""), 0); + assert.equal(parseNvErrNoMemoryCount("not-a-number"), 0); + assert.equal(parseNvErrNoMemoryCount(undefined), 0); + assert.equal(parseNvErrNoMemoryCount("-3"), 0); +}); diff --git a/server/config.js b/server/config.js index 5297cc6d..2bbf2aac 100644 --- a/server/config.js +++ b/server/config.js @@ -34,6 +34,8 @@ const POLL_INTERVAL_LLM = parseInt(process.env.POLL_INTERVAL_LLM || "2000", 10); const POLL_INTERVAL_COMFY = parseInt(process.env.POLL_INTERVAL_COMFY || "2000", 10); // Tailnet membership changes slowly; each poll is an SSH round-trip. const POLL_INTERVAL_TAILSCALE = parseInt(process.env.POLL_INTERVAL_TAILSCALE || "30000", 10); +// Kernel journal scan for NV_ERR_NO_MEMORY — not on the 2s GPU loop. +const POLL_INTERVAL_NVERR = parseInt(process.env.POLL_INTERVAL_NVERR || "60000", 10); // dmon -c 1 -d 1 blocks ~1s; default 2s avoids stacking with in-flight guards const POLL_INTERVAL_BANDWIDTH = parseInt(process.env.POLL_INTERVAL_BANDWIDTH || "2000", 10); // Dedicated liveness (sshTest / local ping) cadence — not a metric domain. @@ -106,6 +108,7 @@ export { POLL_INTERVAL_LLM, POLL_INTERVAL_COMFY, POLL_INTERVAL_TAILSCALE, + POLL_INTERVAL_NVERR, POLL_INTERVAL_BANDWIDTH, POLL_INTERVAL_LIVENESS, POLL_INTERVAL_HERMES, diff --git a/src/api/types.ts b/src/api/types.ts index 440fa09d..89591781 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -211,6 +211,8 @@ export interface GpuMetrics { processes?: Array<{ pid: number; name: string; vramMB: number }>; /** NVIDIA clock throttle / thermal slowdown state from nvidia-smi. */ throttle?: GpuThrottle | null; + /** Kernel NVRM NV_ERR_NO_MEMORY count since boot (cached ~60s). */ + nvErrNoMemory?: number; } // ─── CPU metrics ───────────────────────────────────────── diff --git a/src/components/SparkPage/GpuPanel.tsx b/src/components/SparkPage/GpuPanel.tsx index ceecee29..d1927c6e 100644 --- a/src/components/SparkPage/GpuPanel.tsx +++ b/src/components/SparkPage/GpuPanel.tsx @@ -202,6 +202,18 @@ export function GpuPanel({ gpu, cpu, sparkId, temperatureUnit, className }: GpuP )} + {(gpu?.nvErrNoMemory ?? 0) > 0 && ( +
+ NV_ERR_NO_MEMORY + + {gpu?.nvErrNoMemory} + +
+ )} + {/* Top GPU processes by VRAM usage */} {gpu && gpu.processes && gpu.processes.length > 0 && (