feat: runnable quality bar and escalate HITL (v0.7.1)

Prefer pytest over compileall for greenfield Python, run whitelisted acceptance
commands after workers, and critique until smoke/static/acceptance pass. Pause
on max-retry escalate with Studio modal + TUI /escalate (30s → re-scope).

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: piotrlaczkowski

Formula/slmcode.rb

@@ -9,7 +9,7 @@
 class Slmcode < Formula
   desc "Coding harness for SLMs and any OpenAI-compatible LLM"
   homepage "https://unicolab.ai"
-  version "0.7.0"
+  version "0.7.1"
   license "MIT"
 
   on_macos do

Makefile

@@ -1,6 +1,6 @@
 MODULE := github.com/UnicoLab/slmcode
 BIN    := slmcode
-VERSION ?= 0.7.0
+VERSION ?= 0.7.1
 PREFIX ?= $(HOME)/.local
 GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
 BUILD_TIME := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)

TODO.md

@@ -98,3 +98,12 @@ prompt/params control. Studio Pipeline tab + dynamic progress header follow the
 Also: project completeness reference bar (LangGraph/FastAPI/CLI), real-query eval suite,
 loop SSE (`kind=loop`) with continue/abort UX, preserve harness tasks when capping to 8,
 docs for pipeline + updated agents/studio/config/architecture.
+
+## Runnable quality bar + escalate HITL (v0.7.1)
+**Done.** Push SLM output past "compiles": greenfield Python QA → pytest; whitelisted
+acceptance commands run after workers; critique until smoke/static/acceptance green;
+syntax-only QA cannot alone mark run success.
+
+Mid-execute escalate no longer banner-only: pipeline pauses that task with Studio modal
++ `GET/POST /api/escalate/*`; TUI `/escalate re_scope|retry|mark_done|abort`;
+config `escalate_ask` / `escalate_ask_timeout` (default ask / 30s → re_scope).

cmd/slmcode/cmd_tui.go

@@ -7,12 +7,14 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/spf13/cobra"
 
 	"github.com/UnicoLab/slmcode/pkg/agents"
 	"github.com/UnicoLab/slmcode/pkg/cli"
 	"github.com/UnicoLab/slmcode/pkg/harness"
+	"github.com/UnicoLab/slmcode/pkg/hitl"
 	"github.com/UnicoLab/slmcode/pkg/orchestrator"
 	"github.com/UnicoLab/slmcode/pkg/plan"
 	"github.com/UnicoLab/slmcode/pkg/rewind"
@@ -124,6 +126,20 @@ func runPremiumTUI() error {
 			_ = h.Config.Save()
 			cli.KeyVal("plan_approve", h.Config.PlanApprove)
 			return false, nil
+		case "/escalate":
+			action := plan.NormalizeEscalateAction(strings.TrimSpace(arg))
+			if strings.TrimSpace(arg) == "" {
+				return false, fmt.Errorf("usage: /escalate re_scope|retry|mark_done|abort")
+			}
+			ans := plan.EscalateAnswer{
+				Action:     action,
+				AnsweredAt: time.Now().UTC().Format(time.RFC3339),
+			}
+			if err := hitl.WriteAnswers(h.Config.SlmDir(), "escalate", ans); err != nil {
+				return false, err
+			}
+			fmt.Println(cli.Success("escalate → " + action))
+			return false, nil
 		case "/history":
 			hist := sess.History()
 			if hist == nil {

cmd/slmcode/ui/app.jsx

@@ -126,7 +126,7 @@ function parseLoopEvent(e) {
 
 function PipelineHeader({
   phase, running, liveAgent, counts, intervention, turnMeter,
-  loopState, continueAsk, onContinue,
+  loopState, continueAsk, escalateAsk, onContinue,
   pipeOrder, pipeMeta, pipeGroups, slots,
 }) {
   const order = pipeOrder && pipeOrder.length ? pipeOrder : DEFAULT_PIPE;
@@ -135,7 +135,7 @@ function PipelineHeader({
   const idx = pipeIndex(phase, order);
   const total = order.length;
   const looping = !!(loopState && !LOOP_DONE_ACTIONS.has(loopState.action));
-  const awaiting = !!(continueAsk || (loopState && loopState.awaiting));
+  const awaiting = !!(continueAsk || escalateAsk || (loopState && loopState.awaiting));
   const rewindIdx = looping ? pipeIndex(loopState.to || phase, order) : -1;
   const effectiveIdx = looping && rewindIdx >= 0 ? Math.min(idx, rewindIdx) : idx;
   const doneCount = phase === "done" && !looping ? total : Math.max(0, effectiveIdx);
@@ -791,6 +791,7 @@ function App() {
   const [clarifyAnswers, setClarifyAnswers] = useState({});
   const [planAsk, setPlanAsk] = useState(null);
   const [continueAsk, setContinueAsk] = useState(null);
+  const [escalateAsk, setEscalateAsk] = useState(null);
   const [loopState, setLoopState] = useState(null);
   const [pipelineView, setPipelineView] = useState(null);
   const [pipeDraft, setPipeDraft] = useState(null);
@@ -1027,6 +1028,9 @@ function App() {
                 });
                 setClarifyAnswers(seed);
                 showToast("Scope interview — choose options or use recommended");
+              } else if (ask && ask.kind === "escalate") {
+                setEscalateAsk(ask);
+                showToast("⚠ " + (ask.task_id || "Task") + " needs your decision");
               } else if (ask && ask.kind === "continue") {
                 setContinueAsk(ask);
                 showToast("Retries exhausted — continue another wave?");
@@ -1043,16 +1047,24 @@ function App() {
             setClarifyAsk(null);
             setPlanAsk(null);
             setContinueAsk(null);
+            setEscalateAsk(null);
             setShellAsk(null);
           }
           if (e.kind === "intervention") {
             const banner = {
               code: e.scope || "quality",
               message: e.message || "harness intervention",
               detail: e.output || "",
+              taskId: e.task_id || "",
             };
             setIntervention(banner);
             showToast("⚠ " + banner.message);
+            if (banner.code === "escalate") {
+              // Recover modal if KindAsk raced / was missed.
+              api("/api/escalate/pending").then((p) => {
+                if (p && p.pending && p.ask) setEscalateAsk(p.ask);
+              }).catch(() => {});
+            }
           }
           if (e.kind === "loop") {
             const loop = parseLoopEvent(e);
@@ -1128,6 +1140,7 @@ function App() {
     setPhase("init");
     setLoopState(null);
     setContinueAsk(null);
+    setEscalateAsk(null);
     setIntervention(null);
     setNav("run"); // jump to Live so progress is obvious
     try {
@@ -1359,6 +1372,30 @@ function App() {
     showToast(labels[action] || action);
   }
 
+  async function submitEscalate(action) {
+    if (!escalateAsk) return;
+    await api("/api/escalate/answer", {
+      method: "POST",
+      body: JSON.stringify({ action }),
+    });
+    const taskId = escalateAsk.task_id || "";
+    setEscalateAsk(null);
+    setIntervention(null);
+    setLoopState((prev) => prev ? {
+      ...prev,
+      awaiting: false,
+      action: "escalate_resolved",
+      reason: (taskId || "task") + " → " + action,
+    } : prev);
+    const labels = {
+      re_scope: "Left in backlog for re-scope",
+      retry: "Retrying task",
+      mark_done: "Marked done",
+      abort: "Task aborted",
+    };
+    showToast((taskId ? taskId + ": " : "") + (labels[action] || action));
+  }
+
   async function saveConfig(patch) {
     // Send only patch keys — never round-trip Public() config (api_key "***", etc.).
     const body = { ...patch };
@@ -1583,6 +1620,7 @@ function App() {
         turnMeter={turnMeter}
         loopState={loopState}
         continueAsk={continueAsk}
+        escalateAsk={escalateAsk}
         onContinue={submitContinue}
         pipeOrder={livePipe.order}
         pipeMeta={livePipe.meta}
@@ -2503,6 +2541,39 @@ function App() {
                     <option value="off">off</option>
                   </select>
                 </label>
+                <label>Escalate ask
+                  <select
+                    value={config.escalate_ask || "ask"}
+                    onChange={(e) => saveConfig({ escalate_ask: e.target.value })}
+                  >
+                    <option value="ask">ask (pause for decision)</option>
+                    <option value="auto">auto (retry once)</option>
+                    <option value="off">off (leave in backlog)</option>
+                  </select>
+                </label>
+                <label>Escalate timeout (sec)
+                  <input
+                    type="number"
+                    min={5}
+                    max={600}
+                    defaultValue={
+                      typeof config.escalate_ask_timeout === "number"
+                        ? Math.max(5, Math.round(config.escalate_ask_timeout / 1e9))
+                        : 30
+                    }
+                    onBlur={(e) => saveConfig({ escalate_ask_timeout_sec: Number(e.target.value) })}
+                  />
+                </label>
+                <label>Continue ask
+                  <select
+                    value={config.continue_ask || "ask"}
+                    onChange={(e) => saveConfig({ continue_ask: e.target.value })}
+                  >
+                    <option value="ask">ask (after QA exhausted)</option>
+                    <option value="auto">auto (one more wave)</option>
+                    <option value="off">off</option>
+                  </select>
+                </label>
                 <label>Shell permission
                   <select
                     value={config.shell_permission || "allow"}
@@ -2655,6 +2726,45 @@ function App() {
         </div>
       ) : null}
 
+      {escalateAsk ? (
+        <div className="modal-backdrop escalate-modal" style={{
+          position: "fixed", inset: 0, background: "rgba(0,0,0,0.5)", zIndex: 85,
+          display: "flex", alignItems: "center", justifyContent: "center", padding: 16,
+        }}>
+          <div className="card" style={{
+            maxWidth: 640, width: "100%", maxHeight: "85vh", overflow: "auto",
+            background: "var(--panel)", border: "1px solid var(--border)", borderRadius: 12, padding: 16,
+          }}>
+            <h2 style={{ marginTop: 0 }}>⚠ Human review needed</h2>
+            <p className="lead">{escalateAsk.summary || "Task escalated after max retries."}</p>
+            <p style={{ color: "var(--muted)", fontSize: "0.84rem" }}>
+              Pipeline is paused on this task
+              {escalateAsk.timeout_sec ? ` · timeout ${escalateAsk.timeout_sec}s → re-scope` : ""}
+            </p>
+            <p style={{ fontSize: "0.85rem" }}>
+              <strong>{escalateAsk.task_id}</strong>
+              {escalateAsk.title ? ` — ${escalateAsk.title}` : ""}
+              {escalateAsk.role ? ` · @${escalateAsk.role}` : ""}
+            </p>
+            {(escalateAsk.files || []).length > 0 ? (
+              <p style={{ fontSize: "0.8rem" }}><strong>Files:</strong> {escalateAsk.files.join(", ")}</p>
+            ) : null}
+            {escalateAsk.detail ? (
+              <pre style={{
+                whiteSpace: "pre-wrap", fontSize: 12, maxHeight: 180, overflow: "auto",
+                background: "var(--bg)", padding: 10, borderRadius: 8,
+              }}>{escalateAsk.detail}</pre>
+            ) : null}
+            <div className="row" style={{ gap: 8, justifyContent: "flex-end", marginTop: 12, flexWrap: "wrap" }}>
+              <button className="ghost danger" onClick={() => submitEscalate("abort")}>Abort task</button>
+              <button className="ghost" onClick={() => submitEscalate("re_scope")}>Re-scope / fix later</button>
+              <button className="ghost" onClick={() => submitEscalate("mark_done")}>Mark done</button>
+              <button onClick={() => submitEscalate("retry")}>Retry now</button>
+            </div>
+          </div>
+        </div>
+      ) : null}
+
       {continueAsk ? (
         <div className="modal-backdrop" style={{
           position: "fixed", inset: 0, background: "rgba(0,0,0,0.45)", zIndex: 83,
@@ -2713,7 +2823,9 @@ function App() {
           {health?.root ? health.root.replace(/^\/Users\/[^/]+/, "~") : "…"}
         </span>
         <span className="footer-live">
-          {continueAsk || (loopState && loopState.awaiting)
+          {escalateAsk
+            ? `⏳ escalate ${escalateAsk.task_id || ""}?`
+            : continueAsk || (loopState && loopState.awaiting)
             ? "⏳ continue/abort?"
             : loopState && !LOOP_DONE_ACTIONS.has(loopState.action)
               ? `↺ ${LOOP_LABELS[loopState.action] || "loop"}${loopState.wave ? " W" + loopState.wave : ""}`

cmd/slmcode/version.go

@@ -4,7 +4,7 @@ package main
 //
 //	go build -ldflags "-X main.Version=0.5.0 -X main.SourceRoot=/path -X main.GitCommit=abc -X main.BuildTime=…"
 var (
-	Version    = "0.7.0"
+	Version    = "0.7.1"
 	SourceRoot = "" // absolute path to the slmcode checkout used to build this binary
 	GitCommit  = "unknown"
 	BuildTime  = "unknown"

docs/changelog.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v0.7.1 — Runnable quality bar + escalate HITL
+
+### Highlights
+- Greenfield Python QA defaults to **pytest** (fail closed), not `compileall`
+- **Acceptance smoke** — whitelisted commands from task acceptance run after each worker
+- Worker critique loops until smoke/static/acceptance green (bounded by `max_retries`)
+- Run success requires a **strong** QA gate — syntax-only cannot rubber-stamp incomplete boards
+- **Escalate HITL** — max-retry escalate pauses the task with a Studio modal (or TUI
+  `/escalate`); options: re-scope / retry / mark done / abort; default timeout **30s**
+  → re-scope (`escalate_ask`, `escalate_ask_timeout`)
+
+### API
+- `GET` / `POST` `/api/escalate/pending|answer`
+
+---
+
 ## v0.7.0 — Pipeline control + reference quality
 
 ### Highlights

docs/config.md

@@ -126,6 +126,10 @@ qa_gate: true
 qa_gate_command: ""       # empty = auto-detect (go/pytest/uv/npm/compileall)
 qa_gate_max_rounds: 3
 post_worker_smoke: true   # py_compile / go test after each worker before review
+escalate_ask: ask         # ask | auto | off — pause on max-retry escalate
+escalate_ask_timeout: 30s # timeout → re_scope (Studio modal / TUI /escalate)
+continue_ask: ask         # ask | auto | off — after QA exhausted
+continue_ask_timeout: 2m
 ```
 
 ### Planning / scope
@@ -148,17 +152,24 @@ exit blocks the tool. PostToolUse can run `compileall` after writes.
 `/rewind list` / `/rewind <id>`, API `GET/POST /api/rewind`. Real context
 compact: `/compact context` or `POST /api/compact`.
 
-### QA / smoke
+### QA / smoke / acceptance
 
 After workers, `post_worker_smoke` runs a fast deterministic check (`python -m
 py_compile` / `go test -short`) and blocks approve-on-disk-only when it fails.
 
+When a task's acceptance text includes a **whitelisted** command (`python -m
+pytest`, `go test`, `python main.py`, …), the harness also runs **Acceptance
+smoke** and rejects the task until those commands exit 0. Free-form prose in
+acceptance is never executed as shell.
+
+`worker_critique` keeps refining (up to `max_retries`) while smoke / static /
+acceptance sections stay red — not just a single self-fix pass.
+
 After the finalize tester, `qa_gate` runs a real project command (and bootstraps
 deps when needed: `pip install -r requirements.txt`, `uv sync`, `go mod tidy`).
-Auto-detect covers `go test`, `python -m pytest` / `uv run pytest`,
-`python -m compileall`, and `npm test` / `cargo test` / `make test` — never
-`--help` as a quality proof. On failure, tester diagnoses → corrector patches →
-re-run.
+Auto-detect prefers `pytest` for greenfield Python (`main.py` +
+`requirements.txt`), not `compileall`. Syntax-only gates cannot alone mark the
+run successful. On failure, tester diagnoses → corrector patches → re-run.
 
 ---
 

docs/install.md

@@ -75,7 +75,7 @@ One-liners fetch a shiny GitHub Release. Keep the compiler for contributing (or
 
     ```bash
     curl -fsSL https://raw.githubusercontent.com/UnicoLab/smlcode/main/scripts/install-remote.sh \
-      | bash -s -- --version v0.7.0
+      | bash -s -- --version v0.7.1
     ```
 
 === "🪟 PowerShell"

docs/studio.md

@@ -55,6 +55,32 @@ The loop reloads `board.json` each wave. Chaos, but *structured* chaos.
 
 ---
 
+## Human-in-the-loop modals ✋
+
+Studio blocks the relevant step with a modal (pipeline header shows **Awaiting you**):
+
+| Modal | When | Options | Timeout default |
+|-------|------|---------|-----------------|
+| Clarify | vague PRD interview | pick options | 2m → recommended |
+| Plan approve | before execute | approve / replan | 2m → approve |
+| **Escalate** | task hits max review retries | **re-scope / retry / mark done / abort** | **30s → re-scope** |
+| Continue | QA/retries exhausted | continue / stop / flag | 2m → stop |
+| Shell | `shell_permission=ask` | approve / deny | 2m |
+
+Config (Settings → Planning / scope, or YAML):
+
+```yaml
+escalate_ask: ask              # ask | auto | off
+escalate_ask_timeout: 30s
+continue_ask: ask
+```
+
+TUI: `/escalate re_scope|retry|mark_done|abort` while the banner shows escalate pending.
+
+API: `GET /api/escalate/pending` · `POST /api/escalate/answer` `{"action":"retry"}`.
+
+---
+
 ## Live events (SSE) 📡
 
 Same stream as `slmcode run -v`:
@@ -86,6 +112,8 @@ curl -N http://127.0.0.1:7420/api/events
 | `GET` | `/api/skills` `/api/models` |
 | `POST` | `/api/runs` `/api/runs/stop` |
 | `GET` | `/api/runs/latest` `/api/events` |
+| `GET`/`POST` | `/api/escalate/pending` · `/api/escalate/answer` |
+| `GET`/`POST` | `/api/continue/pending` · `/api/continue/answer` |
 | `GET` | `/` SPA |
 
 → Full pipeline schema: [Pipeline](pipeline.md)