diff --git a/.github/workflows/ci-l0-checks.yml b/.github/workflows/ci-l0-checks.yml index 4c22ceb3d..58aa5909b 100644 --- a/.github/workflows/ci-l0-checks.yml +++ b/.github/workflows/ci-l0-checks.yml @@ -83,6 +83,7 @@ jobs: e2e_workflow/scripts/tests/test_attribute_weights_edges.py \ e2e_workflow/scripts/tests/test_op_bench.py \ e2e_workflow/scripts/tests/test_harness_lib.py \ + e2e_workflow/scripts/tests/test_leg_runner.py \ e2e_workflow/scripts/tests/test_server_teardown.py \ e2e_workflow/scripts/tests/test_bench_e2e_teardown_lookup.py \ geak/test_bootstrap.py diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index 3fddc9244..d78d30a51 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -381,8 +381,9 @@ const E2E_REPEATS = parseInt(A.e2e_repeats != null ? A.e2e_repeats : 2, 10); // This is what guarantees "every A/B runs ref AND cand to completion regardless // of pass/fail" — general, not per-kernel. Bump via args.ab_finish_retries. const AB_FINISH_RETRIES = parseInt(A.ab_finish_retries != null ? A.ab_finish_retries : 3, 10); -// A resolvable FROZEN baseline (baseline_src/ frozen OR importable meta.baseline_callable) is the -// speedup DENOMINATOR and is MANDATORY. If an extraction smoke-passes but froze no baseline, the +// A resolvable FROZEN baseline (a seeded baseline_overlay/ + a declared meta.candidate_bind, or an +// importable meta.baseline_callable on the op track) is the speedup DENOMINATOR and is MANDATORY. +// If an extraction smoke-passes but froze no baseline, the // unittest would silently time the candidate against its own naive same-language scaffold (the // "optimized-HIP vs naive-HIP = fake 15.7× isolated, ~0% e2e" bug). When that happens we RE-EXTRACT // up to this many times; if still missing, the extraction is treated as a FAILURE (flag dominant / @@ -469,7 +470,7 @@ const EXTRACT_OP_SCHEMA = obj({ candidate_backends: arrStr, reference_io_sha256: { type: 'string' }, target_callable: { type: 'string' }, // module:attr rebind seam for an authored kernel ('' if none) baseline_callable: { type: 'string' }, // module:attr of the FROZEN real online kernel (the speedup denominator) - baseline_frozen: { type: 'boolean' }, // true only when baseline_src/ was frozen OR baseline_callable resolves + baseline_frozen: { type: 'boolean' }, // true only when baseline_callable resolves outside the task dir smoke: { type: 'string' }, notes: { type: 'string' }, }, ['op_kind', 'task_dir', 'smoke']); @@ -493,8 +494,11 @@ const EXTRACT_SCHEMA = obj({ source_path_in_sglang: { type: 'string' }, target_callable: { type: 'string' }, num_cases: { type: 'number' }, regimes_captured: arrStr, candidate_backends: arrStr, build: { type: 'boolean' }, unittest_smoke: { type: 'string' }, - baseline_callable: { type: 'string' }, // module:attr of the FROZEN real online kernel (the speedup denominator) - baseline_frozen: { type: 'boolean' }, // true only when baseline_src/ was frozen OR baseline_callable resolves + // the baseline leg is an ENVIRONMENT (baseline_overlay/ = a frozen CURRENT_OVERLAY snapshot), and + // candidate_bind is the ONE entry layered on top of it to make the candidate leg. + candidate_bind: { type: 'object', additionalProperties: true }, + baseline_overlay: { type: 'string' }, + baseline_frozen: { type: 'boolean' }, // true only when baseline_overlay/ was seeded AND candidate_bind is declared reference_io_sha256: { type: 'string' }, notes: { type: 'string' }, }, ['editable', 'task_dir', 'unittest_smoke']); @@ -710,10 +714,12 @@ async function ensureFlydslGate() { } } -// A FROZEN baseline is resolvable when the extractor either froze baseline_src/ (baseline_frozen) -// OR set an importable meta.baseline_callable. That is the language-independent speedup denominator. +// A FROZEN baseline is resolvable when the extractor seeded baseline_overlay/ + declared +// meta.candidate_bind (kernel track), or set an importable meta.baseline_callable (op track). +// That is the language-independent speedup denominator. const hasFrozenBaseline = (ext) => !!(ext && (ext.baseline_frozen === true || + (ext.candidate_bind && typeof ext.candidate_bind === 'object') || (typeof ext.baseline_callable === 'string' && ext.baseline_callable.trim() !== ''))); // Run a kernel_extractor agent and GUARANTEE it froze a real baseline. safeAgent already retries @@ -731,14 +737,14 @@ async function extractWithBaseline(role, phase, intro, inputs, opts) { while (smokeOk(ext) && !hasFrozenBaseline(ext) && tries < BASELINE_EXTRACT_RETRIES) { tries++; log(` ${(opts && opts.label) || role}: extraction froze NO baseline ` + - `(baseline_src/ or meta.baseline_callable) — the speedup denominator would fall back to the ` + + `(baseline_overlay/ + meta.candidate_bind) — the speedup denominator would fall back to the ` + `candidate's own scaffold (fake-win). RE-EXTRACTING (retry ${tries}/${BASELINE_EXTRACT_RETRIES}).`); ext = await safeAgent( roleAgent(role, phase, - intro + ' PRIOR ATTEMPT DID NOT FREEZE A BASELINE. You MUST freeze the real online kernel into ' + - 'an immutable baseline_src/ and set meta.baseline_callable (the speedup denominator), bind the ' + - "unittest's baseline leg to it, then return baseline_frozen:true. An extraction with no frozen " + - 'baseline is INVALID and will be discarded.', + intro + ' PRIOR ATTEMPT DID NOT FREEZE A BASELINE. You MUST seed baseline_overlay/ from ' + + 'CURRENT_OVERLAY (the live serving stack = the speedup denominator), declare meta.candidate_bind ' + + '(the ONE overlay entry built from kernel_src/), prove both legs differ via h.assert_legs_differ, ' + + 'then return baseline_frozen:true. An extraction with no frozen baseline is INVALID and will be discarded.', inputs), opts); } @@ -747,7 +753,7 @@ async function extractWithBaseline(role, phase, intro, inputs, opts) { `re-extractions — ABORTING this extraction (refusing a fake speedup vs the candidate's own scaffold).`); return { ...ext, smoke: 'fail', unittest_smoke: 'fail', notes: `no frozen baseline after ${BASELINE_EXTRACT_RETRIES} re-extractions ` + - `(baseline_src/ / meta.baseline_callable required as the speedup denominator) — ${ext.notes || ''}` }; + `(baseline_overlay/ + meta.candidate_bind required as the speedup denominator) — ${ext.notes || ''}` }; } return ext; } @@ -1254,7 +1260,7 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { 'kernel_extractor', 'extract_op', 'Build a standalone op unittest for a head kernel.', { EVAL_DIR, MODEL_PATH, GPU_ID: GPU_LIST[0], WORKLOAD, KERNEL: h, GEMM_SYNTH: gemmSynthFor(h), ...(profile && profile.profile_workload_json ? { PROFILE_WORKLOAD_JSON: profile.profile_workload_json } : {}), - CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, + CURRENT_OVERLAY: curOverlay, CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, REQUIRE_DECODE_BUCKET: true, DECODE_M_BUCKETS: [1, CONC], PREFILL_M_NOTE: 'also include the profiled large prefill M (chunk size, ~thousands) per (N,K)', }, @@ -1614,7 +1620,7 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { 'kernel_extractor', 'extract_op', 'Build a standalone op unittest for a head kernel.', { EVAL_DIR, MODEL_PATH, GPU_ID: gpu, WORKLOAD, KERNEL: h, GEMM_SYNTH: gemmSynthFor(h), ...(profile && profile.profile_workload_json ? { PROFILE_WORKLOAD_JSON: profile.profile_workload_json } : {}), - CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, + CURRENT_OVERLAY: curOverlay, CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, REQUIRE_DECODE_BUCKET: true, DECODE_M_BUCKETS: [1, CONC], PREFILL_M_NOTE: 'also include the profiled large prefill M (chunk size, ~thousands) per (N,K)', }, @@ -1814,7 +1820,7 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { 'kernel_extractor', 'extract_op', 'Build a standalone op unittest for a head kernel.', { EVAL_DIR, MODEL_PATH, GPU_ID: h.gpu_id, WORKLOAD, KERNEL: h, GEMM_SYNTH: gemmSynthFor(h), ...(profile && profile.profile_workload_json ? { PROFILE_WORKLOAD_JSON: profile.profile_workload_json } : {}), - CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, + CURRENT_OVERLAY: curOverlay, CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, // The unittest MUST span BOTH regimes. Steady-state serving is decode/TPOT-bound, so a // head GEMM tuned only on GPU-time-dominant prefill M regresses decode and loses e2e. // Pass the decode M explicitly (= running batch ≈ conc) so it is never dropped, plus a @@ -2138,7 +2144,7 @@ while (want('kernel') && !TIME_DEADLINE_HIT && dispatched < BUDGET && (dispatche const ext = await extractWithBaseline( 'kernel_extractor', 'extract', 'Capture shapes + oracle; emit an immutable unittest task dir.', { EVAL_DIR, MODEL_PATH, GPU_ID: c.gpu_id, WORKLOAD, KERNEL: c, - CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, + CURRENT_OVERLAY: curOverlay, CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, ...(profile && profile.profile_workload_json ? { PROFILE_WORKLOAD_JSON: profile.profile_workload_json } : {}), }, { phase: 'Milestone', label: `extract ${c.short_name}`, schema: EXTRACT_SCHEMA }); diff --git a/e2e_workflow/roles/e2e_integrator.md b/e2e_workflow/roles/e2e_integrator.md index e823cefe3..56692e5d7 100644 --- a/e2e_workflow/roles/e2e_integrator.md +++ b/e2e_workflow/roles/e2e_integrator.md @@ -113,6 +113,11 @@ unchanged; you just also persist the diagnostics the deep feedback/harness-refin extraction (anti-cheating). If tampered → REJECT. (For a synthesized-GEMM op task with no `reference_io.pt`, instead confirm `meta.json` shapes/dtype are unchanged.) 2. **Build the candidate config/overlay** = current accepted + this ONE change, by `winner_kind`: + > **🔴 If the task dir declares `meta.candidate_bind`, REPLAY IT VERBATIM — do not improvise a seam.** + > That is the exact overlay entry the isolated unittest measured its win with (`kind:"module"` → + > `add-module --module --patched-file /`; `kind:"rebind"` → `add-rebind --target + > --impl-module/--impl-attr`), always seeded `--from "$CURRENT_OVERLAY"`. The branches below + > are for tasks that carry no `candidate_bind`. - **env** (TunableOp CSV, `HIPBLASLT_TUNING_FILE`, …): candidate env = `CURRENT_ENV + KERNEL_RESULT.apply_env`. Keep the tuning artifact under `$EVAL_DIR/config/` so it's reproducible. **Backend-engagement prerequisite:** a tuning artifact only binds if the backend that consumes it is diff --git a/e2e_workflow/roles/kernel_extractor.md b/e2e_workflow/roles/kernel_extractor.md index 9464eaf2f..7f16edf4b 100644 --- a/e2e_workflow/roles/kernel_extractor.md +++ b/e2e_workflow/roles/kernel_extractor.md @@ -14,17 +14,63 @@ You are invoked once per kernel candidate. Read first: ## The task-dir contract you must emit (what the kernel layer expects) ``` /kernels/_task/ - kernel_src/... # editable copy of the kernel source (the sglang/aiter subtree that owns it) — OVERWRITTEN by optimize/author - baseline_src/... # IMMUTABLE frozen copy of the REAL ONLINE kernel — the timing-baseline denominator; sha-checked + kernel_src/... # the EDITABLE optimization workspace — the ONLY path optimize/author may write + baseline_overlay/ # frozen snapshot of CURRENT_OVERLAY = the REAL ONLINE SERVING STACK (the + # install + every already-accepted kernel). This IS the timing denominator. + # It is never edited and never imported FROM the task dir — it goes on + # PYTHONPATH and that is the whole mechanism. + baseline_ref/*.orig # read-only TEXT copy of the baseline file(s) so the optimizer can diff. The + # .orig suffix makes it UNIMPORTABLE, so it can never become a timing leg. + cases.py # task-specific: call(args) / timing_cases / random_shapes / eager_cases. + # Run VERBATIM by both legs — this is why the two legs cannot diverge; IMMUTABLE reference_io.pt # recorded inputs + golden outputs (oracle) — READ-ONLY for optimizers - harness_lib.py # VENDORED copy of scripts/harness_lib.py — the SHARED timing/correctness lib; IMMUTABLE - unittest.py # builds(opt)/runs/checks-correctness vs oracle + random-value parity vs baseline_src/times speedup; IMMUTABLE - meta.json # name, source path in sglang, target callable, baseline_callable (real online kernel), shapes, dtypes, backend, regime, build, random_draws (default 3), checksum + harness_lib.py # VENDORED scripts/harness_lib.py — the SHARED timing/correctness lib; IMMUTABLE + leg_runner.py # VENDORED scripts/leg_runner.py — runs ONE leg under the ambient overlay; IMMUTABLE + overlay_setup.py # VENDORED scripts/overlay_setup.py — builds the candidate overlay; IMMUTABLE + unittest.py # driver: h.measure_legs + h.run_correctness, prints the metric; IMMUTABLE + meta.json # name, source path, target_callable, candidate_bind, shapes, dtypes, backend, + # regime, served_regimes, build, random_draws (default 3), checksum +``` +**Vendor the three shared scripts into the task dir** +(`for f in harness_lib.py leg_runner.py overlay_setup.py; do cp "$SKILL_DIR/scripts/$f" "$TASK/"; done`). +`unittest.py` imports `harness_lib` for ALL timing + correctness — never hand-roll a timing loop or an +allclose check. This is what makes every task measure the same way; it also keeps the task +self-contained + immutable (the validator sha-checks them alongside `reference_io.pt`). + +### 🔴 THE TWO LEGS ARE THE SAME CODE UNDER TWO PYTHONPATHS — read this before writing anything +There is no `baseline_callable`, and no second copy of the source to time against. Both legs run the +SAME `leg_runner.py` + the SAME `cases.py`; the only difference is the overlay on PYTHONPATH: + +| leg | PYTHONPATH | what it is | +|---|---|---| +| baseline | `/baseline_overlay` | the live serving stack — identical to the e2e gate's ref leg | +| candidate | `/_cand_overlay` | that same stack **+ exactly one entry built from `kernel_src/`** | + +`_cand_overlay` is rebuilt from `meta.candidate_bind` on every measurement by +`h.build_candidate_overlay` — you do NOT build it, and `unittest.py` does not either. Consequences you +must not fight: +* **Direction is a property of the environment, not of a name you choose.** `speedup = + baseline_ms / optimized_ms` cannot invert, and `mode=author` cannot time optimized-HIP against its own + naive-HIP seed, because the author's seed only ever lands in `kernel_src/` (the candidate side). +* **`h.assert_legs_differ` refuses to measure** unless the two legs provably resolve `target_callable` + to different code AND the baseline resolves OUTSIDE the task dir. A `no_rebind_seam` candidate is + therefore caught at smoke, not after authoring. +* **The denominator is the same one the e2e gate uses**, so isolated speedup and `e2e_delta_pct` are + measured on the same basis and the Amdahl cross-check is meaningful. + +**`cases.py` — the one file you author, and the only thing the two legs share besides the runner.** +It must import NOTHING from the task dir except `harness_lib` (passed in as `h`), and must reach the op +ONLY through `meta.target_callable` — never through a path, and never by importing `kernel_src/` +directly (that would make the baseline leg run the candidate): +```python +def call(args): # args -> FRESH out tensor. THE seam both legs go through. + fn = _resolve(META["target_callable"]) # importlib on the dotted name; the overlay decides which + return fn(**args) # code that name lands on +def timing_cases(h, meta): # [{sig, regime, m, args}] — sig is the bucket key, stable across + ... # processes; one entry per meta.workload.cases[] +def random_shapes(h, meta): # [{sig, make_inputs(rng)}] — FRESH in-regime draws at FIXED dims +def eager_cases(h, meta): # [{args, ref}] from reference_io.pt (the golden oracle) ``` -**Vendor the shared harness library into the task dir** (`cp "$SKILL_DIR/scripts/harness_lib.py" -"$TASK/harness_lib.py"`). `unittest.py` imports it for ALL timing + correctness — never hand-roll a -timing loop or an allclose check. This is what makes every task measure the same way; it also keeps the -task self-contained + immutable (the validator sha-checks `harness_lib.py` alongside `reference_io.pt`). --- @@ -33,7 +79,8 @@ task self-contained + immutable (the validator sha-checks `harness_lib.py` along Inputs: `EVAL_DIR`, `MODEL_PATH`, `GPU_ID`, `WORKLOAD`, `KERNEL` (the Architect's candidate: short_name, classification, extract_hint = the `module:attr` callable to hook, candidate_backends, regime, and — when an upstream TraceLens prior was available — OPTIONAL `source_hint` (resolved source -file), `launcher_hint` (launcher seam), `bound_type`), `CURRENT_FLAGS`/`CURRENT_ENV`, `SKILL_DIR`. +file), `launcher_hint` (launcher seam), `bound_type`), `CURRENT_OVERLAY` (the accepted-kernel stack +carried forward — may be empty on the first milestone), `CURRENT_FLAGS`/`CURRENT_ENV`, `SKILL_DIR`. ### Resolve + HONOR the ONLINE REGIME first (same contract as PHASE=extract_op) The #1 cause of "isolated win, e2e loss/crash" is a unittest that SYNTHESIZES its inputs with OFFLINE @@ -99,18 +146,39 @@ freeze an out-of-regime oracle nobody should trust. capture overlay, driven by the SAME workload as the profile so shapes match the regime: ```bash TASK="$EVAL_DIR/kernels/_task"; mkdir -p "$TASK" - # write a tiny capture overlay sitecustomize that calls capture_shapes.install(...) - python3 "$SKILL_DIR/scripts/overlay_setup.py" monkeypatch \ - --overlay "$TASK/_capture_overlay" \ - --target "" --impl-module capture_shapes --impl-attr _wrapper \ - --impl-file "$SKILL_DIR/scripts/capture_shapes.py" 2>/dev/null || true - # simpler/robust: drive via env so capture_shapes self-installs on import + # FREEZE the live serving stack as this task's baseline env, then hang the capture hook off a COPY + # of it. --from is what stacks them: two overlay dirs on PYTHONPATH do NOT compound (only the first + # sitecustomize is imported), so capturing on a bare hook overlay would silently capture the + # PRISTINE install instead of the server the accepted kernels actually built. + python3 "$SKILL_DIR/scripts/overlay_setup.py" add-capture \ + --overlay "$TASK/_capture_overlay" --from "$CURRENT_OVERLAY" \ + --target "" --out "$TASK" --max 5 \ + --capture-file "$SKILL_DIR/scripts/capture_shapes.py" + cp -r "$CURRENT_OVERLAY"/. "$TASK/baseline_overlay"/ 2>/dev/null || \ + python3 -c "import sys;sys.path.insert(0,'$SKILL_DIR/scripts');import overlay_setup as o;o._ensure_overlay('$TASK/baseline_overlay')" + BACKEND="" OUT_DIR="$TASK/_capture" GPU="$GPU_ID" MODEL="$MODEL_PATH" \ - ISL= OSL= CONC= REPEATS=0 PROFILE=0 \ - OVERLAY_PYTHONPATH="$SKILL_DIR/scripts" \ + ISL= OSL= CONC= REPEATS=0 PROFILE=0 \ + OVERLAY_PYTHONPATH="$TASK/_capture_overlay" \ EXTRA_ENV="CAPTURE_TARGET= CAPTURE_OUT=$TASK CAPTURE_MAX=5" \ bash "$EVAL_DIR/bench_e2e.sh" 2>&1 | tee "$EVAL_DIR/logs/capture_.log" ``` + 🔴 **Capture on the CURRENT stack, not the install.** The oracle you freeze is the truth source the + candidate is judged against, and the baseline you time against is `baseline_overlay/`. Both must be + the server as it runs RIGHT NOW (config + every accepted kernel). Capturing on the pristine install + while the e2e gate runs on the stack is what made isolated and e2e numbers incomparable. + (An empty `CURRENT_OVERLAY` — the first milestone — is fine: `baseline_overlay/` is then a valid + empty overlay and the baseline leg resolves straight to the install, which IS the live stack.) + **Cold start needs no knob.** `bench_e2e.sh` waits on *progress* (log growth), not on a fixed + budget, so a slow TP4 checkpoint load + AITER compile is allowed to take as long as it keeps + moving; only `STALL_WINDOW_SEC` (default 600s) of total silence ends the wait. If it does fail, + read `$OUT_DIR/server_start.json` — `reason` distinguishes `stalled` / `oom` / `died_early` / + `ceiling_exceeded` (the last one names `SERVER_STARTUP_TIMEOUT_SEC` as the override). + + 🔴 **ISL/OSL/CONC MUST be the deployment `WORKLOAD` values** — shrinking OSL to speed the capture up + freezes a decode regime the deployment never runs, so every downstream speedup is measured on the + wrong shapes. Shorten the window with `REPEATS`/`CAPTURE_MAX` instead. + (REPEATS=0 → just warmup drives a short window; capture flushes incrementally + on server exit.) Verify `reference_io.pt` + `meta.json` exist and `num_cases` ≥ 1. For a head GEMM that serves both regimes you MUST capture/synthesize BOTH a decode case (M ≈ `WORKLOAD.conc`) and a prefill case (large M) — @@ -142,20 +210,33 @@ freeze an out-of-regime oracle nobody should trust. > `trace` > `regime` > prior > `--min-regime-share` floor), because the short / graph-hidden window > under-counts decode — it is NOT the weight authority. `meta.shape_counts` is a COUNT, not a time — a > prefill call is 1 count but huge GPU-time, decode is thousands of tiny calls; never weight by raw count. -3. **Copy the editable source** into `kernel_src/` (the minimal owning subtree), so the kernel layer - and the later overlay can diff against it. - > **🔴 FREEZE THE REAL ONLINE KERNEL AS THE IMMUTABLE TIMING BASELINE — this is what stops a - > cross-language rewrite from timing against a fake baseline.** In addition to `kernel_src/` (which the - > optimizer/author OVERWRITES — e.g. rewrites the Triton kernel as HIP/CK), snapshot the ORIGINAL live - > kernel into a SEPARATE, IMMUTABLE `baseline_src/` (copy the same subtree) AND record its callable in - > `meta.json` as `baseline_callable` (`module:attr` of the real online kernel — for minimax that is the - > Triton `_gqa_sparse_fwd_kernel` via `target_callable`). The unittest's baseline leg is bound to THIS - > frozen callable, NEVER to whatever is currently in `kernel_src/`. Reason: `mode=author` starts - > `kernel_src/` from a naive from-scratch impl in the target language; if the baseline followed - > `kernel_src/`, the reported speedup would be "optimized-HIP vs my-own-naive-HIP" (observed 15.7×) — a - > fake win against a strawman the author itself wrote, not against the production Triton kernel that - > actually serves the workload. Freezing the real online kernel makes the speedup denominator - > **language-independent and always the live path**. sha-check `baseline_src/` alongside `reference_io.pt`. +3. **Seed the editable workspace + declare how the candidate binds.** Resolve the target's file **as the + CURRENT STACK sees it** (not via a bare `import` of the install), copy that ONE file into + `kernel_src/`, keep a `.orig` twin for diffing, and record the bind: + ```bash + MOD="$(python3 -c 'print("".split(":")[0])')" + SRC="$(PYTHONPATH="$TASK/baseline_overlay:$PYTHONPATH" \ + python3 "$TASK/overlay_setup.py" check --module "$MOD" --path-only)" + mkdir -p "$TASK/kernel_src" "$TASK/baseline_ref" + cp "$SRC" "$TASK/kernel_src/$(basename "$SRC")" + cp "$SRC" "$TASK/baseline_ref/$(basename "$SRC").orig" + ``` + Then set `meta.candidate_bind` — the ONE declaration that says how `kernel_src/` becomes a candidate: + ```jsonc + // optimize mode: the candidate IS the patched module file (whole-file swap) + {"kind": "module", "module": "", "file": "kernel_src/.py"} + // author mode: the candidate is a NEW impl (any language, own file name) rebound onto the live seam + {"kind": "rebind", "target": "", "impl_module": "", + "impl_attr": "", "file": "kernel_src/.py"} + ``` + > **🔴 COPY THE FILE, NEVER THE PACKAGE SUBTREE.** A regular package (with `__init__.py`) on an + > earlier PYTHONPATH entry FULLY shadows the install — every sibling submodule vanishes and + > `import sglang` breaks. Single file + `add-module` is the only mechanism (see + > `knowledge/sglang_internals.md` §3). + > **🔴 `kernel_src/` is the ONLY writable path in the task dir.** `baseline_overlay/` is the live + > serving stack and is never touched; `baseline_ref/*.orig` is text for diffing and is deliberately + > unimportable. `candidate_bind` is also exactly what the e2e Integrator replays to wire an accepted + > kernel, so an isolated win is rebindable e2e by construction. 4. **Write `unittest.py`** — backend-agnostic and IMMUTABLE. It is the SINGLE harness: it judges correctness AND measures the workload-weighted speedup; there is no separate downstream perf harness. **It MUST import the vendored `harness_lib` and use it for all timing + correctness** — do NOT @@ -177,23 +258,27 @@ freeze an out-of-regime oracle nobody should trust. Build a `cases` list of `{"args": , "ref": , "sig":