From aa029e9d608e0abc76bf2239e56ee40fd9b69c42 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Aug 2026 06:28:38 +0000 Subject: [PATCH 1/8] deployability gates for authored kernels --- e2e_workflow/e2e_workflow.js | 710 +++++++++++++++++- e2e_workflow/roles/e2e_integrator.md | 34 +- e2e_workflow/roles/kernel_extractor.md | 78 +- e2e_workflow/roles/op_benchmarker.md | 29 + e2e_workflow/roles/profiler.md | 30 +- e2e_workflow/roles/system_architect.md | 16 + .../scripts/check_schema_consumption.py | 204 +++++ e2e_workflow/scripts/op_bench.py | 371 ++++++++- e2e_workflow/scripts/parse_profile.py | 161 +++- e2e_workflow/scripts/seam_contract.py | 516 +++++++++++++ 10 files changed, 2084 insertions(+), 65 deletions(-) create mode 100644 e2e_workflow/scripts/check_schema_consumption.py create mode 100644 e2e_workflow/scripts/seam_contract.py diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index 70ba8746f..ea764b468 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -178,6 +178,21 @@ const HEAD_CORRECTIVE_MAX = parseInt(A.head_corrective_max != null ? A.head_corr // the heavy re-author only when the surgical patch fails. surgical_fix=false => old heavy-only behavior. const SURGICAL_FIX = String(A.surgical_fix != null ? A.surgical_fix : 'true') === 'true'; const FIXABLE_REJECT_RX = /cuda_graph_capture_unsafe|no[_ ]?binary|NO_BINARY_FOR_GPU|hipErrorNoBinaryForGpu|capture[_ ]?(unsafe|hang)|host[_ ]?sync|graph[_ ]?capture|no[_ ]?rebind[_ ]?seam|no[_ ]?engagement|not[_ ]?engaged|signature[_ ]?mismatch|wrong[_ ]?seam/i; +// ---- DEGRADATION LEDGER (INV-3: no silent degradation) ---------------------------------------------- +// Every place the orchestrator accepts a WEAKER input than the contract asks for — a missing machine +// verdict, a fallback field, an unrecognized code — records here instead of quietly carrying on. The +// ledger is emitted with the run report, so "we ran with 3 contracts unenforced" is visible in the +// artifact rather than inferable only by reading the source. GENERIC: no op/kernel/model specifics. +const DEGRADATIONS = []; +// Heads rejected by an admission gate that runs BEFORE `flaggedHeads` exists (Strategize-time entity-kind +// admission). Merged into flaggedHeads so they are surfaced by the same "never silently skipped" path. +const PRE_FLAGGED_HEADS = []; +function noteDegradation(where, what, detail) { + const d = { where, what, detail: detail || '' }; + DEGRADATIONS.push(d); + log(` [degraded] ${where}: ${what}${detail ? ` — ${detail}` : ''}`); + return d; +} // ---- CORRECTNESS-class reject (auto-correct) -------------------------------------------------------- // A SECOND fix-and-retryable class: the candidate ENGAGED and beat the isolated oracle but produces the // WRONG output on the LIVE path (parity/accuracy failure) — OR posts an IMPLAUSIBLE e2e speedup (faster @@ -219,18 +234,91 @@ function isImplausibleSpeedup(pct_gpu_time, isolated, integ) { if (!Number.isFinite(ceilPct)) return false; return ((integ && integ.e2e_delta_pct) || 0) > ceilPct * (1 + IMPLAUSIBLE_SPEEDUP_MARGIN) + 1e-9; } -// Classify a reject reason into a fix-and-retry class ('' = terminal, not auto-correctable). -function rejectClass(reason) { - const r = reason || ''; - if (CORRECTNESS_REJECT_RX.test(r)) return 'correctness'; - if (FIXABLE_REJECT_RX.test(r)) return 'integration'; +// ---- STRUCTURED REJECT CODES (INV-4) + OWNER-STAGE ROUTING (INV-5) --------------------------------- +// A reject carries a CODE from a closed set, not a sentence. Two things are read off that code: +// cls — which corrective instruction applies ('' = terminal, not auto-correctable); +// stage — WHICH PIPELINE STAGE OWNS THE DEFECT, i.e. where a fix has to re-enter. +// The stage column is the part that was missing. Every corrective used to re-enter at `author` +// (kernel_workflow mode:'optimize' on the SAME task dir), but a task dir's unittest.py is immutable by +// contract — so any defect whose fix requires a different call contract or a different seam is +// literally unfixable there, and re-authoring burns hours to arrive back at the same reject. Routing by +// owner stage means an extract-owned defect re-extracts and an author-owned defect re-authors. +// GENERIC — the table is keyed on failure MODE, never on op kind, backend, model or kernel. +const REJECT_CODES = { + // --- owned by EXTRACT: the task itself encodes the wrong seam / wrong contract / wrong denominator + no_rebind_seam: { cls: 'integration', stage: 'extract' }, + signature_mismatch: { cls: 'integration', stage: 'extract' }, + arity_mismatch: { cls: 'integration', stage: 'extract' }, + param_name_mismatch: { cls: 'integration', stage: 'extract' }, + return_contract_mismatch: { cls: 'integration', stage: 'extract' }, + hidden_context_inputs: { cls: 'integration', stage: 'extract' }, + candidate_unresolvable: { cls: 'integration', stage: 'extract' }, + no_seam_descriptor: { cls: 'integration', stage: 'extract' }, + no_engagement: { cls: 'integration', stage: 'extract' }, + wrong_seam: { cls: 'integration', stage: 'extract' }, + invalid_denominator: { cls: 'integration', stage: 'extract' }, + // --- owned by AUTHOR: the kernel is right about the seam, wrong about posture or numerics + cuda_graph_capture_unsafe: { cls: 'integration', stage: 'author' }, + no_binary_for_gpu: { cls: 'integration', stage: 'author' }, + capture_hang: { cls: 'integration', stage: 'author' }, + host_sync_in_hot_path: { cls: 'integration', stage: 'author' }, + oom: { cls: 'integration', stage: 'author' }, + parity_regression: { cls: 'correctness', stage: 'author' }, + accuracy_regression: { cls: 'correctness', stage: 'author' }, + output_corruption: { cls: 'correctness', stage: 'author' }, + implausible_speedup: { cls: 'correctness', stage: 'author' }, + // --- owned UPSTREAM of the kernel track: no amount of kernel work fixes these + wrong_head_granularity: { cls: '', stage: 'profile' }, + delegated_track_disabled: { cls: '', stage: 'strategize' }, + // --- terminal by construction: a correct kernel with no headroom is not a defect + no_win: { cls: '', stage: '' }, + do_no_harm: { cls: '', stage: '' }, +}; +const normalizeRejectCode = (c) => String(c || '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, ''); +// Last-resort recovery of a code from free prose, used ONLY when the integrator returned no +// reason_code at all (older role revision). Scans for an explicit code token FIRST — this is the fix +// for the precedence bug where CORRECTNESS_REJECT_RX's bare `mismatch` swallowed `signature_mismatch` +// and made that FIXABLE_REJECT_RX branch permanently unreachable. +function rejectCodeFromProse(reason) { + const r = String(reason || ''); + for (const code of Object.keys(REJECT_CODES)) { + if (new RegExp(`(^|[^a-z0-9])${code.replace(/_/g, '[_ ]?')}([^a-z0-9]|$)`, 'i').test(r)) return code; + } return ''; } +// Resolve {cls, stage, code, provenance} for a reject. `code` is the integrator's structured +// reason_code when present. Fails closed on an UNKNOWN structured code (we cannot route what we cannot +// name) and records the degradation when it has to fall back to prose. +function rejectVerdict(reason, code) { + const c = normalizeRejectCode(code); + if (c) { + if (REJECT_CODES[c]) return { ...REJECT_CODES[c], code: c, provenance: 'structured' }; + noteDegradation('rejectVerdict', `unknown reason_code '${c}' — not in REJECT_CODES`, + 'treating as terminal; add the code to the table and to INTEGRATE_SCHEMA.reason_code'); + return { cls: '', stage: '', code: c, provenance: 'unknown_code' }; + } + const fromProse = rejectCodeFromProse(reason); + if (fromProse) { + noteDegradation('rejectVerdict', 'integrator returned no reason_code', + `recovered '${fromProse}' from the prose reason`); + return { ...REJECT_CODES[fromProse], code: fromProse, provenance: 'prose_code' }; + } + const r = reason || ''; + const cls = CORRECTNESS_REJECT_RX.test(r) ? 'correctness' : (FIXABLE_REJECT_RX.test(r) ? 'integration' : ''); + if (r) noteDegradation('rejectVerdict', 'no reason_code and no recognizable code token in the prose', + `regex fallback -> class='${cls || 'terminal'}', owner stage UNKNOWN (corrective will re-enter at author)`); + return { cls, stage: cls ? 'author' : '', code: '', provenance: 'prose_regex' }; +} +// Classify a reject reason into a fix-and-retry class ('' = terminal, not auto-correctable). +function rejectClass(reason, code) { return rejectVerdict(reason, code).cls; } +// Which pipeline stage owns this defect — i.e. where a corrective must re-enter to have any chance. +function rejectStage(reason, code) { return rejectVerdict(reason, code).stage; } // A gate 'accept'/'stack' only counts as a REAL win if the measured e2e delta is not an implausible // (corruption) speedup. Centralizes the guard so every integrate site treats a too-good-to-be-true // delta as a reject instead of banking it. GENERIC (uses only pct_gpu_time + isolated speedup). function integAccepted(integ, pct_gpu_time, isolated) { return !!(integ && (integ.gate === 'accepted' || integ.gate === 'stack') + && provenanceOk(integ, 'integrate') // INV-3: consume the integrator's own provenance claim && !isImplausibleSpeedup(pct_gpu_time, isolated, integ)); } // The reason string to feed the corrective loop: if the gate "passed" but the delta is impossible, emit @@ -470,13 +558,28 @@ const EXTRACT_OP_SCHEMA = obj({ target_callable: { type: 'string' }, // module:attr rebind seam for an authored kernel ('' if none) baseline_callable: { type: 'string' }, // module:attr of the FROZEN real online kernel (the speedup denominator) baseline_frozen: { type: 'boolean' }, // true only when baseline_src/ was frozen OR baseline_callable resolves + // MACHINE VERDICTS from scripts/seam_contract.py — pasted verbatim, never hand-written. These are the + // fields hasFrozenBaseline()/seamBindable() actually consume; without them the head fails closed. + baseline_validation: { type: 'object', additionalProperties: true }, // --mode baseline verdict (INV-1) + binding_descriptor: { type: 'object', additionalProperties: true }, // --mode binding descriptor (INV-2) + binding_check: { type: 'object', additionalProperties: true }, // entry-vs-seam bindability (INV-2) + entry_contract_path: { type: 'string' }, // --mode entry generated contract smoke: { type: 'string' }, notes: { type: 'string' }, }, ['op_kind', 'task_dir', 'smoke']); const OPBENCH_SCHEMA = obj({ short_name: { type: 'string' }, op_kind: { type: 'string' }, provenance_ok: { type: 'boolean' }, winner_backend: { type: 'string' }, winner_kind: { type: 'string' }, - isolated_speedup: { type: 'number' }, winner_editable: { type: 'boolean' }, + // null, NOT a number, when op_bench.py withheld the ratio: the denominator was not the live path, so + // there is no speedup to report. A withheld measurement must not be representable as `1.0` or as the + // unpublishable figure — both read as claims. + isolated_speedup: { type: ['number', 'null'] }, winner_editable: { type: 'boolean' }, + // INV-1. What the candidate was divided BY, and whether op_bench.py refused to publish the ratio. + // Copied from opbench_result.json; the gate below will not bank a win on an unsound denominator. + denominator: { type: 'string', enum: [ + 'measured_backend_default', 'verified_baseline', 'unverified_baseline', + 'target_fallback', 'synthesized_reference', 'none'] }, + speedup_withheld: { type: 'boolean' }, best_known_ms: { type: 'number' }, recommend_tier_c: { type: 'boolean' }, author_plan: arrObj, tuning_artifact: { type: 'string' }, apply_env: { type: 'string' }, apply_flags: { type: 'string' }, code_patch: { type: 'string' }, @@ -490,8 +593,14 @@ const EXTRACT_SCHEMA = obj({ source_path_in_sglang: { type: 'string' }, target_callable: { type: 'string' }, num_cases: { type: 'number' }, regimes_captured: arrStr, candidate_backends: arrStr, build: { type: 'boolean' }, unittest_smoke: { type: 'string' }, + synthesized: { type: 'boolean' }, // true when the oracle was fabricated -> can never be the denominator baseline_callable: { type: 'string' }, // module:attr of the FROZEN real online kernel (the speedup denominator) baseline_frozen: { type: 'boolean' }, // true only when baseline_src/ was frozen OR baseline_callable resolves + // Same machine verdicts as EXTRACT_OP_SCHEMA — see the note there. + baseline_validation: { type: 'object', additionalProperties: true }, + binding_descriptor: { type: 'object', additionalProperties: true }, + binding_check: { type: 'object', additionalProperties: true }, + entry_contract_path: { type: 'string' }, reference_io_sha256: { type: 'string' }, notes: { type: 'string' }, }, ['editable', 'task_dir', 'unittest_smoke']); @@ -517,6 +626,17 @@ const INTEGRATE_SCHEMA = obj({ // implausible-speedup guard only distrusts an 'accuracy'/soft accept; a byte_exact accept is trusted. parity_kind: { type: 'string' }, gate: { type: 'string', enum: ['accepted', 'stack', 'rejected', 'incomplete'] }, + // STRUCTURED reject code (INV-4). REQUIRED whenever gate='rejected'. The orchestrator routes the + // corrective off THIS, not off the prose in `reason` — classifying a reject by regex over a sentence + // an LLM wrote is not a decision procedure (it made the `signature_mismatch` branch unreachable, + // because the bare `mismatch` token in the correctness regex matched first). `reason` stays as the + // human-readable detail. The enum is the closed set in REJECT_CODES. + reason_code: { type: 'string', enum: [ + 'no_rebind_seam', 'signature_mismatch', 'arity_mismatch', 'param_name_mismatch', + 'return_contract_mismatch', 'hidden_context_inputs', 'no_engagement', 'wrong_seam', + 'invalid_denominator', 'cuda_graph_capture_unsafe', 'no_binary_for_gpu', 'capture_hang', + 'host_sync_in_hot_path', 'oom', 'parity_regression', 'accuracy_regression', 'output_corruption', + 'implausible_speedup', 'wrong_head_granularity', 'delegated_track_disabled', 'no_win', 'do_no_harm'] }, accepted_overlay: { type: 'string' }, reason: { type: 'string' }, }, ['gate', 'e2e_throughput_tok_s']); @@ -707,11 +827,255 @@ async function ensureFlydslGate() { } } -// A FROZEN baseline is resolvable when the extractor either froze baseline_src/ (baseline_frozen) -// OR set an importable meta.baseline_callable. That is the language-independent speedup denominator. -const hasFrozenBaseline = (ext) => - !!(ext && (ext.baseline_frozen === true || - (typeof ext.baseline_callable === 'string' && ext.baseline_callable.trim() !== ''))); +// ---- INV-1: DENOMINATOR IDENTITY ------------------------------------------------------------------ +// A frozen baseline is the speedup DENOMINATOR, so the only question that matters is whether the thing +// it names is the code the live server actually runs. That is not decidable from the string: a +// task-local `baseline_src.xxx_ref:forward` scaffold the extractor wrote itself is a perfectly +// well-formed non-empty string, and against it any authored kernel posts a large, meaningless win that +// cannot carry end-to-end. So the predicate now consumes the MACHINE VERDICT from the vendored +// validator (scripts/seam_contract.py --mode baseline), which resolves the callable and checks that it +// (a) imports from outside the task/eval dir, (b) lives in an installed distribution, (c) is the seam +// itself or an OBSERVED callee of it, and (d) was not flagged `synthesized`. Op-kind agnostic — the +// validator never asks what kind of op this is. +// Strict by default: no machine verdict => not a frozen baseline (fail closed). baseline_contract_strict +// =false restores the old string check for a legacy role revision, and records the degradation. +const BASELINE_CONTRACT_STRICT = String(A.baseline_contract_strict != null ? A.baseline_contract_strict : 'true') === 'true'; +// An oracle with zero recorded cases, or one whose bytes nobody hashed, cannot certify anything the +// task later claims. Both facts are already REQUESTED in EXTRACT_SCHEMA (num_cases, +// reference_io_sha256) and were, until now, read by nothing — see scripts/check_schema_consumption.py. +function oracleProvenance(ext) { + const problems = []; + if (ext.num_cases != null && Number(ext.num_cases) <= 0) + problems.push('num_cases=0 (the oracle recorded no calls, so correctness is unfalsifiable)'); + if (typeof ext.reference_io_sha256 === 'string' && ext.reference_io_sha256.trim() === '' + && ext.synthesized !== true) + problems.push('reference_io_sha256 empty (the oracle bytes are unpinned; tampering is undetectable)'); + return { ok: problems.length === 0, problems }; +} + +// `provenance_ok` is the agent's own assertion that it re-hashed the oracle / confirmed the baseline +// before reporting a number. It is REQUIRED by both OPBENCH_SCHEMA and INTEGRATE_SCHEMA and was read +// nowhere: an agent could truthfully report provenance_ok=false and still have its speedup banked. +// An explicit false is now disqualifying; an absent value is a recorded degradation, not a pass. +function provenanceOk(res, where) { + if (!res || typeof res !== 'object') return true; + if (res.provenance_ok === false) { + log(` ⚠️ ${where}: provenance_ok=false — the agent states its own numbers are unsourced; not banking them`); + return false; + } + if (res.provenance_ok == null) + noteDegradation(where, 'result carries no provenance_ok', + 'cannot tell whether the oracle/baseline was re-verified before the number was produced'); + return true; +} + +// INV-1 at the ORCHESTRATOR boundary. op_bench.py already withholds a ratio it cannot stand behind, but +// the orchestrator must not depend on the script having been the thing that produced this JSON — an +// agent can hand-assemble the object, and the 4.47x-with-zero-e2e case is precisely a number that was +// divided by something the server never calls. So re-check the declared denominator here and fail +// closed. GENERIC: keyed on the provenance of the measurement, never on op kind or kernel name. +const BANKABLE_DENOMINATORS = ['measured_backend_default', 'verified_baseline']; +function denominatorSound(bake, where) { + if (!bake || typeof bake !== 'object') return false; + if (bake.speedup_withheld === true) { + log(` ⚠️ ${where}: op_bench WITHHELD the speedup (denominator=${bake.denominator || 'unknown'}) — not banking a win`); + return false; + } + const d = bake.denominator; + if (d == null) { + noteDegradation(where, 'bake-off result declares no denominator', + 'cannot tell whether the speedup was measured against the live path or against a fabricated reference'); + return true; // older role revision: degrade loudly, do not silently drop a real win + } + if (!BANKABLE_DENOMINATORS.includes(d)) { + log(` ⚠️ ${where}: denominator='${d}' is not the live path — the ratio is unbankable`); + return false; + } + return true; +} + +// A denominator fails in two DIFFERENT ways and they must not share a consequence: +// 'missing' -- nothing to divide by at all (no baseline_callable, or an oracle with zero cases / +// unpinned bytes). Neither the ratio nor the correctness verdict means anything. +// 'synthesized' -- the reference was written by the extractor. The RATIO is worthless; the KERNEL is +// not. Whether it is actually faster is settled downstream by the live e2e A/B, +// which divides by the real server and by nothing the extractor authored. +// Replaying the 88 archived runs settles which consequence belongs to which: 113 of 210 real +// extractions declared synthesized:true, and TEN entries those runs went on to ACCEPT on a MEASURED +// e2e A/B came from one (+23.3%, +12.6%, +6.5%, +5.0%, ... see geak_inv_verify/corpus_gates.py). +// Killing the extraction on 'synthesized' would have thrown all ten away. Withholding the ratio costs +// nothing, because the ratio was never what those wins were banked on. +function baselineDefect(ext) { + if (!ext) return 'missing'; + if (!oracleProvenance(ext).ok) return 'missing'; + const v = ext.baseline_validation; + if (v && typeof v === 'object' && v.contract === 'baseline_identity') + return v.ok === true ? 'none' : 'missing'; + if (ext.synthesized === true) return 'synthesized'; + return 'none'; +} + +const hasFrozenBaseline = (ext) => { + if (!ext) return false; + const op = oracleProvenance(ext); + if (!op.ok) { + log(` [inv-1] oracle provenance FAILED: ${op.problems.join('; ')}`); + return false; + } + const v = ext.baseline_validation; + if (v && typeof v === 'object' && v.contract === 'baseline_identity') { + if (v.ok === true) return true; + log(` [inv-1] baseline_validation FAILED: ${(v.failed || []).join(', ') || 'unknown'} ` + + `(baseline_callable=${v.baseline_callable || "''"}, origin=${(v.baseline_origin || {}).kind || '?'})`); + return false; + } + // No machine verdict at all. + if (ext.synthesized === true) { // INV-3: a declared field that is now CONSUMED + noteDegradation('hasFrozenBaseline', 'extraction reports synthesized:true and no baseline_validation', + 'a fabricated oracle can never be the denominator — rejecting the extraction'); + return false; + } + const legacyOk = !!(ext.baseline_frozen === true || + (typeof ext.baseline_callable === 'string' && ext.baseline_callable.trim() !== '')); + noteDegradation('hasFrozenBaseline', 'extraction returned no baseline_validation (seam_contract.py not run)', + BASELINE_CONTRACT_STRICT + ? 'baseline_contract_strict=true -> treating as NO frozen baseline (fail closed)' + : `baseline_contract_strict=false -> falling back to the string check (=${legacyOk}); the ` + + 'denominator is UNVERIFIED and any isolated speedup from this task is unaudited'); + return BASELINE_CONTRACT_STRICT ? false : legacyOk; +}; + +// ---- INV-2: BINDING CONTRACT ---------------------------------------------------------------------- +// Whether an authored replacement can be bound at the live seam is decided by inspect.signature of the +// LIVE callable, not by an agent's recollection of it. seam_contract.py --mode binding emits the +// descriptor and (given the entry) the bindability verdict; the extractor returns both. A head whose +// seam is not bindable must never enter the author fan-out: the kernel that comes out is unusable no +// matter how fast it is, and the reject only surfaces hours later at the e2e gate. +// Strict by default; seam_contract_strict=false degrades to "unverified but allowed" and records it. +const SEAM_CONTRACT_STRICT = String(A.seam_contract_strict != null ? A.seam_contract_strict : 'true') === 'true'; +function seamBindable(ext) { + if (!ext) return { ok: false, why: 'no extraction' }; + const chk = ext.binding_check, desc = ext.binding_descriptor; + if (chk && typeof chk === 'object' && chk.contract === 'binding_check') { + return chk.bindable === true + ? { ok: true, why: 'binding_check.bindable' } + : { ok: false, why: `binding_check: ${(chk.codes || []).join(', ') || 'not bindable'}`, + codes: chk.codes || [] }; + } + if (desc && typeof desc === 'object' && desc.contract === 'binding') { + if (desc.ok !== true) return { ok: false, why: `binding_descriptor unusable: ${desc.error || 'unknown'}` }; + if ((desc.hidden_context || []).length) + return { ok: false, why: `seam reads non-parameter inputs ${JSON.stringify(desc.hidden_context)}`, + codes: ['hidden_context_inputs'] }; + // entry_contract_path names the stub seam_contract.py --mode entry GENERATED from the live + // signature. Its presence is the difference between an entry derived from the seam and one the + // author invented, so it upgrades a descriptor-only verdict from "unchecked" to "checked by + // construction". (Also a declared schema field that nothing read before.) + if (typeof ext.entry_contract_path === 'string' && ext.entry_contract_path.trim() !== '') + return { ok: true, why: `seam described; entry generated from it (${ext.entry_contract_path})` }; + return { ok: true, why: 'seam described; no entry to check yet' }; + } + noteDegradation('seamBindable', 'extraction returned no binding_descriptor (seam_contract.py not run)', + SEAM_CONTRACT_STRICT + ? 'seam_contract_strict=true -> head is NOT admitted to the author fan-out (fail closed)' + : 'seam_contract_strict=false -> authoring an UNVERIFIED seam; a contract mismatch will only ' + + 'surface at the e2e gate, after the budget is spent'); + return { ok: !SEAM_CONTRACT_STRICT, why: 'no binding contract (unverified)' }; +} + +// ---- INV-2/INV-3: AUTHORING ADMISSION ------------------------------------------------------------- +// The ONLY precondition on spending the author budget used to be `isolated_speedup > 1.0` — a number +// produced INSIDE the task dir, which says nothing about whether the result can ever reach the live +// path. A head can clear it while having no seam to bind to at all; the kernel is then authored, +// optimized for hours, and rejected at the e2e gate for a reason that was decidable in seconds. +// This gate answers "if this kernel turns out to be fast, can we actually deploy it?" BEFORE the spend. +// Three questions, none of them op-kind specific: +// 1. is there a seam to rebind at? (target_callable / live_call_seam) +// 2. is the denominator real? (not a fabricated oracle) +// 3. does the live signature admit a swap? (seam_contract binding verdict) +// Returns { ok, reason_code, why }; reason_code is a REJECT_CODES key so the caller can route it. +function authoringAdmission(h, ext) { + const seam = String((ext && ext.target_callable) || (h && h.target_callable) || (h && h.live_call_seam) || '').trim(); + if (!seam) return { ok: false, reason_code: 'no_rebind_seam', + why: 'neither the extraction nor the Architect record names a target_callable/live_call_seam — an ' + + 'authored kernel would have nowhere to bind' }; + // NOT a rejection. Deployability is decided by the seam, not by the oracle's provenance: a + // fabricated reference makes the isolated RATIO unusable (withheld upstream), while whether the + // kernel is faster is decided by the live A/B. Rejecting here would have discarded ten measured + // wins in the 88-run archive — see baselineDefect above for the counts. + if (baselineDefect(ext) === 'synthesized') + noteDegradation('authoringAdmission', `${(h && h.short_name) || seam}: authoring a head whose oracle is synthesized`, + 'admitted on the strength of the seam alone; its isolated speedup is unbankable and only a ' + + 'measured e2e A/B can accept it'); + const b = seamBindable(ext); + if (!b.ok) { + const c = (b.codes || []).find((x) => REJECT_CODES[x]) || 'signature_mismatch'; + return { ok: false, reason_code: c, why: b.why }; + } + return { ok: true, reason_code: '', why: b.why, seam }; +} + +// ---- THE RE-EXTRACT CORRECTIVE (INV-5 at the CHEAPEST detection point) ---------------------------- +// A corrective has to name the ACTUAL defect. Telling an extractor "your denominator is invalid" when +// what failed is the BINDING verdict makes it re-run the same validator against the same task dir and +// hand back the same verdict; the retry burns budget and changes nothing. +// +// The two defects do share a root cause often enough to say so out loud, and saying it is what turns +// this from a retry into a fix. When the chosen seam is an OUTER WRAPPER that is not a pure function of +// its arguments — a torch custom-op reading a global layer/KV registry, writing into a caller-owned +// buffer and returning None — it can be neither captured as an oracle NOR rebound. DESCENDING to the +// inner launcher it dispatches to fixes both at once, because when target == the live launcher, the +// frozen baseline IS that launcher. +// 0720: target = baseline = `...ops.chunked_prefill_paged_decode` (the live launcher) -> real +// denominator, kernel bound at the live call site, +18.079% e2e. +// 0802: target = `...attention:unified_attention_with_output` (the custom-op wrapper) -> oracle +// synthesized, entry invented as `attention_forward(args:dict)->fresh_tensor`, rejected +// no_rebind_seam, 0-byte overlay. The run's OWN strategize record already named the inner +// launcher in `live_call_seam`; nothing ever asked the extractor to use it. +// Generic by construction: it describes the SHAPE of the defect (wrapper vs launcher, in-place vs +// fresh-return, hidden context) and points at fields the record already carries. No op, kernel, model +// or backend is named. +function reextractCorrective(needBaseline, bind, codes) { + const DESCEND = + '\n\nSEAM DESCENT — usually the whole fix, and it fixes BOTH defects at once. If the seam you chose ' + + 'is an OUTER WRAPPER (a torch custom-op / dispatcher that reads state which never crosses the ' + + 'parameter boundary, e.g. a global layer or KV-cache registry, and/or writes into a caller-owned ' + + 'buffer and returns None) then it can be neither captured as an oracle nor rebound, and re-running ' + + 'the validator against it will keep returning the same verdict. DESCEND to the innermost launcher ' + + 'that wrapper dispatches to which IS a pure function of its arguments. `KERNEL.live_call_seam` ' + + 'usually already names it. If it does not, read the candidate server.log to see which entry the ' + + 'server ACTUALLY dispatched — backends get overridden at startup, so do NOT trust the env var you ' + + 'set — and grep that module. Then set BOTH `target_callable` and `meta.baseline_callable` to that ' + + 'launcher: when the target IS the live launcher, the frozen baseline is the launcher itself and the ' + + 'denominator defect disappears together with the binding defect.'; + const ENTRY = + '\n\nENTRY CONTRACT — GENERATE IT, DO NOT INVENT IT:\n' + + ' python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --mode entry ' + + '--out /entry_contract.py\n' + + 'This renders the unittest entry FROM the live signature: exact parameter names and order, and ' + + 'whether the seam writes into an out-param and returns None. Bind the unittest to THAT entry and ' + + 'return its path as `entry_contract_path`. The `fn(args) -> FRESH out` shape in the harness ' + + 'anti-exploit rules is the ORACLE\'s timing/correctness contract — it is NOT licence to hand the ' + + 'authored kernel a dict-taking, fresh-returning entry when the live seam is positional and in-place.'; + const BASELINE = + '\n\nDENOMINATOR: set `meta.baseline_callable` to the REAL online kernel — a module:attr that imports ' + + 'from the INSTALLED package, never anything under the task dir and never a reference implementation ' + + 'you wrote — and bind the unittest\'s baseline leg to it.'; + const VERIFY = + '\n\nTHEN RUN THE VENDORED VALIDATOR AND PASTE ITS VERDICT VERBATIM:\n' + + ' python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --eval-dir $EVAL_DIR --mode both\n' + + 'Return its `baseline_validation` and `binding_descriptor`/`binding_check` objects as your fields of ' + + 'the same name, plus baseline_frozen:true. The orchestrator reads the VERDICT, not your description ' + + 'of it: an extraction whose baseline_validation.ok is not true, or whose binding_check.bindable is ' + + 'not true, is INVALID. If after descending there is still no seam that is both capturable and ' + + 'bindable, return editable:false and say why — that routes the op to the config/tune track. NEVER ' + + 'substitute a reference implementation you wrote, and never keep a seam the binding check refused.'; + const head = !bind.ok + ? ' PRIOR ATTEMPT PRODUCED AN UNBINDABLE SEAM' + (codes ? ` (${codes})` : '') + ': ' + bind.why + '.' + + (needBaseline ? ' It also produced no valid speedup denominator.' : '') + : ' PRIOR ATTEMPT DID NOT PRODUCE A VALID SPEEDUP DENOMINATOR.'; + return head + DESCEND + (bind.ok ? '' : ENTRY) + (needBaseline ? BASELINE : '') + VERIFY; +} // Run a kernel_extractor agent and GUARANTEE it froze a real baseline. safeAgent already retries // transient failures; this wraps it to ALSO re-extract when the extraction succeeds (smoke passed, @@ -725,20 +1089,39 @@ async function extractWithBaseline(role, phase, intro, inputs, opts) { const smokeOk = (e) => !!(e && e.task_dir && (e.smoke === 'pass' || e.unittest_smoke === 'pass')); let ext = await safeAgent(roleAgent(role, phase, intro, inputs), opts); let tries = 0; - while (smokeOk(ext) && !hasFrozenBaseline(ext) && tries < BASELINE_EXTRACT_RETRIES) { + // Re-extract for a missing BINDING contract too, not only a missing baseline. The corrective below + // already asks for both verdicts in one seam_contract.py run; without this condition an extraction + // that froze a baseline but skipped --mode binding is never asked again and dies silently at + // authoringAdmission with signature_mismatch, which is a gate firing on an absent field rather than + // on a real defect. Replaying the 88-run archive, that absence alone accounts for every one of the + // 22 accepted-and-measured heads the strict regime would otherwise drop. + const contractOk = (e) => hasFrozenBaseline(e) && seamBindable(e).ok; + while (smokeOk(ext) && !contractOk(ext) && tries < BASELINE_EXTRACT_RETRIES) { tries++; - log(` ${(opts && opts.label) || role}: extraction froze NO baseline ` + - `(baseline_src/ or meta.baseline_callable) — the speedup denominator would fall back to the ` + - `candidate's own scaffold (fake-win). RE-EXTRACTING (retry ${tries}/${BASELINE_EXTRACT_RETRIES}).`); + const needBaseline = !hasFrozenBaseline(ext); + const bind = seamBindable(ext); + const codes = (bind.codes || []).join(', '); + const what = needBaseline && !bind.ok ? 'a frozen baseline AND the seam BINDING contract' + : needBaseline ? 'a frozen baseline (the speedup denominator would fall back to the candidate\'s ' + + 'own scaffold — a fake win)' + : `the seam BINDING contract (${bind.why}) — an authored kernel could not be proven bindable, so ` + + 'its win could never reach the server'; + log(` ${(opts && opts.label) || role}: extraction is missing ${what}. ` + + `RE-EXTRACTING (retry ${tries}/${BASELINE_EXTRACT_RETRIES}).`); ext = await safeAgent( - roleAgent(role, phase, - intro + ' PRIOR ATTEMPT DID NOT FREEZE A BASELINE. You MUST freeze the real online kernel into ' + - 'an immutable baseline_src/ and set meta.baseline_callable (the speedup denominator), bind the ' + - "unittest's baseline leg to it, then return baseline_frozen:true. An extraction with no frozen " + - 'baseline is INVALID and will be discarded.', - inputs), + roleAgent(role, phase, intro + reextractCorrective(needBaseline, bind, codes), inputs), opts); } + // Retries exhausted. Abort only when there is NO usable denominator at all; a synthesized reference + // proceeds with its ratio withheld (see baselineDefect) so the live A/B still gets to decide. + if (smokeOk(ext) && !hasFrozenBaseline(ext) && baselineDefect(ext) === 'synthesized') { + noteDegradation('extractWithBaseline', + `${(opts && opts.label) || role}: baseline is a SYNTHESIZED reference after ${BASELINE_EXTRACT_RETRIES} re-extractions`, + 'keeping the extraction but its isolated speedup is UNBANKABLE (op_bench withholds it and ' + + 'denominatorSound refuses it) — only a measured e2e A/B can accept this head'); + return { ...ext, denominator_invalid: true, + notes: `denominator is a synthesized reference (isolated speedup withheld) — ${ext.notes || ''}` }; + } if (smokeOk(ext) && !hasFrozenBaseline(ext)) { log(` ${(opts && opts.label) || role}: STILL no frozen baseline after ${BASELINE_EXTRACT_RETRIES} ` + `re-extractions — ABORTING this extraction (refusing a fake speedup vs the candidate's own scaffold).`); @@ -826,13 +1209,111 @@ async function trySurgicalFix(spec, reason, fixClass, attempt) { // isolated, base_inputs (the integrate inputs template, carries KERNEL_RESULT), reason, // cur:{overlay,flags,env,tput} }. Returns { banked, integ, isolated } (banked=false if // ineligible or still rejected). See knowledge/learned/method-cudagraph-safe-integration. +// EXTRACT-STAGE RE-ENTRY (INV-5). The reject says the TASK is wrong — wrong seam, wrong call contract, +// or a denominator that was never the online kernel. Fixing that means building a NEW task (new seam, +// new immutable unittest generated FROM the live signature), then authoring against it — not editing a +// kernel that is pinned to the old contract. The caller supplies `spec.reextract(reason, code)`, which +// re-runs extraction with the diagnosis attached and returns a fresh extraction record; without it we +// refuse rather than fall through to a re-author we know cannot work. +// Bounded by HEAD_REEXTRACT_MAX (default 1) and, like every corrective, not charged to HEAD_BUDGET. +const HEAD_REEXTRACT_MAX = parseInt(A.head_reextract_max != null ? A.head_reextract_max : 1, 10); +async function tryExtractReentry(spec, reason, verdict) { + if (typeof spec.reextract !== 'function' || HEAD_REEXTRACT_MAX < 1) { + log(` ${spec.short_name}: reject '${verdict.code || reason}' is EXTRACT-owned (the task's seam/contract ` + + `is wrong, and its unittest is immutable) — re-authoring in place cannot fix it. No re-extract hook ` + + `available here; NOT spending a corrective.`); + return { banked: false, blocked_stage: 'extract', reason_code: verdict.code }; + } + log(` ${spec.short_name}: reject '${verdict.code || reason}' is EXTRACT-owned — RE-EXTRACTING at a ` + + `bindable seam (up to ${HEAD_REEXTRACT_MAX}) instead of re-authoring against the immutable old contract.`); + for (let attempt = 1; attempt <= HEAD_REEXTRACT_MAX; attempt++) { + const ext2 = await spec.reextract(reason, verdict.code, attempt); + if (!ext2 || ext2.smoke !== 'pass' || !ext2.task_dir) { + log(` ${spec.short_name}: re-extract ${attempt}/${HEAD_REEXTRACT_MAX} produced no usable task ` + + `(${ext2 ? ext2.notes || ext2.smoke : 'null'}).`); + continue; + } + const adm = authoringAdmission(spec.head || {}, ext2); + if (!adm.ok) { + log(` ${spec.short_name}: re-extract ${attempt} still not admissible (${adm.reason_code}) — ${adm.why}.`); + continue; + } + if (ext2.task_dir === spec.task_dir) + noteDegradation('tryExtractReentry', 're-extract returned the SAME task_dir', + 'the seam may not have changed; the re-author may reproduce the original reject'); + log(` ${spec.short_name}: re-extracted at ${adm.seam} (task ${ext2.task_dir}); authoring against the ` + + `regenerated contract.`); + let al; + try { + al = await fastBoundedWorkflow({ scriptPath: KERNEL_WF_SCRIPT }, { + kernel_path: ext2.task_dir, workflow_dir: KERNEL_WF_DIR, + mode: 'author', target_language: spec.language || 'triton', + op_spec: { op_kind: ext2.op_kind || spec.op_kind, shapes: ext2.shapes || spec.shapes || {}, + dtype: ext2.dtype || spec.dtype || 'bf16', regime: spec.regime || '', cuda_graph_safe: true, + ...(ext2.workload_path ? { workload_path: ext2.workload_path } : {}) }, + perf_knowledge_dir: KERNEL_KNOWLEDGE_DIR, + use_expert_skills: USE_EXPERT_SKILLS ? 'true' : 'false', expert_skills_dir: EXPERT_SKILLS_DIR, + budget: KERNEL_BUDGET, gpu_ids: spec.gpu_id, exp_root: `${EVAL_DIR}/kernels/_exp`, + task: `RE-EXTRACTED SEAM. The previous attempt was rejected at the e2e gate with reason_code ` + + `'${verdict.code}' ("${reason}"): the kernel was correct and fast in isolation but could not be ` + + `bound at the live call site. This task dir targets a DIFFERENT, verified-bindable seam and its ` + + `unittest entry is GENERATED FROM THE LIVE SIGNATURE — implement exactly that entry contract ` + + `(same parameter names/order, same in-place-vs-return convention). Do not invent a new one. ` + + GRAPH_REQ + (TASK || ''), + apply_to_original: 'false', + }, `${spec.short_name}:reextract`); + } catch (e) { al = { authored: false, validation_status: 'error', reason: String(e) }; } + const iso2 = al && (al.final_weighted != null ? al.final_weighted : al.final_geomean); + if (!al || al.authored === false || !(iso2 > 1.0) || !al.final_patch) { + log(` ${spec.short_name}: re-extract author produced no usable kernel (${al ? al.reason || al.validation_status : 'null'}).`); + continue; + } + const base = spec.base_inputs || {}; + const inputs2 = { ...base, + KERNEL_RESULT: { ...(base.KERNEL_RESULT || {}), + task_dir: ext2.task_dir, target_callable: adm.seam, + code_patch: al.final_patch, final_patch: al.final_patch, + authored_kernel_eval_dir: al.eval_dir || '', verified_isolated_speedup: iso2, + corrective_fix_of: `${verdict.code} (re-extracted seam)` } }; + if (spec.cur) { + inputs2.CURRENT_OVERLAY = spec.cur.overlay; inputs2.CURRENT_FLAGS = spec.cur.flags; + inputs2.CURRENT_ENV = spec.cur.env; inputs2.CURRENT_THROUGHPUT = spec.cur.tput; + } + const integ2 = await runIntegrateBothLegs( + 'Apply the kernel authored against the RE-EXTRACTED seam; gate on e2e throughput.', inputs2, + `integrate ${spec.short_name} reextract`, spec.phase_name || 'HeadKernel'); + const curTput = (spec.cur && spec.cur.tput) || 0; + if (abDone(integ2) && integAccepted(integ2, spec.pct_gpu_time, iso2) && integ2.e2e_throughput_tok_s > curTput) + return { banked: true, integ: integ2, isolated: iso2, via: 'reextract' }; + const r2 = gateRejectReason(integ2, spec.pct_gpu_time, iso2); + log(` ${spec.short_name}: re-extracted candidate still rejected (${r2}).`); + // If the NEW seam is also extract-owned we are going in circles; stop rather than loop. + if (rejectStage(r2, integ2 && integ2.reason_code) === 'extract') break; + } + return { banked: false, blocked_stage: 'extract', reason_code: verdict.code }; +} + async function tryCorrectiveReauthor(spec) { let reason = spec.reason || ''; - // Which fix-and-retry class is this reject? '' = terminal (not auto-correctable). - let fixClass = spec.fix_class || rejectClass(reason); + // Which fix-and-retry class is this reject, and WHICH STAGE OWNS IT? '' = terminal. + const verdict0 = rejectVerdict(reason, spec.reason_code); + let fixClass = spec.fix_class || verdict0.cls; const eligible = HEAD_CORRECTIVE_MAX > 0 && !((FAST_MODE && FAST_DEADLINE_HIT) || TIME_DEADLINE_HIT) && (spec.kernel_eval_dir || spec.task_dir) && (spec.isolated || 0) > 1.0 && fixClass !== ''; if (!eligible) return { banked: false }; + // ---- INV-5: RE-ENTER AT THE STAGE THAT OWNS THE DEFECT -------------------------------------------- + // The loop below re-runs the kernel workflow on spec.task_dir. That dir's unittest.py is IMMUTABLE by + // the extractor's contract, so it pins the entry signature — which means a defect whose fix REQUIRES a + // different call contract or a different seam (no_rebind_seam, signature/arity/return mismatch, hidden + // context, a fabricated denominator) cannot be fixed there, by construction. Re-authoring anyway costs + // hours and lands on the same reject. Route those to the EXTRACT stage instead; route defects the + // kernel track cannot touch at all (profile/strategize) to nobody, loudly. + if (verdict0.stage && verdict0.stage !== 'author') { + if (verdict0.stage === 'extract') return await tryExtractReentry(spec, reason, verdict0); + log(` ${spec.short_name}: reject '${verdict0.code || reason}' is owned by the ${verdict0.stage.toUpperCase()} ` + + `stage — no amount of kernel re-authoring can fix it. NOT spending a corrective; flagging instead.`); + return { banked: false, blocked_stage: verdict0.stage, reason_code: verdict0.code }; + } const curTput = (spec.cur && spec.cur.tput) || 0; // The corrective instruction is CLASS-SPECIFIC. `integration` = the posture is wrong (JIT/capture/ // host-sync); `correctness` = the output is wrong on the live path (parity/accuracy fail or an @@ -933,7 +1414,23 @@ async function tryCorrectiveReauthor(spec) { break; } log(` ${spec.short_name}: corrective still rejected (${reason}).`); - fixClass = implausible2 ? 'correctness' : rejectClass(reason); + if (implausible2) { fixClass = 'correctness'; } + else { + const v2 = rejectVerdict(reason, integ2 && integ2.reason_code); + // If the NEW reject is owned by a different stage, hand off there instead of grinding the + // author loop against a defect it cannot reach (INV-5). + if (v2.stage && v2.stage !== 'author') { + if (v2.stage === 'extract') { + const re = await tryExtractReentry({ ...spec, reason, reason_code: v2.code }, reason, v2); + if (re.banked) return re; + } else { + log(` ${spec.short_name}: corrective surfaced a ${v2.stage.toUpperCase()}-owned defect ` + + `('${v2.code || reason}') — stopping the author loop.`); + } + break; + } + fixClass = v2.cls; + } if (fixClass === '') break; // new failure not auto-correctable -> stop retrying // Progressive: the NEXT attempt builds on this attempt's (partially) fixed kernel, not the original. spec.kernel_eval_dir = fix.eval_dir || spec.kernel_eval_dir; @@ -1108,14 +1605,43 @@ if (want('setup')) { const _isFusedOp = (c) => (c && c.is_fused_kernel === true) || /(?:^|[^a-z])moe(?:[^a-z]|$)|group(?:ed)?[_ ]?gemm|ck_moe|expert|fused[_ ]?moe|fmoe|asm_moe|fused_custom/i .test(`${(c && c.op_kind) || ''} ${(c && c.short_name) || ''} ${(c && c.name) || ''} ${(c && c.classification) || ''} ${(c && c.class) || ''} ${(c && c.backend) || ''}`); - let _fusedTagged = 0; + let _fusedTagged = 0, _seamBackfilled = 0; for (const c of headQueue) { + // SEAM BACKFILL applies to EVERY head, not just fused ones. It used to sit behind the fused test, + // so a non-fused head (attention, a standalone GEMM) whose Architect record carried a live_call_seam + // but no explicit target_callable went into extraction with target_callable=undefined — i.e. the + // extractor was handed the seam as PROSE and had to re-derive it, and any two runs could re-derive + // it differently. Nothing about "copy the seam we already identified into the field that names the + // seam" is specific to fused ops. See INV-2. + if (!c.target_callable && c.live_call_seam) { c.target_callable = c.live_call_seam; _seamBackfilled++; } if (!_isFusedOp(c)) continue; c.op_kind = 'moe'; // grouped-GEMM branch (gemmSynthFor → no dense synth) - if (!c.target_callable && c.live_call_seam) c.target_callable = c.live_call_seam; // bind at the live seam _fusedTagged++; } + if (_seamBackfilled) log(`[op-identity] ${_seamBackfilled} head(s): target_callable backfilled from the Architect's live_call_seam (all op kinds).`); if (_fusedTagged) log(`[op-identity] ${_fusedTagged} fused/grouped head(s): op_kind=moe (never dense-GEMM), bound at live seam — optimized as the fused op, never skipped.`); + // ---- INV-6: ENTITY-KIND ADMISSION ----------------------------------------------------------------- + // The head track optimizes GPU KERNELS. A profile row can also be a torch DISPATCHER op — a Python-side + // span whose `pct_gpu_time` is the SUM of the kernels it dispatches, so routing one as a head both + // double-counts its Amdahl share and points extraction at a wrapper rather than at code that runs on + // the device. parse_profile.py now labels every row `entity_kind`; this admits only gpu_kernel rows. + // Note this is a TYPE check, not a name blacklist: it does not need to recognize `aten::`/`vllm::` + // prefixes, or any future naming convention, to keep a dispatcher span out of the kernel track. + const _dispatcherHeads = headQueue.filter((c) => c && c.entity_kind && c.entity_kind !== 'gpu_kernel'); + if (_dispatcherHeads.length) { + headQueue = headQueue.filter((c) => !(c && c.entity_kind && c.entity_kind !== 'gpu_kernel')); + for (const c of _dispatcherHeads) { + log(` ⚠️ FLAG ${c.short_name || c.name}: profile row is a ${c.entity_kind}, not a gpu_kernel ` + + `(${c.pct_gpu_time || '?'}% is the sum of the kernels it dispatches) — NOT routed to the head ` + + `track; the Architect must name the underlying kernel instead.`); + PRE_FLAGGED_HEADS.push({ short_name: c.short_name || c.name, pct_gpu_time: c.pct_gpu_time, + stage: 'strategize', gate: 'wrong_head_granularity', reason_code: 'wrong_head_granularity', + reason: `entity_kind=${c.entity_kind}; head track requires gpu_kernel` }); + } + } + if (headQueue.some((c) => c && !c.entity_kind)) + noteDegradation('head-admission', 'some head candidates carry no entity_kind', + 'profile rows predate the entity_kind contract — a dispatcher op could still be routed as a kernel head'); log(`Strategy: ${headQueue.length} head candidates, ${kernelQueue.length} kernel candidates, ${(strategy && strategy.config_directions || []).length} config directions.`); // strategize decided the backends -> if any candidate routed flydsl, provision it now (blocking). await ensureFlydslGate(); @@ -1193,7 +1719,7 @@ const acceptedHeads = (ST.accepted_heads || []).slice(); // so Finalize can finish the best one's A/B (Fix C) and so a real isolated win // is surfaced (return.pending_integrations) instead of being silently dropped. const pendingIntegrations = (ST.pending_integrations || []).slice(); -const flaggedHeads = (ST.flagged_heads || []).slice(); // dominant heads that could NOT be optimized (loudly surfaced, never silently skipped) +const flaggedHeads = (ST.flagged_heads || []).slice().concat(PRE_FLAGGED_HEADS); // dominant heads that could NOT be optimized (loudly surfaced, never silently skipped) let headDispatched = 0; const history = ST.history || { insights: [], ledger: [], milestones: [], bottleneck_now: '', suggest_next: '' }; @@ -1201,6 +1727,34 @@ const history = ST.history || { insights: [], ledger: [], milestones: [], bottle // never decomposed into a standalone dense GEMM — so dense-GEMM synth is off for it. function gemmSynthFor(h) { return (h && h.op_kind === 'moe') ? 'false' : GEMM_SYNTH; } +// EXTRACT-STAGE RE-ENTRY HOOK (INV-5), one definition for every track. `tryExtractReentry` can only act +// when the caller hands it a way to rebuild the task; without one it logs "no re-extract hook available" +// and the extract-owned reject dies as a flag. That hook used to exist at exactly ONE of the five +// corrective call sites (fast-mode head integration), so the same defect was repairable or terminal +// depending only on which scheduling path the run happened to take — deep mode, the serial head path and +// the milestone track all fell through. Factored out here so all five behave identically. +// `rec` is the head/kernel record (carries live_call_seam + engagement_check), `priorExt` the extraction +// being replaced. Op-kind agnostic: everything specific to the failure is passed through as PRIOR_*. +function headReextractor(rec, priorExt, phaseName) { + return async (why, code) => await extractWithBaseline( + 'kernel_extractor', 'extract_op', + 'RE-EXTRACT at a BINDABLE seam. The previous task for this op was authored successfully but ' + + 'REJECTED at the live e2e gate with reason_code=' + (code || 'n/a') + ' ("' + why + '"): the kernel ' + + 'could not be bound at (or never ran on) the live call site, so its isolated win could never reach ' + + 'the server. Do NOT rebuild the same task. Pick a seam that the binding contract admits and prove it ' + + 'mechanically before returning.', + { EVAL_DIR, MODEL_PATH, GPU_ID: SERVING_GPU, WORKLOAD, KERNEL: rec, GEMM_SYNTH: gemmSynthFor(rec), + ...(profile && profile.profile_workload_json ? { PROFILE_WORKLOAD_JSON: profile.profile_workload_json } : {}), + CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, + REQUIRE_DECODE_BUCKET: true, DECODE_M_BUCKETS: [1, CONC], + PRIOR_TASK_DIR: (priorExt && priorExt.task_dir) || '', + PRIOR_TARGET_CALLABLE: (priorExt && priorExt.target_callable) || (rec && rec.target_callable) || '', + PRIOR_REJECT_CODE: code || '', PRIOR_REJECT_REASON: why || '', + REQUIRE_BINDING_CONTRACT: true }, + { phase: phaseName || 'HeadKernel', label: `re-extract ${(rec && rec.short_name) || 'op'}`, + schema: EXTRACT_OP_SCHEMA }); +} + // =========================================================================== // PHASE: HeadKernel — the highest-pct_gpu_time ops (GEMM / attention), optimized // regardless of edit flag, via the bake-off ladder. This is the lever the old @@ -1309,6 +1863,17 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { `Return {roofline_note, target_geomean}.`, { phase: 'HeadKernel', label: `roofline ${h.short_name}`, schema: ROOFLINE_SCHEMA }); const rooflineTarget = anchor && Number.isFinite(anchor.target_geomean) ? anchor.target_geomean : 0; + // PRE-AUTHOR ADMISSION (INV-2) — same gate as the fast path, before any lane is opened. Deep mode + // spends the most budget per head, so an unbindable seam is most expensive here. + const admD = authoringAdmission(h, ext); + if (!admD.ok) { + log(` ⚠️ [deep] ${h.short_name}: NOT admitted to authoring (${admD.reason_code}) — ${admD.why}. No lanes opened.`); + flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, stage: 'extract', + gate: 'author_not_admitted', reason_code: admD.reason_code, reason: admD.why }); + history.ledger.push({ direction: h.short_name, verdict: 'flagged', + lesson: `deep author route withheld: ${admD.reason_code} — ${admD.why}` }); + return null; + } const lanes = lanesSpec.map((b) => ({ uid: `${h.short_name}::${b.key || b.lang}`, key: b.key || b.lang, lang: b.lang, mode: b.mode, steer: b.steer || '', state_dir: `${deepDir}/state/${b.key || b.lang}`, best: 1.0, noImprove: 0, active: true, ran: 0, lastEval: '', patch: '', @@ -1443,6 +2008,8 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { short_name: c.head.short_name, op_kind: c.ext.op_kind, shapes: c.ext.shapes, dtype: c.ext.dtype, regime: c.head.regime, gpu_id: SERVING_GPU, kernel_eval_dir: c.lastEval, task_dir: c.ext.task_dir, language: c.lang, isolated: c.best, reason: dreason, fix_class: rejectClass(dreason), pct_gpu_time: c.head.pct_gpu_time, + reason_code: (integ && integ.reason_code) || '', head: c.head, phase_name: 'HeadKernel', + reextract: headReextractor(c.head, c.ext, 'HeadKernel'), base_inputs: { EVAL_DIR, MODEL_PATH, GPU_ID: SERVING_GPU, WORKLOAD, NOISE_BAND_PCT: NOISE_BAND, E2E_REPEATS, KERNEL_RESULT: { @@ -1662,10 +2229,28 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { } const st = { h, ext, cands: [] }; headState.set(h.short_name, st); - if (bake && bake.gate === 'have_winner' && bake.isolated_speedup > 1.0) + // INV-1/INV-3: a bake-off win is only a win if the benchmarker vouched for its own provenance + // (it re-hashed the oracle and confirmed the denominator). op_bench.py now WITHHOLDS + // isolated_speedup entirely when the denominator is unsourced, so a null here is also a no-win. + if (bake && bake.gate === 'have_winner' && bake.isolated_speedup > 1.0 && provenanceOk(bake, 'op_bench') + && denominatorSound(bake, 'op_bench')) st.cands.push({ kind: 'direct_light', source: bake.winner_backend, winner_kind: bake.winner_kind, apply_env: bake.apply_env || '', apply_flags: bake.apply_flags || '', code_patch: bake.code_patch || '', tuning_artifact: bake.tuning_artifact || '', isolated: bake.isolated_speedup, parity_note: bake.parity_note || 'expected_close' }); + // PRE-AUTHOR ADMISSION (INV-2): decide deployability BEFORE the author fan-out, not after it. + // A failing verdict does NOT kill the head — direct_light (env/flag/backend-swap) candidates need + // no rebind and stay in play; only the AUTHORED route, whose whole value depends on being able to + // bind at the seam, is withheld. + const adm = authoringAdmission(h, ext); + if (!adm.ok) { + log(` ⚠️ ${h.short_name}: NOT admitted to the author fan-out (${adm.reason_code}) — ${adm.why}. ` + + `Author budget withheld; ${st.cands.length ? 'the non-authored candidate(s) still proceed' : 'head flagged'}.`); + flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, stage: 'extract', + gate: 'author_not_admitted', reason_code: adm.reason_code, reason: adm.why }); + history.ledger.push({ direction: h.short_name, verdict: 'flagged', + lesson: `author route withheld: ${adm.reason_code} — ${adm.why}` }); + continue; + } for (const ap of (bake && bake.author_plan ? bake.author_plan.slice(0, HEAD_AUTHOR_MAX) : [])) authorJobs.push({ short_name: h.short_name, h, ext, ap, best_known_ms: bake.best_known_ms }); } @@ -1764,11 +2349,19 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { // gateRejectReason injects an implausible_speedup verdict when the gate "passed" but the delta is // impossible (corruption) — so a fake win routes to the correctness corrective instead of banking. const reason = gateRejectReason(integ, h.pct_gpu_time, cand.isolated); - const corr = (cand.kind === 'authored' && rejectClass(reason) !== '') + // Route on the STRUCTURED code when the integrator supplied one (INV-4). `reason_code` also + // decides, inside the corrective, whether to re-author or re-extract (INV-5). + const rcode = (integ && integ.reason_code) || ''; + const rverdict = rejectVerdict(reason, rcode); + const corr = (cand.kind === 'authored' && rverdict.cls !== '') ? await tryCorrectiveReauthor({ short_name: h.short_name, op_kind: st.ext.op_kind, shapes: st.ext.shapes, dtype: st.ext.dtype, regime: h.regime, gpu_id: SERVING_GPU, kernel_eval_dir: cand.kernel_eval_dir, task_dir: st.ext.task_dir, language: cand.language, - isolated: cand.isolated, reason, fix_class: rejectClass(reason), pct_gpu_time: h.pct_gpu_time, phase_name: 'HeadKernel', + isolated: cand.isolated, reason, reason_code: rcode, head: h, + fix_class: rverdict.cls, pct_gpu_time: h.pct_gpu_time, phase_name: 'HeadKernel', + // EXTRACT-stage re-entry hook (INV-5): rebuild the task at a seam that is actually + // bindable, with a unittest entry GENERATED from the live signature, and re-author there. + reextract: headReextractor(h, st.ext, 'HeadKernel'), base_inputs: { EVAL_DIR, MODEL_PATH, GPU_ID: SERVING_GPU, WORKLOAD, NOISE_BAND_PCT: NOISE_BAND, E2E_REPEATS, KERNEL_RESULT: { short_name: h.short_name, task_dir: st.ext.task_dir, op_kind: st.ext.op_kind, @@ -1790,7 +2383,16 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { history.ledger.push({ direction: h.short_name, isolated_speedup: corr.isolated, e2e_delta_pct: corr.integ.e2e_delta_pct, verdict: 'confirmed_corrective', lesson: `fixed: ${reason}` }); } else { log(` ${h.short_name}: REJECTED at e2e gate (${reason}).`); - history.ledger.push({ direction: h.short_name, isolated_speedup: cand.isolated, e2e_delta_pct: integ ? integ.e2e_delta_pct : 0, verdict: 'dead_end', lesson: reason || 'no e2e gain' }); + // A defect the kernel track structurally cannot fix is FLAGGED with its owning stage, not + // filed as a plain dead end — the run report then says WHERE the pipeline has to change. + if (corr.blocked_stage) { + flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, + stage: corr.blocked_stage, gate: 'stage_owned_defect', + reason_code: corr.reason_code || rverdict.code, reason }); + log(` ⚠️ FLAG ${h.short_name}: defect owned by the ${corr.blocked_stage.toUpperCase()} stage ` + + `(${corr.reason_code || rverdict.code || 'unclassified'}) — surfaced, not silently dropped.`); + } + history.ledger.push({ direction: h.short_name, isolated_speedup: cand.isolated, e2e_delta_pct: integ ? integ.e2e_delta_pct : 0, verdict: 'dead_end', lesson: reason || 'no e2e gain', reason_code: rverdict.code || '', owner_stage: rverdict.stage || '' }); } } } @@ -1864,7 +2466,8 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { // Build the candidate list: the cheap direct_light winner (if any) + any authored implementations. const headCands = []; - if (bake.gate === 'have_winner' && bake.isolated_speedup > 1.0) { + if (bake.gate === 'have_winner' && bake.isolated_speedup > 1.0 + && provenanceOk(bake, 'op_bench') && denominatorSound(bake, 'op_bench')) { headCands.push({ kind: 'direct_light', source: bake.winner_backend, winner_kind: bake.winner_kind, apply_env: bake.apply_env || '', apply_flags: bake.apply_flags || '', code_patch: bake.code_patch || '', tuning_artifact: bake.tuning_artifact || '', isolated: bake.isolated_speedup, @@ -1873,7 +2476,17 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { // Author/rewrite route: write (+optimize) a fresh impl per planned language via the recursive kernel // layer. mode=author writes a from-scratch baseline then optimizes it; mode=optimize rewrites an // existing editable impl. The immutable oracle in ext.task_dir is the judge for both. - const plan = (bake.author_plan || []).slice(0, HEAD_AUTHOR_MAX); + // PRE-AUTHOR ADMISSION (INV-2) — same gate as the fast/deep paths. The direct_light candidate above + // needs no rebind and is unaffected; only the author route is withheld when the seam cannot take it. + const admS = authoringAdmission(h, ext); + const plan = admS.ok ? (bake.author_plan || []).slice(0, HEAD_AUTHOR_MAX) : []; + if (!admS.ok && (bake.author_plan || []).length) { + log(` ⚠️ ${h.short_name}: author route withheld (${admS.reason_code}) — ${admS.why}.`); + flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, stage: 'extract', + gate: 'author_not_admitted', reason_code: admS.reason_code, reason: admS.why }); + history.ledger.push({ direction: h.short_name, verdict: 'flagged', + lesson: `author route withheld: ${admS.reason_code} — ${admS.why}` }); + } for (const ap of plan) { const lang = ap.language || 'triton'; let al; @@ -2017,6 +2630,8 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { short_name: h.short_name, op_kind: ext.op_kind, shapes: ext.shapes, dtype: ext.dtype, regime: h.regime, gpu_id: h.gpu_id, kernel_eval_dir: cand.kernel_eval_dir, task_dir: ext.task_dir, language: cand.language, isolated: cand.isolated, base_inputs: headIntegrateInputs, reason, + reason_code: (integ && integ.reason_code) || '', head: h, pct_gpu_time: h.pct_gpu_time, + reextract: headReextractor(h, ext, 'HeadKernel'), cur: { overlay: curOverlay, flags: curFlags, env: curEnv, tput: curTput }, }) : { banked: false }; @@ -2044,6 +2659,8 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { short_name: h.short_name, op_kind: ext.op_kind, shapes: ext.shapes, dtype: ext.dtype, regime: h.regime, gpu_id: h.gpu_id, kernel_eval_dir: cand.kernel_eval_dir, task_dir: ext.task_dir, language: cand.language, isolated: cand.isolated, base_inputs: headIntegrateInputs, reason, fix_class: rejectClass(reason), pct_gpu_time: h.pct_gpu_time, + reason_code: (integ && integ.reason_code) || '', head: h, + reextract: headReextractor(h, ext, 'HeadKernel'), cur: { overlay: curOverlay, flags: curFlags, env: curEnv, tput: curTput }, }) : { banked: false }; @@ -2214,6 +2831,8 @@ while (want('kernel') && !TIME_DEADLINE_HIT && dispatched < BUDGET && (dispatche short_name: c.short_name, op_kind: ext.op_kind, shapes: ext.shapes, dtype: ext.dtype, regime: c.regime, gpu_id: c.gpu_id, kernel_eval_dir: kl.kernel_eval_dir, task_dir: ext.task_dir, language: kl.language || '', isolated: kl.final_geomean, base_inputs: mileIntegrateInputs, reason, fix_class: rejectClass(reason), pct_gpu_time: c.pct_gpu_time, phase_name: 'Milestone', + reason_code: (integ && integ.reason_code) || '', head: c, + reextract: headReextractor(c, ext, 'Milestone'), cur: { overlay: curOverlay, flags: curFlags, env: curEnv, tput: curTput }, }) : { banked: false }; @@ -2425,6 +3044,24 @@ if (want('final')) { // carried best-accepted throughput so a real, parity-checked win is never // reported as 0 / no_gain downstream. validatedOk = !!(validation && validation.director_verified_throughput_tok_s > 0 && validation.throughput_speedup > 0); + // INV-3: `claimed_throughput_tok_s` and `applied_to_original` are asked for precisely so the pipeline + // can be caught overclaiming — and were read by nothing. A gap between what the run claimed and what + // the Director independently measured is the single most important number in the whole report, and a + // win that was never applied to the original tree is not a delivered win. Both are now recorded. + if (validatedOk && validation.claimed_throughput_tok_s > 0) { + const v = validation.director_verified_throughput_tok_s; + const gapPct = 100 * (validation.claimed_throughput_tok_s - v) / v; + if (gapPct > 2) { + noteDegradation('validate', 'the run OVERCLAIMED its throughput', + `claimed ${validation.claimed_throughput_tok_s} tok/s vs Director-verified ${v} tok/s ` + + `(+${gapPct.toFixed(1)}% overclaim) — the reported speedup is the verified one`); + log(` ⚠️ overclaim: claimed ${validation.claimed_throughput_tok_s} vs verified ${v} tok/s (+${gapPct.toFixed(1)}%)`); + } + } + if (validatedOk && !String(validation.applied_to_original || '').trim()) + noteDegradation('validate', 'no applied_to_original path', + 'the Director verified a number but did not state where the win was applied in the original ' + + 'tree — the result may not be reproducible outside the eval dir'); finalSpeedup = validatedOk ? validation.throughput_speedup : (BASELINE_TPUT ? finalTput / BASELINE_TPUT : finalSpeedup); log(`COMPLETE. ${MODEL_NAME}: ${BASELINE_TPUT} -> ${validatedOk ? validation.director_verified_throughput_tok_s : finalTput} tok/s ` + `(${finalSpeedup ? finalSpeedup.toFixed(3) : '?'}x, status ${validation ? validation.validation_status : '?'}` + @@ -2478,6 +3115,7 @@ const carryState = { // Carry pending (verified-isolated, A/B-incomplete) wins WITH their inputs so a // resumed phase run can finish their A/B instead of re-discovering them. pending_integrations: pendingIntegrations, + degradations: DEGRADATIONS, // carried so a resumed phase run inherits, not resets, the audit trail history, }; @@ -2515,6 +3153,12 @@ const wfReturn = { pct_gpu_time: p.pct_gpu_time, partial: p.partial || null, })), flagged_heads: flaggedHeads, // dominant heads surfaced but not optimized (harness/extract/no-candidate) — never silently dropped + // INV-3: every contract this run could NOT enforce, and what it fell back to. An empty list means + // every gate ran on a machine verdict; a non-empty one is the run telling you which conclusions are + // unaudited. Silence used to be indistinguishable from enforcement. + degradations: DEGRADATIONS, + contracts: { baseline_contract_strict: BASELINE_CONTRACT_STRICT, seam_contract_strict: SEAM_CONTRACT_STRICT, + head_reextract_max: HEAD_REEXTRACT_MAX }, config_tune_enabled: CONFIG_TUNE_ENABLED, head_budget: HEAD_BUDGET, head_used: headDispatched, diff --git a/e2e_workflow/roles/e2e_integrator.md b/e2e_workflow/roles/e2e_integrator.md index e823cefe3..e491cdc5f 100644 --- a/e2e_workflow/roles/e2e_integrator.md +++ b/e2e_workflow/roles/e2e_integrator.md @@ -53,10 +53,35 @@ e2e_optimization.md` (measurement discipline + the Amdahl stop rule). waived) and the measured e2e delta BLOWS PAST that ceiling, the kernel is likely doing LESS / degenerate work (corruption) that squeaked past a small accuracy sample — a fast-but-wrong server (truncated / degenerate generations). Re-check accuracy on a LARGER sample vs the TRUE baseline; if it does not - genuinely hold, report `gate:"rejected"` with reason **`implausible_speedup`**. (A byte-exact accept is - NOT subject to this — trust it.) Use the reason vocabulary the orchestrator's auto-correct classifier - keys on — `parity_regression`, `accuracy_regression`, `implausible_speedup`, `output_corruption` — so a - fixable correctness reject is routed to a corrective re-author rather than dropped. + genuinely hold, report `gate:"rejected"` with `reason_code:"implausible_speedup"`. (A byte-exact accept + is NOT subject to this — trust it.) + +### 🔴 `reason_code` — the reject vocabulary is a CLOSED ENUM, and it is what routes the fix +Whenever `gate:"rejected"`, you MUST set `reason_code` to exactly one of these tokens. The orchestrator +routes the corrective off this field, **not** off your prose in `reason`. Prose is for humans; a code is +a decision. (Classifying a reject by pattern-matching an English sentence is not a decision procedure — +it is how `signature_mismatch` became unreachable, because a bare "mismatch" matched the correctness +rule first, so a seam defect was retried as a numerics defect for hours.) + +| owning stage | `reason_code` | means | +|---|---|---| +| **extract** (the TASK encodes the wrong seam/contract/denominator — re-extraction, not re-authoring) | `no_rebind_seam`, `signature_mismatch`, `arity_mismatch`, `param_name_mismatch`, `return_contract_mismatch`, `hidden_context_inputs`, `candidate_unresolvable`, `no_seam_descriptor`, `no_engagement`, `wrong_seam`, `invalid_denominator` | the kernel may be perfect; it cannot be bound where the server actually calls, or its speedup was measured against something that is not the live path | +| **author** (the seam is right; posture or numerics are wrong) | `cuda_graph_capture_unsafe`, `no_binary_for_gpu`, `capture_hang`, `host_sync_in_hot_path`, `oom`, `parity_regression`, `accuracy_regression`, `output_corruption`, `implausible_speedup` | re-authoring on the SAME task dir can fix it | +| **upstream** (no amount of kernel work fixes it) | `wrong_head_granularity`, `delegated_track_disabled` | | +| **terminal** (not a defect) | `no_win`, `do_no_harm` | a correct kernel with no headroom | + +Pick the code that names the FIRST thing that went wrong, not the symptom you noticed last: if the +candidate could not be rebound at the seam, that is `no_rebind_seam`/`signature_mismatch` even though +the visible outcome was "e2e delta was zero". An extract-owned code sent as an author-owned one causes a +guaranteed-futile re-author loop. If nothing in the enum fits, use the closest extract-owned code and +explain in `reason` — an unrecognised token is logged as a degradation and the head is dropped. + +### 🔴 `provenance_ok` +Set `provenance_ok:false` whenever any number you are reporting was not measured by you in this run +against a verified baseline — a carried-over figure, an estimate, a number read out of an earlier +report, or an A/B where the reference leg did not actually run. The orchestrator will not bank a result +with `provenance_ok:false`, which is the correct outcome; reporting `true` for an unsourced number is +the failure this field exists to prevent. Omitting the field entirely is recorded as a degradation. If any fails, REJECT and record why (with the numbers) for the eval-dir timeline report — a real isolated speedup that doesn't show up e2e is an expected Amdahl outcome, not a bug. @@ -310,6 +335,7 @@ Return JSON: "output_parity": "pass|fail", "parity_kind": "byte_exact|accuracy|none", "gate": "accepted|stack|rejected|incomplete", + "reason_code": "", "accepted_overlay": "", "reason": "why accepted/rejected/incomplete (cite Amdahl + measured delta vs noise band)" } diff --git a/e2e_workflow/roles/kernel_extractor.md b/e2e_workflow/roles/kernel_extractor.md index 9464eaf2f..9da5741b7 100644 --- a/e2e_workflow/roles/kernel_extractor.md +++ b/e2e_workflow/roles/kernel_extractor.md @@ -82,7 +82,7 @@ freeze an out-of-regime oracle nobody should trust. (`python3 -c "import sglang,os;print(os.path.dirname(sglang.__file__))"`, then grep the `short_name` / the `module:attr` target). **OP-IDENTITY IS THE RULE: extract the op the LIVE kernel actually is, at the seam it is actually called - from — never a different op.** Two cases: + from — never a different op.** Three cases: - **Standalone LIBRARY op** (a discrete hipBLASLt/rocBLAS `gemm(...)` / library attention whose only call site is that library call, no editable body) → STOP, report `editable=false`, `target_callable=""`; it belongs to the config/tune-hook track (per-shape DB tune / backend env), not a source rewrite. Do @@ -95,6 +95,21 @@ freeze an out-of-regime oracle nobody should trust. library/asm `.so`. That dispatcher seam is what lets a fused op be BACKEND-SWAPPED (aiter/flydsl/triton fused) or AUTHOR-fused-replaced regardless of the underlying kernel's editability. Report `editable=true` (the seam is rebindable). NEVER decompose it into a dense A·Bᵀ GEMM — no live call site. + - **OUTER WRAPPER that is NOT a pure function of its arguments** (a torch custom-op / dispatch entry that + reads a global registry or layer object, writes its result IN PLACE into a caller-owned buffer, and/or + returns `None`): it can be **neither captured as an oracle NOR rebound**, so do NOT try to force it and + do NOT synthesize a reference implementation to stand in for it. **DESCEND** to the innermost launcher + that wrapper dispatches to which IS a pure function of its arguments — + `KERNEL.live_call_seam` usually already names it. Confirm which entry the server ACTUALLY dispatched by + reading the candidate `server.log` (backends get overridden at startup — do NOT trust the env var you + set). Then set **BOTH** `target_callable` **and** `meta.baseline_callable` to that launcher, so the + authored kernel and the speedup denominator are the same live seam. Generate its deployable entry with + `python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --mode entry` — never hand-write it — + and verify with `--mode both` that `baseline_validation.ok` and `binding_check.bindable` are BOTH true. + A synthesized oracle plus an unbindable wrapper is the single failure mode this case exists to prevent: + it yields a kernel-level "speedup" against a number nothing in the server ever computed, and an overlay + that patches nothing. If after descending no seam is both capturable and bindable, report + `editable=false` — that is an honest stop, not a fallback to synthesis. 2. **Capture shapes + oracle** from a live server using `scripts/capture_shapes.py` via a temporary capture overlay, driven by the SAME workload as the profile so shapes match the regime: ```bash @@ -340,6 +355,13 @@ freeze an out-of-regime oracle nobody should trust. > caller. `h.check_correct_multi` catches it (later call overwrites the earlier return; distinct > `data_ptr` + no-mutation asserted). Never write a correctness check that reads each output right > after its own call — check them all together, as the shared lib does. + > **Scope: this is the ORACLE HARNESS's timing/correctness contract, not the deployable entry's.** It + > says the function the unittest times must not alias a static buffer across calls. It is NOT licence + > to give the authored kernel a dict-taking, fresh-returning entry when the LIVE seam is positional and + > writes in place — the deployed entry must match the live seam's real signature (that is what + > `seam_contract.py --mode entry` generates and `--mode bind` checks). An in-place live seam is served + > by an entry that writes into the caller's `out` and returns what the live seam returns; the harness + > still gets its fresh-output wrapper around it. 5. **Finalize `meta.json`**: set `build` (false for pure-Triton; true + a build cmd for HIP/CK/asm candidates), `candidate_backends`, `regime`, the source path in sglang, and re-confirm the `reference_io_sha256` checksum (the validator re-checks it to detect tampering). @@ -369,6 +391,39 @@ freeze an out-of-regime oracle nobody should trust. > Do **NOT** record `unittest_smoke:"fail"` or drop the head for exit 3 — that status is reserved for a > genuine baseline-bind / correctness failure (exit 1). Only after 3 failed regenerations set > `unittest_smoke:"fail"` with `reason="harness_incomplete_unrecoverable"`. +7. **🔴 MANDATORY — machine-check the baseline and the seam binding. Do not hand-write these verdicts.** + The orchestrator gates head admission on the OUTPUT of this script, not on your prose. Run it and + paste the three objects it prints back VERBATIM into your Return JSON: + ```bash + python3 "$SKILL_DIR/scripts/seam_contract.py" \ + --task-dir "" --eval-dir "$EVAL_DIR" \ + --spec "" \ + --mode both --json + ``` + - It imports the spec from the LIVE site-packages, checks the module file actually lives under a real + install root (not a directory you just created), and rejects a synthesized `baseline_src.*` strawman. + `baseline_validation.ok=false` ⇒ the head is NOT admissible: return `editable:false` / + `baseline_frozen:false` with the reported reason. Do not "fix" it by pointing at a file you wrote. + - `binding_descriptor` is DERIVED from `inspect.signature` of the live callable — parameter names, + kinds, defaults, and which parameters are written in place. It is a fact about the seam, not a + guess; never edit it by hand. + - `binding_check` compares your candidate entry point against that descriptor. If it fails, the + candidate cannot be rebound at the seam and the isolated speedup is unbankable — regenerate the + entry from the descriptor instead of arguing with it: + ```bash + python3 "$SKILL_DIR/scripts/seam_contract.py" --task-dir "" --eval-dir "$EVAL_DIR" \ + --spec "" --mode entry --entry-name --out "/entry_contract.py" + ``` + and report that path as `entry_contract_path`. + - `seam_runtime_evidence.inplace_params` MUST list the parameter names the live callable writes + through (e.g. an `output=` buffer). `op_bench.py` uses this list — and ONLY this list, never a name + heuristic — to decide where a replayed in-place seam's result is read from. Getting it wrong makes + correctness unmeasurable, not merely inaccurate. + - `num_cases` MUST be the real number of records in `reference_io.pt`. `0` means the oracle recorded + no calls, so correctness is unfalsifiable, and the orchestrator will reject the head. Report the + true count; do not round it up. + - `reference_io_sha256` MUST be the checksum of the oracle bytes you actually shipped. An empty string + means the oracle is unpinned and tampering is undetectable — also a rejection. Return JSON: ```json @@ -380,6 +435,12 @@ Return JSON: "target_callable": "", "baseline_callable": "", "baseline_frozen": true, + "synthesized": false, + "baseline_validation": { "...": "verbatim from seam_contract.py --mode both" }, + "binding_descriptor": { "...": "verbatim from seam_contract.py --mode both" }, + "binding_check": { "...": "verbatim from seam_contract.py --mode both" }, + "entry_contract_path": "/entry_contract.py or \"\" if the candidate already matches", + "seam_runtime_evidence": { "inplace_params": ["output"] }, "num_cases": 0, "regimes_captured": ["prefill","decode"], "candidate_backends": ["triton","hip","ck"], @@ -718,6 +779,15 @@ force real compact-operand compute: > those M values for every (N,K) — these are non-negotiable; the smoke-test and downstream gate depend on > them.** Combine with the prefill M per `PREFILL_M_NOTE`. +**🔴 Before returning, run the same machine check as `PHASE=extract` step 7** (`seam_contract.py +--mode both`) and paste `baseline_validation` / `binding_descriptor` / `binding_check` back verbatim. +It applies with FULL force here, because `PHASE=extract_op` is the path on which a synthesized oracle is +legal: `synthesized:true` says the reference IO was manufactured, not captured from the live server, and +the orchestrator treats a synthesized baseline as an INVALID speedup denominator. Report it honestly — +a head with `synthesized:true` can still be benchmarked, but its isolated speedup will be withheld +rather than banked, which is the correct outcome. Claiming `synthesized:false` for a manufactured oracle +is the single defect that produced a 4.47× "win" with zero end-to-end effect. + Return JSON: ```json { @@ -731,9 +801,15 @@ Return JSON: "regimes_captured": ["prefill"], "candidate_backends": ["aiter","hipblaslt","triton","ck"], "reference_io_sha256": "", + "num_cases": 0, "target_callable": "", "baseline_callable": "", "baseline_frozen": true, + "baseline_validation": { "...": "verbatim from seam_contract.py --mode both" }, + "binding_descriptor": { "...": "verbatim from seam_contract.py --mode both" }, + "binding_check": { "...": "verbatim from seam_contract.py --mode both" }, + "entry_contract_path": "/entry_contract.py or \"\"", + "seam_runtime_evidence": { "inplace_params": [] }, "smoke": "pass|fail", "notes": "transpose/bias inference, regime, whether oracle was synthesized vs captured" } diff --git a/e2e_workflow/roles/op_benchmarker.md b/e2e_workflow/roles/op_benchmarker.md index a583da50b..596e8e7e1 100644 --- a/e2e_workflow/roles/op_benchmarker.md +++ b/e2e_workflow/roles/op_benchmarker.md @@ -227,6 +227,32 @@ Inputs: `EVAL_DIR`, `OP_TASK_DIR` (from the Kernel Extractor `extract_op`), `OP_ NOTE: the experimental triton GEMM stub is NOT a real implementation — treat "no editable triton kernel for this op" as author-needed. FlyDSL DOES have a real importable GEMM (`flydsl_hgemm` / `flydsl_preshuffle_gemm_a8`), so a flydsl author baseline reuses it rather than starting from zero. +2a. **🔴 THE DENOMINATOR — a speedup is only as real as what it was divided by.** + `op_bench.py` now names the thing it timed as the baseline and reports it as `denominator` (also on + every row as `denominator_provenance`). It is one of: + | `denominator` | meaning | speedup | + |---|---|---| + | `measured_backend_default` | the default backend that the live server actually dispatches, timed here | **bankable** | + | `verified_baseline` | the frozen real online kernel, machine-verified by `seam_contract.py` | **bankable** | + | `unverified_baseline` | a declared baseline whose binding was never checked | withheld unless `--no-denominator-strict` | + | `target_fallback` | no baseline resolved; the candidate was timed against the target itself | **withheld** | + | `synthesized_reference` | the reference IO was manufactured, not captured from the live server | **withheld** | + | `none` | nothing to divide by | **withheld** | + + When the denominator is not sound the script sets `speedup_withheld:true` with + `speedup_withheld_reason`, moves the number to `isolated_speedup_unpublishable`, suppresses + `amdahl_ceiling_e2e_pct`, and prints `speedup=WITHHELD ()`. **In that case you MUST report + `isolated_speedup: null` and `reason_code:"invalid_denominator"` — do NOT copy the unpublishable + figure into `isolated_speedup`, and do NOT re-run with `--no-denominator-strict` to make the number + reappear.** A withheld speedup is not a missing measurement; it is a measurement of the wrong thing. + This is the exact failure being prevented: a kernel timed against a manufactured reference scored + 4.47× and moved end-to-end throughput by nothing at all, because the server never called it. The right + response is to fix the seam (send it back to extract), not to publish the ratio. + + Set `provenance_ok:false` whenever any number you report was not measured by you in this run against a + sound denominator — including a figure carried over from an earlier round or read out of a report. The + orchestrator will not bank a result with `provenance_ok:false`, which is the correct outcome. + 2b. **HARNESS SELF-CHECK + bounded self-repair (do NOT mistake a broken harness for "no win").** Distinguish two completely different outcomes in `opbench_result.json`: - a backend that **ran and produced a number** but was slower / not correct → a legitimate per-backend @@ -301,6 +327,9 @@ Return JSON: "winner_backend": "aiter|hipblaslt|triton|flydsl|ck|none", "winner_kind": "env|flag|patch|none", "isolated_speedup": 1.0, + "denominator": "measured_backend_default|verified_baseline|unverified_baseline|target_fallback|synthesized_reference|none", + "speedup_withheld": false, + "reason_code": "<'invalid_denominator' when speedup_withheld; else ''>", "winner_editable": false, "best_known_ms": 0.0, "recommend_tier_c": false, diff --git a/e2e_workflow/roles/profiler.md b/e2e_workflow/roles/profiler.md index ba1f2a887..3b8715dbb 100644 --- a/e2e_workflow/roles/profiler.md +++ b/e2e_workflow/roles/profiler.md @@ -100,8 +100,17 @@ An upstream orchestrator may already have profiled the SAME baseline workload wi `
` args, `classification`←map from `kernel_category`/`bound_type` (MoE/grouped-GEMM→library_gemm or triton per `kernel_kind`; attention→library_attn; etc.), `editable`←`op_to_source_patchable`. Carry `source_file`/`kernel_path` into each entry's `notes` (the Architect/Extractor reuse them). Write - `profile_topN.json` + `.md` via your own Write (you may shell out to `parse_profile.py` only if you - also have a trace; otherwise assemble the JSON yourself) and set `source:"tracelens"`. + `profile_topN.json` + `.md` via your own Write and set `source:"tracelens"`. + **🔴 Then you MUST annotate the hand-assembled rows — do not hand-write `entity_kind`:** + ```bash + python3 "$EVAL_DIR/parse_profile.py" --annotate "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" \ + --torch-trace "$TLT" --annotate-out "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" + ``` + Annotation needs an evidence source (`--torch-trace` and/or `--rocprof-dir`): if `TLT` is empty there + is no trace to cross-check the TraceLens rows against, so **do not hand-stamp `entity_kind` — fall + back to the normal collection (steps 1–5)** and annotate from your own trace. The annotator exits + non-zero if any Top-N row is `unresolved`: resolve those before returning, because a row whose kind is + unknown may not be a GPU kernel at all. - **If `TRACELENS_TRACE_FILE` is also a non-empty path that EXISTS → run an ADDITIONAL trace-analysis pass on top of analysis.md to sharpen the picture** (this is required by contract when the trace is present). `TRACELENS_TRACE_FILE` is a `torch_trace` **directory** that holds one steady-state serving @@ -230,6 +239,20 @@ degrade to whatever is available, and if both analysis.md and trace are unusable skill errors out at any point, note it and return the Top-N anyway — a failed analysis skill must never fail or block the profile.** +### 🔴 `entity_kind` — every Top-N row must say WHAT KIND OF THING it is +`parse_profile.py` stamps each row with `entity_kind ∈ {gpu_kernel, memory_op, dispatcher_op, +python_launcher, unresolved}` plus `entity_evidence` (the profiler category the kind was DERIVED from). +This is a fact about how the work was observed, not a guess from the name — so never edit it, and never +substitute a name pattern for it. The existing `classification`/`backend_guess` fields ARE name guesses +and are labelled as such; they are not a substitute. + +Why it gates: only a `gpu_kernel` row is a rewrite target. A `dispatcher_op` row is a host span that +ENCLOSES the kernels it dispatched, so its time is already counted in them — routing a head at it +double-counts and the "optimization" cannot move e2e. A `memory_op` row is a copy (fix the allocation, +not the kernel). A `python_launcher` row is host overhead. An `unresolved` row is one this profiler +never saw dispatched at all, and the orchestrator's head-admission gate refuses it. Rows you have not +annotated are treated as `unresolved`, so run the annotator; do not fill the field in by hand. + Return JSON: ```json { @@ -241,7 +264,8 @@ Return JSON: "total_gpu_time_ms": 0.0, "top_kernels": [ {"rank": 1, "short_name": "...", "classification": "...", "pct_gpu_time": 0.0, - "calls": 0, "avg_us": 0.0, "shapes": [[...]], "editable": true, "regime_note": "prefill|decode|both"} + "calls": 0, "avg_us": 0.0, "shapes": [[...]], "editable": true, "regime_note": "prefill|decode|both", + "entity_kind": "gpu_kernel", "entity_evidence": ""} ], "shift_note": "for reprofile: how the bottleneck moved vs previous round", "notes": "resolved 'other' entries, rocprof availability, anything unusual" diff --git a/e2e_workflow/roles/system_architect.md b/e2e_workflow/roles/system_architect.md index b888c8309..0c7b86744 100644 --- a/e2e_workflow/roles/system_architect.md +++ b/e2e_workflow/roles/system_architect.md @@ -241,6 +241,20 @@ OPTIONAL upstream TraceLens prior (may be empty strings — treat empty/missing e2e by MORE than the noise band. Otherwise drop it — say so. 5. Write `EVAL_DIR/strategy.md` (human-readable plan) and return the routing. +> **🔴 Every `head_candidates` entry MUST carry `entity_kind` and `target_callable`, copied forward — +> not re-derived.** +> - `entity_kind` comes verbatim from the profile row (`parse_profile.py --annotate` stamped it from the +> profiler's own event category). Only `gpu_kernel` rows may be routed to the head track: the +> orchestrator drops and loudly flags anything else. A `dispatcher_op` row is a HOST span that encloses +> the kernels it dispatched, so its GPU time is already counted in them — scheduling a head there books +> Amdahl mass twice and guarantees the "win" cannot appear e2e. If a row's `entity_kind` is `unresolved`, +> send it back to the Profiler rather than routing it on a name that looks like a kernel. +> - `target_callable` is the `module:attr` the Extractor must actually bind — normally the callable part +> of `live_call_seam`. State it explicitly: the Extractor's speedup is only bankable if it is measured +> at the seam the server really dispatches, and leaving the seam implicit is what lets a plausible-looking +> reference implementation be frozen as "the baseline" and produce a large isolated speedup with zero +> end-to-end effect. + Return JSON: ```json { @@ -253,6 +267,8 @@ Return JSON: "head_candidates": [ {"id": "h0", "short_name": "...", "op_kind": "gemm|attn", "pct_gpu_time": 0.0, "shapes": "[[1024,5120],[5120,34816]]", "dtype": "bf16", "regime": "prefill|decode|both", + "entity_kind": "gpu_kernel", + "target_callable": "", "transpose_b": true, "bias": false, "candidate_backends": ["aiter","hipblaslt","triton","ck"], "is_fused_kernel": false, diff --git a/e2e_workflow/scripts/check_schema_consumption.py b/e2e_workflow/scripts/check_schema_consumption.py new file mode 100644 index 000000000..4bdd377ad --- /dev/null +++ b/e2e_workflow/scripts/check_schema_consumption.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""CI check: every field an agent is ASKED to return must be READ by something. + +Why this exists +--------------- +The orchestrator declares response schemas (`const X_SCHEMA = obj({...})`) that agents are contractually +required to fill in. A declared-but-never-read field is worse than a missing one: the agent spends +tokens computing it, the reviewer sees it in the JSON and assumes it gated something, and the pipeline +proceeds on a check that does not exist. `synthesized` was exactly this — declared in EXTRACT_OP_SCHEMA, +asked for in the role prompt, written truthfully by the extractor, and consumed by no line of code, so a +fabricated baseline sailed through the head gate. + +This check is deliberately syntactic and conservative: it flags a field only when NO plausible read of +that name occurs anywhere outside a schema literal. It cannot prove a field is used correctly; it can +prove a field is used nowhere, which is the failure mode above. + +Usage: + python3 check_schema_consumption.py [--file e2e_workflow.js] [--json] [--list] + exit 0 = every declared field has a reader; exit 1 = at least one does not. + +Waivers: add a field name to ALLOWED_UNCONSUMED below WITH a reason. A waiver is a statement that the +field is documentation for the agent, not an input to a decision. + +Stdlib only. +""" +import argparse +import json +import os +import re +import sys + +DEFAULT_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "e2e_workflow.js") + +# name -> why it is legitimately write-only. +ALLOWED_UNCONSUMED = { + "notes": "free-text rationale carried into the report/ledger for humans, not a gate input", + "smoke": "required evidence string the agent must produce; read by humans reviewing the ledger", + "why": "human-facing justification", + "note": "human-facing justification", + "summary": "human-facing narrative", + "rationale": "human-facing narrative", + "evidence": "human-facing narrative", + "risk": "human-facing narrative", + + # --- artifact PATHS: written so a human (or a later run) can open the file. Nothing branches on + # them; the orchestrator addresses those artifacts by its own EVAL_DIR convention, not by the path + # the agent reports. If one ever becomes an input to a decision, delete its waiver. + "baseline_summary_path": "artifact path for the reader; the orchestrator uses its own EVAL_DIR path", + "bench_script": "artifact path for reproduction; the orchestrator re-derives the bench invocation", + "profile_topN_md": "human-readable twin of profile_topN.json, which IS consumed", + "strategy_path": "artifact path for the reader; the strategy object itself is consumed", + + # --- telemetry that lands in the report/ledger. Reported, not gated. + "baseline_spread_pct": "run-quality telemetry surfaced in the report; the gate uses the A/B medians", + "total_gpu_time_ms": "profile telemetry; ranking uses pct_gpu_time, which IS consumed", + "throughput_speedup_vs_baseline": "sweep telemetry; the sweep winner is chosen on absolute tok/s", + "trials": "sweep search log kept for the report", + "workload": "the setup's echo of the workload config it was given; the orchestrator holds the source of truth", + "regime_summary": "prose summary of the regime split for the report; the regime object is consumed", + "order_of_work": "the strategist's suggested ordering, superseded by the orchestrator's own head queue", + "drop_list": "advisory 'do not pursue' list for the report; heads are dropped by measured gates", + "accepted_config": "config bundle passed through to the report writer verbatim", + "playbook_appended": "acknowledgement that the experience curator wrote its file", + "per_backend": "full bake-off table kept for the report; the winner fields are consumed", + "recommend_tier_c": "advisory routing hint; the actual route is decided by winner_kind + admission", + "winner_editable": "duplicate of winner_kind=='patch_candidate', which is what routes", + "arbitration_note": "the Director's prose when report and validation disagree", + "build": "extraction fact recorded for the surgeon's build step, which reads the task meta.json directly", + "regimes_captured": "recorded in the ledger; regime coverage is enforced inside the frozen unittest", +} + + +def find_schema_blocks(src): + """-> [(schema_name, body_text, start, end)] for `const NAME_SCHEMA = obj({ ... })` declarations.""" + out = [] + for m in re.finditer(r"const\s+([A-Za-z0-9_]*SCHEMA)\s*=\s*obj\(\s*\{", src): + i = m.end() - 1 # at the '{' + depth, j = 0, i + while j < len(src): + if src[j] == "{": + depth += 1 + elif src[j] == "}": + depth -= 1 + if depth == 0: + break + j += 1 + out.append((m.group(1), src[i + 1:j], m.start(), j + 1)) + return out + + +def _strip_comments(s): + s = re.sub(r"/\*.*?\*/", " ", s, flags=re.S) + return re.sub(r"//[^\n]*", " ", s) + + +def fields_of(body): + """Top-level property names of a schema body, ignoring nested object/array literals. + + Walks the text tracking brace/bracket depth so `properties` of an inline nested obj({...}) are not + mistaken for fields of the outer schema — those are checked as part of their own nested read. + """ + body = _strip_comments(body) + names, depth, i, tok_start = [], 0, 0, 0 + while i < len(body): + c = body[i] + if c in "{[(": + depth += 1 + elif c in "}])": + depth -= 1 + elif c == ":" and depth == 0: + seg = body[tok_start:i].strip().strip(",").strip() + m = re.search(r"([A-Za-z_][A-Za-z0-9_]*)\s*$", seg) + if m: + names.append(m.group(1)) + tok_start = i + 1 + elif c == "," and depth == 0: + tok_start = i + 1 + i += 1 + return names + + +def nested_field_names(body): + """Every property name at ANY depth — used to catch nested declarations too.""" + body = _strip_comments(body) + return set(re.findall(r"([A-Za-z_][A-Za-z0-9_]*)\s*:\s*(?:\{|arr|obj\()", body)) + + +def consumption_sites(code, field): + """Plausible reads of `field` in `code` (which already has schema literals removed).""" + pats = [ + rf"\.{re.escape(field)}\b", # o.field + rf"\[\s*['\"]{re.escape(field)}['\"]\s*\]", # o['field'] + rf"\b{re.escape(field)}\s*[,}}]", # const { field } = o / { field, x } + rf"['\"]{re.escape(field)}['\"]", # 'field' as a key/lookup string + ] + return sum(len(re.findall(p, code)) for p in pats) + + +def check(path): + with open(path) as fh: + src = fh.read() + blocks = find_schema_blocks(src) + if not blocks: + return {"error": f"no `const *SCHEMA = obj({{...}})` declarations found in {path}"}, 1 + + # Code = the file with every schema literal cut out, so a field's own declaration is not + # mistaken for a read of it. + code, last = [], 0 + for _, _, s, e in sorted(blocks, key=lambda b: b[2]): + code.append(src[last:s]) + last = e + code.append(src[last:]) + code = _strip_comments("".join(code)) + + declared, findings = {}, [] + for name, body, _, _ in blocks: + for f in fields_of(body) + sorted(nested_field_names(body)): + declared.setdefault(f, set()).add(name) + + for f in sorted(declared): + if f in ("type", "properties", "required", "items", "enum", "additionalProperties", + "description", "default", "format"): + continue # JSON-Schema vocabulary, not a payload field + n = consumption_sites(code, f) + if n == 0: + findings.append({"field": f, "schemas": sorted(declared[f]), + "waived": f in ALLOWED_UNCONSUMED, + "waiver_reason": ALLOWED_UNCONSUMED.get(f, "")}) + unwaived = [x for x in findings if not x["waived"]] + return {"file": os.path.relpath(path), "num_schemas": len(blocks), + "num_fields": len(declared), "unconsumed": findings, + "num_unconsumed_unwaived": len(unwaived)}, (1 if unwaived else 0) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--file", default=DEFAULT_FILE) + ap.add_argument("--json", action="store_true") + ap.add_argument("--list", action="store_true", help="also print every declared field") + a = ap.parse_args() + + rep, rc = check(os.path.abspath(a.file)) + if a.json: + print(json.dumps(rep, indent=2)) + return rc + if "error" in rep: + print("ERROR: " + rep["error"]) + return rc + print(f"{rep['file']}: {rep['num_schemas']} schemas, {rep['num_fields']} declared fields") + for x in rep["unconsumed"]: + tag = "WAIVED " if x["waived"] else "UNREAD " + print(f" {tag} {x['field']:<32} declared in {', '.join(x['schemas'])}" + + (f" ({x['waiver_reason']})" if x["waived"] else "")) + if rc: + print(f"\nFAIL: {rep['num_unconsumed_unwaived']} field(s) are requested from an agent and read " + f"by nothing. Either consume them or add a waiver with a reason in " + f"check_schema_consumption.ALLOWED_UNCONSUMED.") + else: + print("\nOK: every declared field has a reader (or a documented waiver).") + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/e2e_workflow/scripts/op_bench.py b/e2e_workflow/scripts/op_bench.py index 82f7a3158..10cc07142 100644 --- a/e2e_workflow/scripts/op_bench.py +++ b/e2e_workflow/scripts/op_bench.py @@ -122,6 +122,294 @@ def _correct(torch, out, ref, tol): return False, float("inf") +# ------------------------------------------------------------------- INV-1/INV-3: denominator identity +# A speedup is a RATIO. The numerator is always measured here; the denominator is only meaningful if it +# is the thing the deployment actually runs. Every previous silent `baseline or target` fallback made the +# ratio a self-comparison (or a comparison against a reference this script itself invented) while still +# printing a headline number. These helpers make the denominator's provenance an explicit, recorded fact, +# and let main() WITHHOLD the ratio rather than publish an unsourced one. +# +# Op-kind agnostic by construction: nothing below inspects op_kind, dtype, shapes or kernel names. + +DEGRADATIONS = [] + + +def note_degradation(where, what, detail=""): + d = {"where": where, "what": what, "detail": str(detail)[:400]} + DEGRADATIONS.append(d) + sys.stderr.write(f"[op_bench][degraded] {where}: {what}" + (f" — {detail}\n" if detail else "\n")) + return d + + +# severity: "ok" -> the denominator is the deployed path; a ratio against it is publishable +# "unverified"-> plausibly the deployed path, but nothing proved it; publishable only with +# --no-denominator-strict, and always carries its provenance +# "invalid" -> provably not a denominator (self-comparison, or a reference we synthesized) +DENOM_SEVERITY = { + "measured_backend_default": "ok", # baseline row is a real backend we timed on this box + "verified_baseline": "ok", # meta.baseline_validation proved it is the live seam + "unverified_baseline": "unverified", # meta.baseline_callable, but no seam_contract.py verdict + "target_fallback": "invalid", # no baseline declared -> candidate vs itself + "synthesized_reference": "invalid", # oracle was fabricated, not captured from the seam + "none": "invalid", # nothing to divide by +} + + +def resolve_denominator(meta): + """Resolve the baseline spec AND why we believe it is one. Never falls back silently. + + Returns {spec, provenance, severity, why}. `spec` may be non-empty even when severity is "invalid": + callers may still want to RUN it (a self-comparison is a valid sanity check), they just may not + report a speedup against it. + """ + bl = str(meta.get("baseline_callable") or "").strip() + tgt = str(meta.get("target_callable") or "").strip() + v = meta.get("baseline_validation") + verified = (isinstance(v, dict) and v.get("contract") == "baseline_identity" and v.get("ok") is True) + + def out(spec, prov, why): + return {"spec": spec, "provenance": prov, "severity": DENOM_SEVERITY[prov], "why": why} + + if meta.get("synthesized") is True: + return out(bl or tgt, "synthesized_reference", + "meta.synthesized=true: the reference was written by the extractor, not captured " + "from the live seam, so it is an oracle for CORRECTNESS only, never a perf denominator") + if bl and verified: + return out(bl, "verified_baseline", + "seam_contract.py --mode baseline reported ok=true for this spec") + if bl: + return out(bl, "unverified_baseline", + "meta.baseline_callable is declared but carries no baseline_validation verdict " + "(run scripts/seam_contract.py --mode baseline)") + if tgt: + return out(tgt, "target_fallback", + "no baseline_callable in meta.json; using target_callable would compare the " + "candidate against ITSELF") + return out("", "none", "meta.json declares neither baseline_callable nor target_callable") + + +def _delegation_status(meta): + """Some heads have no in-process bake-off at all: their only lever is a server-level flag owned by + the Config-Tuner track. That is a legitimate outcome ONLY while that track is actually running. If it + is off, 'delegated' means 'nobody measured this', which must surface as a reject code, not as a pass. + + Recognised meta keys (any one of them, all optional): + delegated_config_track: true|false config_track_enabled: true|false + Absent -> "unknown": we say so instead of assuming enabled. + """ + for key in ("delegated_config_track", "config_track_enabled"): + if key in meta and meta[key] is not None: + return "enabled" if bool(meta[key]) else "disabled" + return "unknown" + + +# --------------------------------------------------------------- INV-3: generic captured-oracle replay +def _rehydrate(obj, torch, device): + """Inverse of capture_shapes._snapshot: {'__tensor__':True,'data':...} -> a device tensor. + Structure-preserving over list/tuple/dict; scalars pass through; un-snapshottable objects + (recorded as {'__repr__': ...}) come back as a sentinel so the caller can refuse to replay.""" + if isinstance(obj, dict) and obj.get("__tensor__"): + return obj["data"].to(device) + if isinstance(obj, dict) and "__repr__" in obj: + return _UNREPLAYABLE + if isinstance(obj, (list, tuple)): + return type(obj)(_rehydrate(v, torch, device) for v in obj) + if isinstance(obj, dict): + return {k: _rehydrate(v, torch, device) for k, v in obj.items()} + return obj + + +class _Unreplayable: + def __repr__(self): + return "" + + +_UNREPLAYABLE = _Unreplayable() + + +def _has_unreplayable(obj): + if obj is _UNREPLAYABLE: + return True + if isinstance(obj, (list, tuple)): + return any(_has_unreplayable(v) for v in obj) + if isinstance(obj, dict): + return any(_has_unreplayable(v) for v in obj.values()) + return False + + +def load_oracle_records(task, torch, device): + """Return (records, err). Each record: {sig, regime, args, kwargs, output} with tensors on `device`. + `err` is a human string when the oracle cannot be replayed; records is then [].""" + iopath = os.path.join(task, "reference_io.pt") + if not os.path.exists(iopath): + return [], "no reference_io.pt in the task dir" + try: + blob = torch.load(iopath, map_location="cpu", weights_only=False) + except Exception as e: + return [], f"reference_io.pt unreadable: {e!r}" + recs = blob.get("records") if isinstance(blob, dict) else None + if not recs: + return [], "reference_io.pt carries no 'records' (not a capture_shapes oracle)" + out = [] + for r in recs: + args = _rehydrate(r.get("args", ()), torch, device) + kwargs = _rehydrate(r.get("kwargs", {}), torch, device) + ref = _rehydrate(r.get("output", None), torch, device) + if _has_unreplayable(args) or _has_unreplayable(kwargs): + continue + out.append({"sig": r.get("sig", ""), "regime": r.get("regime", ""), + "args": tuple(args) if isinstance(args, (list, tuple)) else (args,), + "kwargs": dict(kwargs) if isinstance(kwargs, dict) else {}, + "ref": ref}) + if not out: + return [], (f"all {len(recs)} recorded case(s) contain non-tensor arguments the capture could " + f"only store as repr() — cannot replay this seam in-process") + return out, "" + + +def _out_param_of(rec, meta): + """For an IN-PLACE seam (returns None, writes into a caller-supplied buffer) the oracle's `output` + snapshot is None and the golden values live in one of the INPUTS after the call. Which one is a fact + the extractor records; we do not guess from names here beyond a last-resort default, and we say so. + + Returns (kind, key) with kind in {"kwarg","arg",""}. + """ + ev = meta.get("seam_runtime_evidence") or {} + names = [str(n) for n in (ev.get("inplace_params") or []) if str(n)] + for n in names: + if n in rec["kwargs"]: + return "kwarg", n + if names: + return "", "" # declared, but not present in this record -> refuse, don't guess + return "", "" + + +def _clone_inputs(rec, torch): + """A fresh copy of args/kwargs so an in-place seam cannot poison the next timing iteration.""" + def cl(o): + if torch.is_tensor(o): + return o.clone() + if isinstance(o, (list, tuple)): + return type(o)(cl(v) for v in o) + if isinstance(o, dict): + return {k: cl(v) for k, v in o.items()} + return o + return cl(rec["args"]), cl(rec["kwargs"]) + + +def bench_captured_replay(args, meta): + """Time the CURRENT seam callable against its captured oracle by replaying the recorded call. + + This replaces the former attention stub, which returned {"correct": True, "ms": None} unconditionally + — a pass with no measurement behind it. It is op-kind agnostic: it replays whatever capture_shapes + recorded at whatever seam, so it serves attention, MoE routing, norms, or anything else that has an + oracle but no cross-backend bake-off. + + Each result row carries `denominator_provenance` so main() can decide whether the ratio is + publishable. + """ + torch = _torch() + device = "cuda" if torch.cuda.is_available() else "cpu" + denom = resolve_denominator(meta) + delegation = _delegation_status(meta) + results = [] + + recs, err = load_oracle_records(args.task, torch, device) + if err: + note_degradation("bench_captured_replay", "no replayable oracle", err) + return [{"backend": "current", "available": False, "correct": False, "ms": None, + "raised": False, "reason_code": "invalid_denominator", + "denominator_provenance": denom["provenance"], + "note": f"cannot measure this seam: {err}"}] + + # Bench the DOMINANT recorded case (most-repeated sig) — same convention as the blockscale path. + recs.sort(key=lambda r: -int((meta.get("shape_counts_by_sig") or {}).get(r["sig"], 0))) + rec = recs[0] + + plan = [] + tgt = str(meta.get("target_callable") or "").strip() + if tgt: + plan.append(("current", tgt)) + if denom["spec"] and denom["spec"] != tgt: + plan.append(("baseline", denom["spec"])) + + if not plan: + note_degradation("bench_captured_replay", "no callable to time", denom["why"]) + return [{"backend": "current", "available": False, "correct": False, "ms": None, + "raised": False, "reason_code": "invalid_denominator", + "denominator_provenance": denom["provenance"], "note": denom["why"]}] + + # In-place seam: capture_shapes snapshots the arguments AFTER invoking the original, so the recorded + # out-buffer already holds the golden values. Replay therefore takes the GOLDEN from that buffer and + # zeroes the copy it hands to the callee — otherwise the callee would be handed the answer. + ip_kind, ip_key = _out_param_of(rec, meta) + golden = rec["ref"] + if ip_kind == "kwarg" and torch.is_tensor(rec["kwargs"].get(ip_key)): + golden = rec["kwargs"][ip_key].clone() + + for name, spec in plan: + fn = _resolve_callable(spec) + if fn is None: + results.append({"backend": name, "available": False, "correct": False, "ms": None, + "raised": False, "reason_code": "candidate_unresolvable", + "denominator_provenance": denom["provenance"], + "note": f"callable not importable: {spec}"}) + continue + + def call_once(): + a, k = _clone_inputs(rec, torch) + if ip_kind == "kwarg" and torch.is_tensor(k.get(ip_key)): + k[ip_key].zero_() + r = fn(*a, **k) + return r if r is not None else (k.get(ip_key) if ip_kind == "kwarg" else None) + + try: + out = call_once(); _sync(torch) + out = call_once() + except Exception as e: + results.append({"backend": name, "available": True, "correct": False, "ms": None, + "raised": True, "denominator_provenance": denom["provenance"], + "note": f"replay raised: {e!r}"}) + continue + + ref = golden + if out is None or ref is None or not torch.is_tensor(out) or not torch.is_tensor(ref): + # We CAN time it, but we cannot certify it. Say exactly that instead of asserting correct. + ms, wall_ms = _time_call(call_once, args.warmup, args.repeats) + why = ("seam returns None and meta.seam_runtime_evidence.inplace_params does not name the " + "output buffer" if out is None else + "oracle recorded no comparable output tensor for this case") + note_degradation("bench_captured_replay", f"{name}: timed but unverified", why) + results.append({"backend": name, "available": True, "correct": None, "ms": round(ms, 4) if ms else None, + "wall_ms": round(wall_ms, 4) if wall_ms else None, "raised": False, + "denominator_provenance": denom["provenance"], + "note": f"{spec} @ {rec['sig']} ({rec['regime']}) — UNVERIFIED: {why}"}) + continue + + ok, rel = _correct(torch, out, ref, args.tol) + ms, wall_ms = _time_call(call_once, args.warmup, args.repeats) + results.append({"backend": name, "available": True, "correct": bool(ok), + "max_rel_err": round(rel, 5) if math.isfinite(rel) else None, + "ms": round(ms, 4) if ms else None, + "wall_ms": round(wall_ms, 4) if wall_ms else None, "raised": False, + "denominator_provenance": denom["provenance"], + "note": f"{spec} @ {rec['sig']} ({rec['regime']})"}) + + if delegation == "disabled": + note_degradation("bench_captured_replay", "delegated config track is DISABLED", + "cross-backend comparison for this head has no owner; op-level timing above is " + "a reference only") + results.append({"backend": "delegated_config_track", "available": False, "correct": False, + "ms": None, "raised": False, "reason_code": "delegated_track_disabled", + "denominator_provenance": denom["provenance"], + "note": "server-level backend comparison is delegated to the Config Tuner, and " + "that track is disabled for this run — nothing measured it"}) + elif delegation == "unknown": + note_degradation("bench_captured_replay", "delegated config track state unknown", + "meta declares neither delegated_config_track nor config_track_enabled") + return results + + # ----------------------------------------------------------------------------- GEMM bake-off def _dtype(torch, name): # Prefer the shared, ARCH-DRIVEN resolver so a bare "fp8"/"fp8_e4m3" picks the running GPU's fp8 @@ -275,6 +563,7 @@ def record(name, fn, note=""): # An EXCEPTION (not a slow/incorrect number) -> candidate could not run. The op_benchmarker # treats "all candidates raised" as a harness self-fault (see its role); we surface it clearly. results.append({"backend": name, "available": True, "correct": False, "ms": None, + "denominator_provenance": denom["provenance"], "note": f"call raised: {e!r}", "raised": True}) return ok, err = _correct(torch, out, case["ref"], args.tol) @@ -283,9 +572,16 @@ def record(name, fn, note=""): "max_rel_err": round(err, 5) if math.isfinite(err) else None, "ms": round(ms, 4) if ms else None, "wall_ms": round(wall_ms, 4) if wall_ms else None, + "denominator_provenance": denom["provenance"], "note": note, "raised": False}) - base_spec = meta.get("baseline_callable") or meta.get("target_callable") + # INV-1: the denominator is resolved WITH its provenance, never by a silent `baseline or target`. + # We still run whatever spec we have (a self-comparison is a usable sanity check); what changes is + # that main() now knows whether the resulting ratio may be published. + denom = resolve_denominator(meta) + if denom["severity"] != "ok": + note_degradation("bench_blockscale_gemm", f"denominator is {denom['provenance']}", denom["why"]) + base_spec = denom["spec"] tgt_spec = meta.get("target_callable") or base_spec seen = set() plan = [("aiter_blockscale", base_spec)] @@ -668,22 +964,13 @@ def _triton_matmul(torch, A, B, bias, transpose_b, autotune): return out -# ----------------------------------------------------------------------------- attention (best-effort) +# --------------------------------------------------------- non-GEMM heads (captured-oracle replay path) def bench_attn(args, meta): - """Attention op-level timing of the CURRENT captured callable against its oracle. Cross-backend - comparison for attention is done at the SERVER level by the Config Tuner (--attention-backend), - so here we only (a) confirm the oracle reproduces and (b) time the current path as a reference. - Returns a single-entry result list; backend swaps are reported as 'delegated to config track'.""" - torch = _torch() - device = "cuda" if torch.cuda.is_available() else "cpu" - iopath = os.path.join(args.task, "reference_io.pt") - if not os.path.exists(iopath): - return [{"backend": "current", "available": False, "correct": False, "ms": None, - "note": "attn bake-off needs reference_io.pt (captured q/k/v/meta); none found"}] - note = ("attention backend comparison is a SERVER-level flag (--attention-backend) -> delegated to " - "the Config Tuner fast path; op-level here only validates the oracle") - return [{"backend": "current", "available": True, "correct": True, "ms": None, - "note": note, "artifact": iopath}] + """Back-compat alias. The former body returned {"correct": True, "ms": None} for every attention + task without calling anything — a green result with no measurement under it, which is how an + unvalidated head reached the integrator. Timing now goes through the generic captured-oracle replay + (see bench_captured_replay), which is not attention-specific.""" + return bench_captured_replay(args, meta) # ----------------------------------------------------------------------------- main @@ -697,6 +984,10 @@ def main(): ap.add_argument("--triton-autotune", action="store_true") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--out", default="") + # INV-1: refuse to publish a speedup whose denominator nobody verified. Off only for deliberate + # exploratory runs, and even then the provenance travels with the number. + ap.add_argument("--denominator-strict", dest="denominator_strict", action="store_true", default=True) + ap.add_argument("--no-denominator-strict", dest="denominator_strict", action="store_false") a = ap.parse_args() meta_path = os.path.join(a.task, "meta.json") @@ -714,14 +1005,18 @@ def main(): _GRAPH_MODE = _hlib.deployment_graph_mode(meta.get("regime")) try: - results = bench_gemm(a, meta) if op_kind == "gemm" else bench_attn(a, meta) + # GEMM has a real cross-backend bake-off; every other op kind is timed by replaying its captured + # oracle at its own seam. No op kind falls through to an unmeasured pass. + results = bench_gemm(a, meta) if op_kind == "gemm" else bench_captured_replay(a, meta) except Exception as e: results = [{"backend": "ERROR", "available": False, "correct": False, "ms": None, "note": f"{e!r}", "trace": traceback.format_exc()[-800:]}] correct = [r for r in results if r.get("correct") and r.get("ms")] correct.sort(key=lambda r: r["ms"]) - baseline = next((r for r in results if r["backend"] in ("hipblaslt", "current", "aiter_blockscale") and r.get("ms")), None) + # Prefer an explicitly-named baseline row (the replay path emits one) over the library default. + baseline = next((r for r in results if r["backend"] == "baseline" and r.get("ms")), None) or \ + next((r for r in results if r["backend"] in ("hipblaslt", "current", "aiter_blockscale") and r.get("ms")), None) winner = correct[0] if correct else None # ---- Harness self-fault signal (for op_benchmarker self-repair + orchestrator dominant-head guard). @@ -739,6 +1034,25 @@ def main(): harness_error = str(r0.get("note") or r0.get("trace") or "unknown harness error")[:400] speedup = (baseline["ms"] / winner["ms"]) if (winner and baseline and winner["ms"]) else ( 1.0 if winner else 0.0) + + # ---- INV-1: is that ratio publishable? + # The denominator's provenance is whatever the bench function attached to the baseline row. A row we + # actually timed as a deployed library backend (hipblaslt/rocblas/...) is self-evidently the deployed + # path; a row sourced from meta is only as good as meta's validation. + denom = resolve_denominator(meta) + prov = (baseline or {}).get("denominator_provenance") + if not prov: + prov = "measured_backend_default" if baseline else denom["provenance"] + severity = DENOM_SEVERITY.get(prov, "invalid") + denom_why = denom["why"] if prov == denom["provenance"] else \ + f"baseline row '{(baseline or {}).get('backend')}' was timed on this box as the deployed default" + withhold = (severity == "invalid") or (severity == "unverified" and a.denominator_strict) + speedup_reason = "" + if withhold: + speedup_reason = (f"denominator provenance is '{prov}' ({severity}): {denom_why}. " + f"Reporting a ratio against it would be a number without a comparison.") + note_degradation("main", "isolated_speedup WITHHELD", speedup_reason) + reported_speedup = None if withhold else round(speedup, 4) wb = winner["backend"] if winner else None # Only triton/hip are source-editable (-> Tier-C kernel-squad rewrite). ck is a library backend. editable = bool(wb in ("triton", "hip")) @@ -763,7 +1077,8 @@ def main(): # the bake-off result. pct_gpu_time absent -> ceiling omitted (None). pct_gpu = meta.get("pct_gpu_time", meta.get("pct_gpu", None)) amdahl_ceiling_pct = None - if _hlib is not None and pct_gpu is not None and winner: + # A ceiling derived from a withheld speedup would launder the unpublishable number back in. + if _hlib is not None and pct_gpu is not None and winner and not withhold: try: amdahl_ceiling_pct = round(_hlib.amdahl_ceiling(float(pct_gpu), float(speedup)), 3) except Exception: @@ -779,7 +1094,16 @@ def main(): "winner_ms": winner["ms"] if winner else None, "baseline_backend": baseline["backend"] if baseline else None, "baseline_ms": baseline["ms"] if baseline else None, - "isolated_speedup": round(speedup, 4), + "isolated_speedup": reported_speedup, + "isolated_speedup_unpublishable": round(speedup, 4) if withhold else None, + "speedup_withheld": bool(withhold), + "speedup_withheld_reason": speedup_reason, + "denominator": {"spec": denom["spec"], "provenance": prov, "severity": severity, + "why": denom_why, "strict": bool(a.denominator_strict)}, + "delegated_config_track": _delegation_status(meta), + "degradations": DEGRADATIONS, + "reason_code": next((r.get("reason_code") for r in results if r.get("reason_code")), + "invalid_denominator" if withhold else ""), "pct_gpu_time": pct_gpu, "amdahl_ceiling_e2e_pct": amdahl_ceiling_pct, "winner_editable": editable, @@ -797,10 +1121,13 @@ def main(): with open(a.out, "w") as fh: fh.write(out) print(out) - print(f"OPBENCH winner={summary['winner_backend']} speedup={summary['isolated_speedup']}x " + print(f"OPBENCH winner={summary['winner_backend']} " + + (f"speedup=WITHHELD ({prov})" if withhold else f"speedup={summary['isolated_speedup']}x") + + f" denominator={prov} " f"editable={summary['winner_editable']} kind={summary['winner_kind']} " f"harness_suspect={summary['harness_suspect']}" - + (f" harness_error={summary['harness_error']!r}" if summary['harness_suspect'] else "")) + + (f" harness_error={summary['harness_error']!r}" if summary['harness_suspect'] else "") + + (f" degradations={len(DEGRADATIONS)}" if DEGRADATIONS else "")) if __name__ == "__main__": diff --git a/e2e_workflow/scripts/parse_profile.py b/e2e_workflow/scripts/parse_profile.py index d6e7146cc..f89db0e72 100644 --- a/e2e_workflow/scripts/parse_profile.py +++ b/e2e_workflow/scripts/parse_profile.py @@ -24,10 +24,17 @@ "total_gpu_time_ms": float, "num_kernel_launches": int, "num_distinct_kernels": int, + "entity_kind_contract": "v1", # present => every row carries entity_kind + evidence + "entity_kind_counts": {kind: n, ...}, "top_kernels": [ { "rank", "name", "short_name", "calls", "total_ms", "avg_us", "pct_gpu_time", "shapes": [[...dims...], ...], # up to 5 distinct input-dim sets "dtypes": [...], # distinct input dtypes seen + "entity_kind": "gpu_kernel|memory_op|dispatcher_op|python_launcher|unresolved", + # OBSERVED (profiler event category / rocprof kernel + # stats), not inferred from the name. The head track + # admits gpu_kernel only. + "entity_evidence": {...}, # what justified that kind "classification": "triton|library_gemm|library_attn|fused_custom|" "elementwise_overhead|reduction_norm|memory|other", "backend_guess": "triton|hipblaslt|aiter|ck|rocblas|torch_native|unknown", @@ -259,9 +266,13 @@ def _phase_of(ts): total_us += dur launches += 1 d = agg.setdefault(name, {"calls": 0, "total_us": 0.0, "shapes": set(), - "dtypes": set(), "by_case": {}, "by_phase": {}}) + "dtypes": set(), "by_case": {}, "by_phase": {}, "cat_counts": {}}) d["calls"] += 1 d["total_us"] += dur + # INV-6 evidence: remember WHICH profiler category this row's samples came from, so the Top-N + # can state whether it is a dispatched kernel or a memory op rather than leaving it to a guess. + cat = e.get("cat") + d["cat_counts"][cat] = d["cat_counts"].get(cat, 0) + 1 # attribute this launch to its serving phase (measured from the step span it falls in) phase, stepM = _phase_of(e.get("ts")) if phase: @@ -341,7 +352,8 @@ def parse_rocprof_dir(d): us = ns / 1000.0 total_us += us launches += calls - e = agg.setdefault(name, {"calls": 0, "total_us": 0.0, "shapes": set(), "dtypes": set()}) + e = agg.setdefault(name, {"calls": 0, "total_us": 0.0, "shapes": set(), "dtypes": set(), + "src": "rocprof_kernel_stats"}) e["calls"] += calls e["total_us"] += us break # one stats file is the authoritative aggregate @@ -353,6 +365,110 @@ def norm_key(name): return re.sub(r"[^a-z0-9]", "", short_name(name).lower()) +# --------------------------------------------------------------------------- # +# INV-6: what KIND of thing is this row? +# +# The head track rewrites GPU kernels. A row that is actually a dispatcher op or a Python launcher can +# top a table by wall time while owning no device code to rewrite, and the downstream stages have no way +# to tell — a name is not evidence. So every Top-N row now carries `entity_kind` plus the EVIDENCE that +# produced it, derived from the profiler event category, never from a name pattern or a blacklist. +# +# gpu_kernel a dispatched compute kernel (rocprofv3 kernel-stats row, or a torch trace event +# with cat="kernel"). The only kind the head track accepts. +# memory_op a device memcpy/memset. Real device time, but no kernel source to rewrite. +# dispatcher_op a host-side operator span (torch cat="cpu_op"/"user_annotation"): its duration +# includes everything it dispatched, so it double-counts its own children. +# python_launcher a host span with no device work attributed to it at all. +# unresolved claimed by an upstream tool but not found among the dispatched kernels. +# --------------------------------------------------------------------------- # +ENTITY_KINDS = ("gpu_kernel", "memory_op", "dispatcher_op", "python_launcher", "unresolved") + +# torch profiler event category -> entity kind. Categories are emitted by the profiler itself, so this +# is a fact about how the work was observed, not a guess about what the symbol is called. +_CAT_ENTITY = { + "kernel": "gpu_kernel", + "gpu_memcpy": "memory_op", + "gpu_memset": "memory_op", + "cpu_op": "dispatcher_op", + "user_annotation": "dispatcher_op", + "cuda_runtime": "python_launcher", + "hip_runtime": "python_launcher", +} + + +def classify_entity(d, source): + """-> (entity_kind, evidence). `d` is an agg entry; `source` the profile source string.""" + cats = d.get("cat_counts") or {} + if cats: + # A name can appear under several categories; the device categories decide. + kinds = {} + for cat, n in cats.items(): + kinds[_CAT_ENTITY.get(cat, "unresolved")] = kinds.get(_CAT_ENTITY.get(cat, "unresolved"), 0) + n + for k in ("gpu_kernel", "memory_op", "dispatcher_op", "python_launcher"): + if kinds.get(k): + return k, {"basis": "torch_profiler_event_category", + "categories": dict(sorted(cats.items())), "device_events": kinds[k]} + if d.get("src") == "rocprof_kernel_stats": + return "gpu_kernel", {"basis": "rocprofv3_kernel_stats_row", + "why": "rocprofv3 kernel-stats aggregates dispatched kernels only", + "calls": d.get("calls", 0)} + return "unresolved", {"basis": "no_category_evidence", + "why": f"source={source!r} carried no per-event category for this row"} + + +def index_host_entities(path): + """Name -> agg-shaped stub for HOST-side spans (cpu_op / annotations / runtime launch stubs). + + These are deliberately excluded from the timing aggregate — a cpu_op's duration subsumes the + kernels it dispatched, so counting it would double-count device time. But for --annotate we still + want to be able to say "this claimed head IS a dispatcher op" rather than the weaker "not found". + Used only by the annotate path, so the normal parse is unchanged. + """ + idx = {} + try: + with _open(path) as fh: + data = json.load(fh) + except Exception: + return idx + events = data.get("traceEvents", data if isinstance(data, list) else []) + host = {"cpu_op", "user_annotation", "cuda_runtime", "hip_runtime"} + for e in events: + if not isinstance(e, dict): + continue + cat = e.get("cat") + if cat not in host: + continue + d = idx.setdefault(e.get("name", "?"), {"calls": 0, "total_us": 0.0, "cat_counts": {}}) + d["calls"] += 1 + d["total_us"] += float(e.get("dur", 0.0) or 0.0) + d["cat_counts"][cat] = d["cat_counts"].get(cat, 0) + 1 + return idx + + +def annotate_rows(rows, agg, source): + """Stamp entity_kind onto Top-N rows that were assembled OUTSIDE this script (the TraceLens + fast path builds them by hand from a third-party report). A row survives as a gpu_kernel only if its + name matches something this profiler actually observed being dispatched; otherwise it is + `unresolved` and the head track will refuse it. Returns (rows, stats).""" + by_key = {} + for name, d in (agg or {}).items(): + by_key.setdefault(norm_key(name), (name, d)) + stats = {k: 0 for k in ENTITY_KINDS} + for r in rows: + hit = by_key.get(norm_key(r.get("name") or r.get("short_name") or "")) + if hit: + kind, ev = classify_entity(hit[1], source) + ev["matched_profiled_kernel"] = hit[0] + else: + kind = "unresolved" + ev = {"basis": "name_not_found_in_profile", + "why": f"not among the {len(by_key)} kernels observed in {source}"} + r["entity_kind"] = kind + r["entity_evidence"] = ev + stats[kind] = stats.get(kind, 0) + 1 + return rows, stats + + def build_summary(agg, total_us, launches, source, top_n, enrich=None, conc=0, isl=0, osl=0, chunk=None, capture_sizes=None, phase_meta=None): items = [] @@ -392,6 +508,7 @@ def build_summary(agg, total_us, launches, source, top_n, enrich=None, top = [] for rank, (name, d) in enumerate(items[:top_n], 1): cls, backend, editable, hint = classify(name) + entity_kind, entity_evidence = classify_entity(d, source) shapes = sorted(d["shapes"]) if d["shapes"] else [] dtypes = sorted(d["dtypes"]) if d["dtypes"] else [] if not shapes and enrich: @@ -410,6 +527,10 @@ def build_summary(agg, total_us, launches, source, top_n, enrich=None, "shapes": [json.loads(s) for s in shapes[:5]], "dtypes": dtypes[:8], "classification": cls, + # INV-6: `classification`/`backend_guess` are name-pattern GUESSES (see RULES) and are + # labelled as such. `entity_kind` is not a guess — it is what the profiler observed. + "entity_kind": entity_kind, + "entity_evidence": entity_evidence, "backend_guess": backend, "editable": editable, "opt_hint": hint, @@ -438,6 +559,10 @@ def build_summary(agg, total_us, launches, source, top_n, enrich=None, "total_gpu_time_ms": round(total_us / 1000.0, 4), "num_kernel_launches": launches, "num_distinct_kernels": len(agg), + "entity_kind_contract": "v1", # consumers may require this before trusting entity_kind + "entity_kind_counts": {k: sum(1 for e in top if e["entity_kind"] == k) + for k in ENTITY_KINDS + if any(e["entity_kind"] == k for e in top)}, "top_kernels": top, } if serving: @@ -571,6 +696,13 @@ def main(): help="max_num_batched_tokens (chunked-prefill budget) from server.log") ap.add_argument("--capture-sizes", default="", help="comma list of cudagraph_capture_sizes (to snap decode est_shape M)") + # INV-6: stamp entity_kind onto a Top-N JSON that was assembled elsewhere (TraceLens fast path). + # The rows are cross-checked against a trace/rocprof aggregate parsed here; an unmatched row is + # marked `unresolved`, which the head-admission gate refuses. + ap.add_argument("--annotate", default="", + help="existing profile_topN json to stamp with entity_kind (needs --torch-trace " + "and/or --rocprof-dir as the evidence source)") + ap.add_argument("--annotate-out", default="", help="where to write the annotated json (default: in place)") args = ap.parse_args() if not args.torch_trace and not args.rocprof_dir: @@ -589,6 +721,31 @@ def main(): if args.rocprof_dir: rp_agg, rp_total, rp_launch = parse_rocprof_dir(args.rocprof_dir) + if args.annotate: + with open(args.annotate) as fh: + doc = json.load(fh) + ev_agg = dict(rp_agg or {}) + # Host spans first, so a device row of the same name overrides them below. + if args.torch_trace: + ev_agg.update(index_host_entities(args.torch_trace)) + ev_agg.update(torch_agg or {}) # torch categories are richer evidence; let them win + ev_src = doc.get("source") or ("merged" if (rp_agg and torch_agg) else + "rocprofv3" if rp_agg else "torch-trace") + rows, stats = annotate_rows(doc.get("top_kernels") or [], ev_agg, ev_src) + doc["top_kernels"] = rows + doc["entity_kind_contract"] = "v1" + doc["entity_kind_counts"] = {k: v for k, v in stats.items() if v} + dest = args.annotate_out or args.annotate + with open(dest, "w") as fh: + fh.write(json.dumps(doc, indent=2)) + sys.stderr.write(f"annotated {len(rows)} row(s) -> {dest}: " + + ", ".join(f"{k}={v}" for k, v in stats.items() if v) + "\n") + bad = stats.get("unresolved", 0) + print(json.dumps({"annotated": len(rows), "entity_kind_counts": + {k: v for k, v in stats.items() if v}}, indent=2)) + # Non-zero exit when a claimed head is not a kernel this profiler ever saw dispatched. + sys.exit(1 if bad else 0) + if rp_agg and torch_agg: summ = build_summary(rp_agg, rp_total, rp_launch, "merged", args.top, enrich=torch_agg, phase_meta=torch_pmeta, **phk) diff --git a/e2e_workflow/scripts/seam_contract.py b/e2e_workflow/scripts/seam_contract.py new file mode 100644 index 000000000..5ebf5df6d --- /dev/null +++ b/e2e_workflow/scripts/seam_contract.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +"""seam_contract.py -- machine-checkable contracts for the two things an extraction ASSERTS but the +orchestrator has never been able to CHECK: (1) what the speedup denominator actually is, and (2) what +call contract an authored kernel has to satisfy to be rebindable at the live seam. + +Why this file exists +-------------------- +The role prompts already carry these rules as prose (kernel_extractor.md: "THE BASELINE LEG IS ALWAYS +THE FROZEN REAL ONLINE KERNEL", "never fabricate an oracle", "prove engagement before authoring"). +Prose rules are followed on some runs and not on others, and the orchestrator's only check was +`typeof baseline_callable === 'string' && !== ''` -- which a task-local `baseline_src.attn_ref: +attention_forward` scaffold satisfies perfectly. A run can then post a large isolated speedup measured +against a pure-torch strawman it wrote itself, spend the whole kernel budget on it, and only discover +at integrate time that the number was never going to carry end-to-end. + +Both problems are op-kind-agnostic and both are decidable by reflection, so they belong in code: + + INV-1 denominator identity -- the object named by meta.baseline_callable must be importable from + OUTSIDE the task dir, live in an installed distribution, and be the + same seam as (or an observed callee of) meta.target_callable. + INV-2 binding contract -- the live callable's signature is captured mechanically and becomes + THE contract. An authored entry is checked against it BEFORE any + authoring budget is spent, so "the overlay cannot bind" is caught in + seconds by inspect.signature instead of in hours by a serving A/B. + +Nothing here knows about attention, MoE, GEMM or any backend. It only knows `module:attr`. + +Usage (the extractor runs this and pastes the JSON into its return value): + python3 seam_contract.py --task-dir --mode both --json + python3 seam_contract.py --spec pkg.mod:fn --mode binding --json + python3 seam_contract.py --task-dir --mode entry --out entry_contract.py + +Exit code is 0 when every requested contract holds, 1 otherwise -- so a shell caller fails closed too. +Stdlib only (importing the seam itself may of course pull in torch; that is the caller's environment). +""" +from __future__ import annotations + +import argparse +import importlib +import inspect +import json +import os +import site +import sys +import sysconfig + +CONTRACT_VERSION = 1 + +# Parameter names that conventionally denote a caller-provided output buffer written IN PLACE. Used +# only to RAISE a flag (out_params.evidence == "name_convention"); runtime evidence from the capture +# step, when present in meta, overrides it. Kept deliberately small and generic. +_OUT_PARAM_NAMES = {"out", "output", "o", "dst", "dest", "y", "result", "out_tensor", "output_tensor"} + + +# --------------------------------------------------------------------------------- spec resolution +def parse_spec(spec): + """'pkg.mod:attr' or 'pkg.mod.attr' -> (module_name, attr_path). Returns (None, None) if empty.""" + if not spec or not str(spec).strip(): + return None, None + s = str(spec).strip() + if ":" in s: + mod, _, attr = s.partition(":") + return mod.strip(), attr.strip() + # dotted form: last component is the attribute + if "." not in s: + return None, None + mod, _, attr = s.rpartition(".") + return mod.strip(), attr.strip() + + +def resolve_spec(spec): + """Import and resolve a module:attr spec. Never raises -- returns a verdict dict.""" + mod_name, attr_path = parse_spec(spec) + if not mod_name or not attr_path: + return {"ok": False, "spec": spec, "error": "unparseable spec (want 'module:attr')"} + try: + # The extractor freezes files and validates in the same process; without this, a module + # written moments ago is invisible to the import system's cached directory listings. + importlib.invalidate_caches() + mod = importlib.import_module(mod_name) + except Exception as e: # noqa: BLE001 -- any import failure is a resolution failure + return {"ok": False, "spec": spec, "module": mod_name, "error": f"import failed: {e!r}"} + obj = mod + for part in attr_path.split("."): + if not hasattr(obj, part): + return {"ok": False, "spec": spec, "module": mod_name, + "error": f"module has no attribute {attr_path!r}"} + obj = getattr(obj, part) + try: + file = inspect.getfile(obj) + except Exception: # builtins / C extensions have no source file + file = getattr(sys.modules.get(getattr(obj, "__module__", ""), None), "__file__", None) + return {"ok": True, "spec": spec, "module": mod_name, "attr": attr_path, "obj": obj, + "file": os.path.realpath(file) if file else None, + "qualname": getattr(obj, "__qualname__", getattr(obj, "__name__", str(obj))), + "callable": callable(obj)} + + +# --------------------------------------------------------------------------------- origin analysis +def _site_dirs(): + dirs = [] + for fn in ("purelib", "platlib"): + try: + p = sysconfig.get_paths().get(fn) + if p: + dirs.append(os.path.realpath(p)) + except Exception: + pass + try: + dirs.extend(os.path.realpath(p) for p in site.getsitepackages()) + except Exception: + pass + try: + usp = site.getusersitepackages() + if isinstance(usp, str): + dirs.append(os.path.realpath(usp)) + except Exception: + pass + # Overlay / editable / vendored install roots that sysconfig does not know about. The overlay the + # integrator builds is a legitimate install location, so a run must be able to declare it rather + # than have every baseline in it classified `unknown` and fail-closed for the wrong reason. + for extra in (os.environ.get("GEAK_SEAM_SITE_DIRS", "") or "").split(os.pathsep): + if extra.strip(): + dirs.append(os.path.realpath(extra.strip())) + return sorted(set(d for d in dirs if d)) + + +def _stdlib_dirs(): + out = [] + for fn in ("stdlib", "platstdlib"): + try: + p = sysconfig.get_paths().get(fn) + if p: + out.append(os.path.realpath(p)) + except Exception: + pass + return sorted(set(out)) + + +def _under(path, root): + if not path or not root: + return False + path = os.path.realpath(path) + root = os.path.realpath(root) + return path == root or path.startswith(root.rstrip(os.sep) + os.sep) + + +def origin_of(path, task_dir=None, eval_dir=None): + """Classify where a resolved callable's source file lives. + + Order matters: a file under the task dir is task_local EVEN IF the task dir happens to sit inside + site-packages, because the point of the check is "did the extraction time itself against something + it wrote". `unknown` is a FAILING classification, not a benign one -- an anonymous path is exactly + what a scaffold dropped in cwd looks like. + """ + if not path: + return {"kind": "unresolved", "distribution": None, "path": None} + rp = os.path.realpath(path) + if task_dir and _under(rp, task_dir): + return {"kind": "task_local", "distribution": None, "path": rp} + if eval_dir and _under(rp, eval_dir): + return {"kind": "eval_local", "distribution": None, "path": rp} + for d in _stdlib_dirs(): + if _under(rp, d) and not any(_under(rp, s) for s in _site_dirs()): + return {"kind": "stdlib", "distribution": None, "path": rp} + for d in _site_dirs(): + if _under(rp, d): + return {"kind": "installed", "distribution": _distribution_for(rp, d), "path": rp} + return {"kind": "unknown", "distribution": None, "path": rp} + + +def _distribution_for(realpath, site_dir): + """Best-effort distribution name: the first path component under site-packages.""" + rel = os.path.relpath(realpath, site_dir) + top = rel.split(os.sep)[0] + top = top[:-3] if top.endswith(".py") else top + try: + import importlib.metadata as md + for dist, tops in (md.packages_distributions() or {}).items(): + if dist == top and tops: + return tops[0] + except Exception: + pass + return top or None + + +# ------------------------------------------------------------------------- INV-1 baseline identity +def validate_baseline(task_dir=None, meta=None, eval_dir=None): + """INV-1: is meta.baseline_callable a legitimate speedup DENOMINATOR? + + Six checks, each independently reportable so a failure says which one broke. A `False` on any of + them means the isolated speedup this task will produce is not comparable to anything the live + server runs, and the task must be dropped (`editable:false`) rather than authored against. + """ + meta = dict(meta or {}) + checks = [] + + def add(cid, ok, detail): + checks.append({"id": cid, "ok": bool(ok), "detail": detail}) + + base_spec = (meta.get("baseline_callable") or "").strip() + tgt_spec = (meta.get("target_callable") or "").strip() + + add("B1_baseline_declared", bool(base_spec), + f"meta.baseline_callable={base_spec!r}" if base_spec else "meta.baseline_callable is missing/empty") + + add("B6_not_synthesized", meta.get("synthesized") is not True, + "meta.synthesized is true -- the oracle was fabricated, it cannot be the denominator" + if meta.get("synthesized") is True else "meta.synthesized is not true") + + base_res = resolve_spec(base_spec) if base_spec else {"ok": False, "error": "no spec"} + add("B2_baseline_resolvable", base_res.get("ok"), + base_res.get("error") or f"resolved to {base_res.get('qualname')} @ {base_res.get('file')}") + + base_origin = origin_of(base_res.get("file"), task_dir, eval_dir) if base_res.get("ok") else \ + {"kind": "unresolved", "distribution": None, "path": None} + add("B3_baseline_origin_installed", base_origin["kind"] == "installed", + f"origin={base_origin['kind']}" + + (f" dist={base_origin['distribution']}" if base_origin["distribution"] else "") + + (" -- a baseline living in the task/eval dir is the candidate's own scaffold, not the online kernel" + if base_origin["kind"] in ("task_local", "eval_local") else "") + + (" -- source file is not inside any installed distribution" if base_origin["kind"] == "unknown" else "")) + + tgt_res = resolve_spec(tgt_spec) if tgt_spec else {"ok": False, "error": "meta.target_callable missing/empty"} + add("B4_target_resolvable", tgt_res.get("ok"), + tgt_res.get("error") or f"resolved to {tgt_res.get('qualname')} @ {tgt_res.get('file')}") + + # B5: the denominator must BE the seam, or be provably reached FROM it. Identity is checked on the + # resolved code object (so two spellings of the same function pass). Otherwise we require positive + # evidence recorded at capture time -- an assertion in prose is not evidence. + same_obj = bool(base_res.get("ok") and tgt_res.get("ok") + and (base_res.get("obj") is tgt_res.get("obj") + or getattr(base_res.get("obj"), "__code__", None) + is getattr(tgt_res.get("obj"), "__code__", object()))) + ev = meta.get("baseline_capture_evidence") or {} + observed = isinstance(ev, dict) and int(ev.get("observed_calls") or 0) > 0 \ + and (ev.get("from_seam") or "").strip() == tgt_spec + add("B5_denominator_is_the_seam", same_obj or observed, + "baseline_callable IS target_callable" if same_obj else + (f"observed {ev.get('observed_calls')} live call(s) from {ev.get('from_seam')!r}" if observed else + "baseline_callable is neither the seam nor an OBSERVED callee of it " + "(set meta.baseline_capture_evidence={from_seam, observed_calls} at capture time)")) + + ok = all(c["ok"] for c in checks) + return { + "contract": "baseline_identity", "contract_version": CONTRACT_VERSION, "ok": ok, + "baseline_callable": base_spec, "target_callable": tgt_spec, + "baseline_origin": {k: v for k, v in base_origin.items() if k != "obj"}, + "checks": checks, + "failed": [c["id"] for c in checks if not c["ok"]], + "verdict": ("denominator is the live online kernel" if ok else + "INVALID DENOMINATOR -- any speedup measured against it is not comparable to e2e"), + } + + +# -------------------------------------------------------------------------- INV-2 binding contract +def describe_binding(spec, meta=None): + """Capture the LIVE callable's call contract by reflection. This descriptor -- not an agent's + recollection of it -- is what an authored entry has to satisfy.""" + meta = dict(meta or {}) + res = resolve_spec(spec) + if not res.get("ok"): + return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": spec, "error": res.get("error")} + obj = res["obj"] + if not callable(obj): + return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": spec, "error": "seam resolves to a non-callable"} + try: + sig = inspect.signature(obj) + except Exception as e: # noqa: BLE001 + return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": spec, "error": f"signature unavailable: {e!r}"} + + params, required_positional, out_params = [], [], [] + accepts_varargs = accepts_varkw = False + for name, p in sig.parameters.items(): + if p.kind is inspect.Parameter.VAR_POSITIONAL: + accepts_varargs = True + if p.kind is inspect.Parameter.VAR_KEYWORD: + accepts_varkw = True + ann = "" if p.annotation is inspect.Parameter.empty else _ann_str(p.annotation) + has_def = p.default is not inspect.Parameter.empty + params.append({"name": name, "kind": p.kind.name, "has_default": has_def, "annotation": ann}) + if (not has_def and p.kind in (inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY)): + required_positional.append(name) + if name.lower() in _OUT_PARAM_NAMES: + out_params.append(name) + + ret_ann = "" if sig.return_annotation is inspect.Signature.empty else _ann_str(sig.return_annotation) + returns_none = ret_ann in ("None", "NoneType") + + # Runtime evidence from the capture step, when the extractor recorded it, beats the name heuristic. + ev = meta.get("seam_runtime_evidence") or {} + if isinstance(ev, dict) and ev.get("inplace_params") is not None: + out_params = list(ev.get("inplace_params") or []) + evidence = "observed_inplace" + elif isinstance(ev, dict) and ev.get("returns_none") is not None: + returns_none = bool(ev.get("returns_none")) + evidence = "observed_return" + else: + evidence = "name_convention" if out_params else "none" + + # Inputs the callable reads that do NOT arrive through its parameters (forward-context, layer + # registries, module globals). A non-empty list means the seam is NOT a pure function of its + # arguments, so an out-of-tree rewrite cannot be given the same inputs -- see check_binding. + hidden_ctx = list((meta.get("seam_runtime_evidence") or {}).get("hidden_context") or []) + + return { + "contract": "binding", "contract_version": CONTRACT_VERSION, "ok": True, + "seam": spec, "qualname": res.get("qualname"), "resolved_file": res.get("file"), + "params": params, "required_positional": required_positional, + "accepts_varargs": accepts_varargs, "accepts_varkw": accepts_varkw, + "arity_required": len(required_positional), "arity_total": len(params), + "returns_annotation": ret_ann, "returns_none": returns_none, + "out_params": out_params, "out_params_evidence": evidence, + "hidden_context": hidden_ctx, + "signature": f"{res.get('qualname')}{sig}", + } + + +def _ann_str(ann): + if ann is None: + return "None" + if isinstance(ann, str): + return ann + return getattr(ann, "__name__", None) or str(ann) + + +def check_binding(descriptor, candidate): + """Can `candidate` be bound AT the seam described by `descriptor`? + + `candidate` may be a module:attr spec, an already-built descriptor, or a dict + {params, returns_none, ...}. Mismatch codes are a closed set so the orchestrator can route on them + instead of pattern-matching prose. + """ + if not descriptor or not descriptor.get("ok"): + return {"contract": "binding_check", "contract_version": CONTRACT_VERSION, "bindable": False, + "seam": (descriptor or {}).get("seam", ""), "candidate": str(candidate), + "mismatches": [{"code": "no_seam_descriptor", + "detail": (descriptor or {}).get("error", "seam was never described")}], + "codes": ["no_seam_descriptor"]} + + if isinstance(candidate, str): + cand = describe_binding(candidate) + if not cand.get("ok"): + return {"contract": "binding_check", "contract_version": CONTRACT_VERSION, "bindable": False, + "seam": descriptor.get("seam"), "candidate": candidate, + "mismatches": [{"code": "candidate_unresolvable", "detail": cand.get("error")}], + "codes": ["candidate_unresolvable"]} + else: + cand = dict(candidate or {}) + + m = [] + live_req = list(descriptor.get("required_positional") or []) + cand_req = list(cand.get("required_positional") or []) + cand_varargs = bool(cand.get("accepts_varargs")) + cand_varkw = bool(cand.get("accepts_varkw")) + + # Arity. A single opaque `args`/`kwargs`-style parameter standing in for N live tensors is the + # classic out-of-tree rewrite that can never be rebound; it shows up here as an arity mismatch. + if not cand_varargs and len(cand_req) != len(live_req): + m.append({"code": "arity_mismatch", + "detail": f"seam requires {len(live_req)} positional arg(s) {live_req}; " + f"candidate entry requires {len(cand_req)} {cand_req}"}) + + # Names. The overlay rebinds by NAME at keyword call sites, so a renamed required parameter is a + # hard break even when the arity lines up. + if not (cand_varargs and cand_varkw): + missing = [n for n in live_req if n not in [p["name"] for p in (cand.get("params") or [])]] + if missing and not cand_varkw: + m.append({"code": "param_name_mismatch", + "detail": f"seam parameter(s) {missing} absent from the candidate entry"}) + + # Return contract. A seam that writes into a caller-owned buffer and returns None cannot be + # replaced by something that allocates and returns a fresh tensor -- the caller never reads it. + live_inplace = bool(descriptor.get("out_params")) or bool(descriptor.get("returns_none")) + cand_inplace = bool(cand.get("out_params")) or bool(cand.get("returns_none")) + if live_inplace and not cand_inplace: + m.append({"code": "return_contract_mismatch", + "detail": f"seam writes in place (out_params={descriptor.get('out_params')}, " + f"returns_none={descriptor.get('returns_none')}) but the candidate returns a " + f"fresh value; the live caller would discard the result"}) + if cand_inplace and not live_inplace: + m.append({"code": "return_contract_mismatch", + "detail": "candidate writes in place but the seam's callers consume a returned value"}) + + # Hidden context. If the seam reads state that never crosses the parameter boundary, an authored + # replacement cannot be handed the same inputs -- authoring it is wasted budget regardless of how + # fast it is. This is the check that costs seconds and saves a kernel budget. + if descriptor.get("hidden_context"): + m.append({"code": "hidden_context_inputs", + "detail": f"seam reads non-parameter inputs {descriptor['hidden_context']}; it is not a " + f"pure function of its arguments -- rebind at an inner seam that is"}) + + return {"contract": "binding_check", "contract_version": CONTRACT_VERSION, + "bindable": not m, "seam": descriptor.get("seam"), + "candidate": cand.get("seam") or cand.get("qualname") or "", + "mismatches": m, "codes": [x["code"] for x in m]} + + +def render_entry(descriptor, entry_name="entry"): + """Emit the unittest entry contract FROM the live signature, so an authored kernel is written + against the real call shape instead of one the extractor invented. Generated, never hand-written: + that is what makes 'signature mismatch' structurally impossible rather than merely detected.""" + if not descriptor or not descriptor.get("ok"): + raise ValueError("cannot render an entry from a failed binding descriptor") + parts = [] + for p in descriptor.get("params") or []: + k, n = p["kind"], p["name"] + if k == "VAR_POSITIONAL": + parts.append(f"*{n}") + elif k == "VAR_KEYWORD": + parts.append(f"**{n}") + elif p["has_default"]: + parts.append(f"{n}=None") + else: + parts.append(n) + has_star = any(p["kind"] == "VAR_POSITIONAL" for p in descriptor.get("params") or []) + kwonly = [p["name"] for p in (descriptor.get("params") or []) if p["kind"] == "KEYWORD_ONLY"] + if kwonly and not has_star: + i = min(parts.index(n) if n in parts else parts.index(f"{n}=None") for n in kwonly) + parts.insert(i, "*") + sig = ", ".join(parts) + out = descriptor.get("out_params") or [] + ret = (" # The live seam writes into %s and returns None -- write there, return None.\n" + " raise NotImplementedError\n" % (out,)) if (out or descriptor.get("returns_none")) else \ + " # The live seam returns its result -- return it.\n raise NotImplementedError\n" + return ( + '"""AUTO-GENERATED from the live seam by seam_contract.render_entry -- DO NOT EDIT BY HAND.\n' + f'Seam: {descriptor.get("seam")}\n' + f'Live signature: {descriptor.get("signature")}\n' + 'Any authored kernel MUST implement exactly this contract; the overlay rebinds this name.\n' + '"""\n\n' + f"CONTRACT_SEAM = {descriptor.get('seam')!r}\n" + f"CONTRACT_VERSION = {CONTRACT_VERSION}\n\n\n" + f"def {entry_name}({sig}):\n{ret}") + + +# ------------------------------------------------------------------------------------------- CLI +def _load_meta(task_dir): + p = os.path.join(task_dir, "meta.json") + if not os.path.exists(p): + return {}, f"no meta.json in {task_dir}" + try: + with open(p) as fh: + return json.load(fh), None + except Exception as e: # noqa: BLE001 + return {}, f"meta.json unreadable: {e!r}" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--task-dir", default="", help="op task dir containing meta.json") + ap.add_argument("--eval-dir", default="", help="run EVAL_DIR (also disqualified as a baseline origin)") + ap.add_argument("--spec", default="", help="module:attr to describe (overrides meta.target_callable)") + ap.add_argument("--mode", default="both", choices=["baseline", "binding", "both", "entry"]) + ap.add_argument("--candidate", default="", help="module:attr of the authored entry, for --mode binding") + ap.add_argument("--entry-name", default="entry") + ap.add_argument("--out", default="", help="write the rendered entry contract here (--mode entry)") + ap.add_argument("--json", action="store_true", help="print the verdict as JSON (default)") + ap.add_argument("--site-root", action="append", default=[], + help="extra install root to count as 'installed' (overlay/editable/vendored); repeatable") + args = ap.parse_args(argv) + + if args.site_root: + prev = os.environ.get("GEAK_SEAM_SITE_DIRS", "") + os.environ["GEAK_SEAM_SITE_DIRS"] = os.pathsep.join([p for p in ([prev] + args.site_root) if p]) + + meta, meta_err = ({}, None) + if args.task_dir: + meta, meta_err = _load_meta(args.task_dir) + + result = {"contract_version": CONTRACT_VERSION, "task_dir": args.task_dir or None} + if meta_err: + result["meta_error"] = meta_err + + if args.mode in ("baseline", "both"): + result["baseline_validation"] = validate_baseline(args.task_dir or None, meta, args.eval_dir or None) + + if args.mode in ("binding", "both", "entry"): + spec = args.spec or (meta.get("target_callable") or "") + desc = describe_binding(spec, meta) if spec else { + "contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": "", "error": "no target_callable / --spec given"} + result["binding_descriptor"] = desc + if args.candidate: + result["binding_check"] = check_binding(desc, args.candidate) + if args.mode == "entry": + if not desc.get("ok"): + result["entry_error"] = desc.get("error") + else: + src = render_entry(desc, args.entry_name) + result["entry_contract"] = src + if args.out: + with open(args.out, "w") as fh: + fh.write(src) + result["entry_path"] = os.path.realpath(args.out) + + ok = True + if "baseline_validation" in result: + ok = ok and bool(result["baseline_validation"].get("ok")) + if args.mode in ("binding", "entry") or ("binding_check" in result): + ok = ok and bool(result.get("binding_descriptor", {}).get("ok")) + if "binding_check" in result: + ok = ok and bool(result["binding_check"].get("bindable")) + result["ok"] = ok + + print(json.dumps(result, indent=2, default=str)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) From d20e826af1759446fe73c01f10febfd070663823 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Aug 2026 14:06:42 +0000 Subject: [PATCH 2/8] harden deployability contracts and add regression coverage --- e2e_workflow/e2e_workflow.js | 175 ++++++-- e2e_workflow/roles/e2e_integrator.md | 2 +- e2e_workflow/roles/kernel_extractor.md | 49 ++- e2e_workflow/roles/profiler.md | 9 +- e2e_workflow/scripts/op_bench.py | 126 +++++- e2e_workflow/scripts/parse_profile.py | 84 +++- e2e_workflow/scripts/seam_contract.py | 183 ++++++-- e2e_workflow/scripts/tests/test_op_bench.py | 403 +++++++++++++++++- .../scripts/tests/test_parse_profile.py | 76 +++- .../scripts/tests/test_seam_contract.py | 311 ++++++++++++++ 10 files changed, 1300 insertions(+), 118 deletions(-) create mode 100644 e2e_workflow/scripts/tests/test_seam_contract.py diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index ea764b468..262141d8c 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -251,7 +251,10 @@ const REJECT_CODES = { arity_mismatch: { cls: 'integration', stage: 'extract' }, param_name_mismatch: { cls: 'integration', stage: 'extract' }, return_contract_mismatch: { cls: 'integration', stage: 'extract' }, + optional_param_dropped: { cls: 'integration', stage: 'extract' }, + param_kind_mismatch: { cls: 'integration', stage: 'extract' }, hidden_context_inputs: { cls: 'integration', stage: 'extract' }, + seam_mismatch: { cls: 'integration', stage: 'extract' }, candidate_unresolvable: { cls: 'integration', stage: 'extract' }, no_seam_descriptor: { cls: 'integration', stage: 'extract' }, no_engagement: { cls: 'integration', stage: 'extract' }, @@ -633,7 +636,8 @@ const INTEGRATE_SCHEMA = obj({ // human-readable detail. The enum is the closed set in REJECT_CODES. reason_code: { type: 'string', enum: [ 'no_rebind_seam', 'signature_mismatch', 'arity_mismatch', 'param_name_mismatch', - 'return_contract_mismatch', 'hidden_context_inputs', 'no_engagement', 'wrong_seam', + 'return_contract_mismatch', 'optional_param_dropped', 'param_kind_mismatch', + 'hidden_context_inputs', 'seam_mismatch', 'no_engagement', 'wrong_seam', 'invalid_denominator', 'cuda_graph_capture_unsafe', 'no_binary_for_gpu', 'capture_hang', 'host_sync_in_hot_path', 'oom', 'parity_regression', 'accuracy_regression', 'output_corruption', 'implausible_speedup', 'wrong_head_granularity', 'delegated_track_disabled', 'no_win', 'do_no_harm'] }, @@ -845,11 +849,22 @@ const BASELINE_CONTRACT_STRICT = String(A.baseline_contract_strict != null ? A.b // reference_io_sha256) and were, until now, read by nothing — see scripts/check_schema_consumption.py. function oracleProvenance(ext) { const problems = []; - if (ext.num_cases != null && Number(ext.num_cases) <= 0) - problems.push('num_cases=0 (the oracle recorded no calls, so correctness is unfalsifiable)'); - if (typeof ext.reference_io_sha256 === 'string' && ext.reference_io_sha256.trim() === '' - && ext.synthesized !== true) - problems.push('reference_io_sha256 empty (the oracle bytes are unpinned; tampering is undetectable)'); + // A fabricated oracle (synthesized:true) is handled as its OWN case downstream (baselineDefect -> + // 'synthesized': the RATIO is withheld, but the kernel can still be accepted on a measured e2e A/B — + // ten such wins live in the 88-run archive). It has no captured live calls to count or bytes to pin, + // so the num_cases/sha256 provenance requirements do not apply to it. + const synth = ext.synthesized === true; + // A claimed captured oracle must carry complete identity evidence. Partial/failed extractions never + // reach this acceptance predicate, so missing fields must fail closed here rather than be treated as + // harmless schema degradation. op_bench.py independently recomputes the digest from the bytes. + if (!synth) { + if (!Number.isFinite(Number(ext.num_cases)) || Number(ext.num_cases) <= 0) + problems.push('num_cases missing/zero (the oracle recorded no provable calls)'); + const sha = typeof ext.reference_io_sha256 === 'string' + ? ext.reference_io_sha256.trim().toLowerCase() : ''; + if (!/^[0-9a-f]{64}$/.test(sha)) + problems.push('reference_io_sha256 missing/invalid (oracle bytes are not identity-pinned)'); + } return { ok: problems.length === 0, problems }; } @@ -858,14 +873,10 @@ function oracleProvenance(ext) { // nowhere: an agent could truthfully report provenance_ok=false and still have its speedup banked. // An explicit false is now disqualifying; an absent value is a recorded degradation, not a pass. function provenanceOk(res, where) { - if (!res || typeof res !== 'object') return true; - if (res.provenance_ok === false) { - log(` ⚠️ ${where}: provenance_ok=false — the agent states its own numbers are unsourced; not banking them`); + if (!res || typeof res !== 'object' || res.provenance_ok !== true) { + log(` ⚠️ ${where}: provenance_ok is not true — the number is unsourced; not banking it`); return false; } - if (res.provenance_ok == null) - noteDegradation(where, 'result carries no provenance_ok', - 'cannot tell whether the oracle/baseline was re-verified before the number was produced'); return true; } @@ -883,9 +894,8 @@ function denominatorSound(bake, where) { } const d = bake.denominator; if (d == null) { - noteDegradation(where, 'bake-off result declares no denominator', - 'cannot tell whether the speedup was measured against the live path or against a fabricated reference'); - return true; // older role revision: degrade loudly, do not silently drop a real win + log(` ⚠️ ${where}: bake-off result declares no denominator — not banking an unauditable ratio`); + return false; } if (!BANKABLE_DENOMINATORS.includes(d)) { log(` ⚠️ ${where}: denominator='${d}' is not the live path — the ratio is unbankable`); @@ -924,7 +934,21 @@ const hasFrozenBaseline = (ext) => { } const v = ext.baseline_validation; if (v && typeof v === 'object' && v.contract === 'baseline_identity') { - if (v.ok === true) return true; + if (v.ok === true) { + // C5: the verdict certifies the spec IT names. A stale/foreign verdict (from a prior retry or + // another task) whose callable differs must NOT certify this baseline. Cross-check the callables. + const vb = String(v.baseline_callable || '').trim(), eb = String(ext.baseline_callable || '').trim(); + const vt = String(v.target_callable || '').trim(), et = String(ext.target_callable || '').trim(); + if (!vb || !eb || vb !== eb) { + log(` [inv-1] baseline_validation ok=true but certifies baseline '${vb}', not this task's '${eb}' — stale/foreign verdict, not a frozen baseline`); + return false; + } + if (!vt || !et || vt !== et) { + log(` [inv-1] baseline_validation ok=true but certifies target '${vt}', not this task's '${et}' — stale/foreign verdict, not a frozen baseline`); + return false; + } + return true; + } log(` [inv-1] baseline_validation FAILED: ${(v.failed || []).join(', ') || 'unknown'} ` + `(baseline_callable=${v.baseline_callable || "''"}, origin=${(v.baseline_origin || {}).kind || '?'})`); return false; @@ -953,27 +977,58 @@ const hasFrozenBaseline = (ext) => { // matter how fast it is, and the reject only surfaces hours later at the e2e gate. // Strict by default; seam_contract_strict=false degrades to "unverified but allowed" and records it. const SEAM_CONTRACT_STRICT = String(A.seam_contract_strict != null ? A.seam_contract_strict : 'true') === 'true'; -function seamBindable(ext) { +function seamBindable(ext, seam) { if (!ext) return { ok: false, why: 'no extraction' }; const chk = ext.binding_check, desc = ext.binding_descriptor; + // C5: a binding verdict is only valid for the seam it was computed against. seam_contract.py records + // that seam on both binding_check and binding_descriptor (field `seam`). If the selected deployment + // seam differs, the verdict is stale/foreign and must not admit the head. + const want = String(seam || '').trim(); + const seamMismatch = (obj, kind) => { + // Pre-selection contract-PRESENCE checks (extractWithBaseline's re-extract loop) call seamBindable + // with no seam yet: there is no selected seam to match against, so skip the seam check here and only + // verify a usable binding contract exists. The real admission gate (authoringAdmission) always + // passes the selected seam, so C5's stale/foreign-verdict rejection still fires there. Without this + // guard an empty `want` made every valid extraction look seam-mismatched, forcing pointless + // re-extractions with an unsatisfiable "not the selected seam ''" corrective. + if (!want) return null; + const got = String((obj && obj.seam) || '').trim(); + if (!got || got !== want) + return { ok: false, why: `${kind} certifies seam '${got}', not the selected seam '${want}'`, + codes: ['seam_mismatch'] }; + return null; + }; + // A binding_check cannot replace the descriptor: hidden-context/purity evidence belongs to the live + // seam descriptor and must be present even when the generated entry's signature is bindable. + if (!desc || typeof desc !== 'object' || desc.contract !== 'binding' || desc.ok !== true) { + if (SEAM_CONTRACT_STRICT) + return { ok: false, why: 'missing/unusable binding_descriptor', codes: ['no_seam_descriptor'] }; + } else { + const dm = seamMismatch(desc, 'binding_descriptor'); if (dm) return dm; + if (desc.hidden_context_evidence !== 'declared') + return { ok: false, why: 'hidden-context purity evidence is missing', + codes: ['hidden_context_inputs'] }; + if ((desc.hidden_context || []).length) + return { ok: false, why: `seam reads non-parameter inputs ${JSON.stringify(desc.hidden_context)}`, + codes: ['hidden_context_inputs'] }; + } if (chk && typeof chk === 'object' && chk.contract === 'binding_check') { + const mm = seamMismatch(chk, 'binding_check'); if (mm) return mm; + const entryPath = typeof ext.entry_contract_path === 'string' ? ext.entry_contract_path.trim() : ''; return chk.bindable === true - ? { ok: true, why: 'binding_check.bindable' } + ? { ok: true, why: `binding_check.bindable${entryPath ? `; generated contract=${entryPath}` : ''}` } : { ok: false, why: `binding_check: ${(chk.codes || []).join(', ') || 'not bindable'}`, codes: chk.codes || [] }; } + if (chk && SEAM_CONTRACT_STRICT) + return { ok: false, why: 'malformed binding_check', codes: ['no_seam_descriptor'] }; if (desc && typeof desc === 'object' && desc.contract === 'binding') { if (desc.ok !== true) return { ok: false, why: `binding_descriptor unusable: ${desc.error || 'unknown'}` }; - if ((desc.hidden_context || []).length) - return { ok: false, why: `seam reads non-parameter inputs ${JSON.stringify(desc.hidden_context)}`, - codes: ['hidden_context_inputs'] }; - // entry_contract_path names the stub seam_contract.py --mode entry GENERATED from the live - // signature. Its presence is the difference between an entry derived from the seam and one the - // author invented, so it upgrades a descriptor-only verdict from "unchecked" to "checked by - // construction". (Also a declared schema field that nothing read before.) - if (typeof ext.entry_contract_path === 'string' && ext.entry_contract_path.trim() !== '') - return { ok: true, why: `seam described; entry generated from it (${ext.entry_contract_path})` }; - return { ok: true, why: 'seam described; no entry to check yet' }; + const mm = seamMismatch(desc, 'binding_descriptor'); if (mm) return mm; + if (SEAM_CONTRACT_STRICT) + return { ok: false, why: 'binding_check missing; generated entry was not verified', + codes: ['no_seam_descriptor'] }; + return { ok: true, why: 'legacy mode: seam described but generated entry unchecked' }; } noteDegradation('seamBindable', 'extraction returned no binding_descriptor (seam_contract.py not run)', SEAM_CONTRACT_STRICT @@ -983,6 +1038,46 @@ function seamBindable(ext) { return { ok: !SEAM_CONTRACT_STRICT, why: 'no binding contract (unverified)' }; } +// ---- INV-6: ENTITY-KIND ADMISSION (applied at EVERY headQueue (re)assignment) --------------------- +// The head track optimizes GPU KERNELS. A torch DISPATCHER op is a Python-side span whose pct_gpu_time +// is the SUM of the kernels it dispatches; routing one as a head double-counts its Amdahl share and +// points extraction at a wrapper. The filter used to run ONCE, right after the initial strategy — but +// headQueue is ALSO (re)assigned from carried state on a phase resume and from the post-config +// re-strategize, and neither of those was re-admitted. A dispatcher_op arriving by either path reached +// the author fan-out unchecked. admitHeads() is now called at all three sites. +// A TYPE check, not a name blacklist: it needs no aten::/vllm:: prefix knowledge. +const HEAD_ENTITY_KIND_STRICT = String(A.head_entity_kind_strict != null ? A.head_entity_kind_strict : 'true') === 'true'; +function admitHeads(q, stage, flagged) { + q = (q || []).filter(Boolean); + const bad = q.filter((c) => c.entity_kind && c.entity_kind !== 'gpu_kernel'); + const noKind = q.filter((c) => !c.entity_kind); + let admitted = q.filter((c) => !(c.entity_kind && c.entity_kind !== 'gpu_kernel')); + for (const c of bad) { + log(` ⚠️ FLAG ${c.short_name || c.name}: profile row is a ${c.entity_kind}, not a gpu_kernel ` + + `(${c.pct_gpu_time || '?'}% is the sum of the kernels it dispatches) — NOT routed to the head ` + + `track (${stage}); the Architect must name the underlying kernel instead.`); + flagged.push({ short_name: c.short_name || c.name, pct_gpu_time: c.pct_gpu_time, + stage, gate: 'wrong_head_granularity', reason_code: 'wrong_head_granularity', + reason: `entity_kind=${c.entity_kind}; head track requires gpu_kernel` }); + } + if (noKind.length) { + if (HEAD_ENTITY_KIND_STRICT) { + admitted = admitted.filter((c) => c.entity_kind); + for (const c of noKind) { + log(` ⚠️ FLAG ${c.short_name || c.name}: profile row carries no entity_kind — ` + + `head_entity_kind_strict=true rejects it (cannot confirm it is a gpu_kernel).`); + flagged.push({ short_name: c.short_name || c.name, pct_gpu_time: c.pct_gpu_time, + stage, gate: 'wrong_head_granularity', reason_code: 'wrong_head_granularity', + reason: 'entity_kind missing; head track requires a confirmed gpu_kernel (strict)' }); + } + } else { + noteDegradation('head-admission', `${noKind.length} head candidate(s) carry no entity_kind at ${stage}`, + 'profile rows predate the entity_kind contract — a dispatcher op could still be routed as a kernel head'); + } + } + return admitted; +} + // ---- INV-2/INV-3: AUTHORING ADMISSION ------------------------------------------------------------- // The ONLY precondition on spending the author budget used to be `isolated_speedup > 1.0` — a number // produced INSIDE the task dir, which says nothing about whether the result can ever reach the live @@ -1007,7 +1102,7 @@ function authoringAdmission(h, ext) { noteDegradation('authoringAdmission', `${(h && h.short_name) || seam}: authoring a head whose oracle is synthesized`, 'admitted on the strength of the seam alone; its isolated speedup is unbankable and only a ' + 'measured e2e A/B can accept it'); - const b = seamBindable(ext); + const b = seamBindable(ext, seam); // C5: verify the binding verdict is for THIS seam, not a stale one if (!b.ok) { const c = (b.codes || []).find((x) => REJECT_CODES[x]) || 'signature_mismatch'; return { ok: false, reason_code: c, why: b.why }; @@ -1627,21 +1722,7 @@ if (want('setup')) { // the device. parse_profile.py now labels every row `entity_kind`; this admits only gpu_kernel rows. // Note this is a TYPE check, not a name blacklist: it does not need to recognize `aten::`/`vllm::` // prefixes, or any future naming convention, to keep a dispatcher span out of the kernel track. - const _dispatcherHeads = headQueue.filter((c) => c && c.entity_kind && c.entity_kind !== 'gpu_kernel'); - if (_dispatcherHeads.length) { - headQueue = headQueue.filter((c) => !(c && c.entity_kind && c.entity_kind !== 'gpu_kernel')); - for (const c of _dispatcherHeads) { - log(` ⚠️ FLAG ${c.short_name || c.name}: profile row is a ${c.entity_kind}, not a gpu_kernel ` + - `(${c.pct_gpu_time || '?'}% is the sum of the kernels it dispatches) — NOT routed to the head ` + - `track; the Architect must name the underlying kernel instead.`); - PRE_FLAGGED_HEADS.push({ short_name: c.short_name || c.name, pct_gpu_time: c.pct_gpu_time, - stage: 'strategize', gate: 'wrong_head_granularity', reason_code: 'wrong_head_granularity', - reason: `entity_kind=${c.entity_kind}; head track requires gpu_kernel` }); - } - } - if (headQueue.some((c) => c && !c.entity_kind)) - noteDegradation('head-admission', 'some head candidates carry no entity_kind', - 'profile rows predate the entity_kind contract — a dispatcher op could still be routed as a kernel head'); + headQueue = admitHeads(headQueue, 'strategize', PRE_FLAGGED_HEADS); log(`Strategy: ${headQueue.length} head candidates, ${kernelQueue.length} kernel candidates, ${(strategy && strategy.config_directions || []).length} config directions.`); // strategize decided the backends -> if any candidate routed flydsl, provision it now (blocking). await ensureFlydslGate(); @@ -1657,7 +1738,7 @@ if (want('setup')) { profile = { profile_topN_json: ST.profile_topn_json || '' }; strategy = { config_directions: ST.config_directions || [] }; kernelQueue = ST.kernelQueue || []; - headQueue = ST.headQueue || []; + headQueue = admitHeads(ST.headQueue || [], 'resume', PRE_FLAGGED_HEADS); // C8: re-admit on resume log(`Loaded carried state: EVAL_DIR=${EVAL_DIR}, baseline ${BASELINE_TPUT}, flags='${curFlags}', env='${curEnv}', ${headQueue.length} head + ${kernelQueue.length} kernel candidates.`); } @@ -1696,7 +1777,8 @@ if (want('config') && CONFIG_TUNE_ENABLED && strategy && (strategy.config_direct }), { phase: 'Strategize', label: 'architect:re-strategize', schema: STRATEGY_SCHEMA }); if (restrat && restrat.kernel_candidates) kernelQueue = restrat.kernel_candidates.slice(); - if (restrat && restrat.head_candidates) headQueue = restrat.head_candidates.slice(); + if (restrat && restrat.head_candidates) + headQueue = admitHeads(restrat.head_candidates.slice(), 're-strategize', PRE_FLAGGED_HEADS); // C8 // re-strategize may have (re)routed flydsl -> provision it (idempotent; no-op if already done). await ensureFlydslGate(); } else { @@ -3158,6 +3240,7 @@ const wfReturn = { // unaudited. Silence used to be indistinguishable from enforcement. degradations: DEGRADATIONS, contracts: { baseline_contract_strict: BASELINE_CONTRACT_STRICT, seam_contract_strict: SEAM_CONTRACT_STRICT, + head_entity_kind_strict: HEAD_ENTITY_KIND_STRICT, head_reextract_max: HEAD_REEXTRACT_MAX }, config_tune_enabled: CONFIG_TUNE_ENABLED, head_budget: HEAD_BUDGET, diff --git a/e2e_workflow/roles/e2e_integrator.md b/e2e_workflow/roles/e2e_integrator.md index e491cdc5f..050f6267e 100644 --- a/e2e_workflow/roles/e2e_integrator.md +++ b/e2e_workflow/roles/e2e_integrator.md @@ -65,7 +65,7 @@ rule first, so a seam defect was retried as a numerics defect for hours.) | owning stage | `reason_code` | means | |---|---|---| -| **extract** (the TASK encodes the wrong seam/contract/denominator — re-extraction, not re-authoring) | `no_rebind_seam`, `signature_mismatch`, `arity_mismatch`, `param_name_mismatch`, `return_contract_mismatch`, `hidden_context_inputs`, `candidate_unresolvable`, `no_seam_descriptor`, `no_engagement`, `wrong_seam`, `invalid_denominator` | the kernel may be perfect; it cannot be bound where the server actually calls, or its speedup was measured against something that is not the live path | +| **extract** (the TASK encodes the wrong seam/contract/denominator — re-extraction, not re-authoring) | `no_rebind_seam`, `signature_mismatch`, `arity_mismatch`, `param_name_mismatch`, `optional_param_dropped`, `param_kind_mismatch`, `return_contract_mismatch`, `seam_mismatch`, `hidden_context_inputs`, `candidate_unresolvable`, `no_seam_descriptor`, `no_engagement`, `wrong_seam`, `invalid_denominator` | the kernel may be perfect; it cannot be bound where the server actually calls, or its speedup was measured against something that is not the live path | | **author** (the seam is right; posture or numerics are wrong) | `cuda_graph_capture_unsafe`, `no_binary_for_gpu`, `capture_hang`, `host_sync_in_hot_path`, `oom`, `parity_regression`, `accuracy_regression`, `output_corruption`, `implausible_speedup` | re-authoring on the SAME task dir can fix it | | **upstream** (no amount of kernel work fixes it) | `wrong_head_granularity`, `delegated_track_disabled` | | | **terminal** (not a defect) | `no_win`, `do_no_harm` | a correct kernel with no headroom | diff --git a/e2e_workflow/roles/kernel_extractor.md b/e2e_workflow/roles/kernel_extractor.md index 9da5741b7..b8ea9e259 100644 --- a/e2e_workflow/roles/kernel_extractor.md +++ b/e2e_workflow/roles/kernel_extractor.md @@ -105,7 +105,10 @@ freeze an out-of-regime oracle nobody should trust. set). Then set **BOTH** `target_callable` **and** `meta.baseline_callable` to that launcher, so the authored kernel and the speedup denominator are the same live seam. Generate its deployable entry with `python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --mode entry` — never hand-write it — - and verify with `--mode both` that `baseline_validation.ok` and `binding_check.bindable` are BOTH true. + and verify with `--mode both` that `baseline_validation.ok`, `binding_descriptor.ok`, AND + `binding_check.bindable` are ALL true. With no `--candidate`, `--mode both` checks the entry it just + rendered against its own descriptor, so a `bindable:false` here means the GENERATED entry does not fit + the live seam — regenerate it, do not paste the failing verdict. A synthesized oracle plus an unbindable wrapper is the single failure mode this case exists to prevent: it yields a kernel-level "speedup" against a number nothing in the server ever computed, and an overlay that patches nothing. If after descending no seam is both capturable and bindable, report @@ -359,7 +362,7 @@ freeze an out-of-regime oracle nobody should trust. > says the function the unittest times must not alias a static buffer across calls. It is NOT licence > to give the authored kernel a dict-taking, fresh-returning entry when the LIVE seam is positional and > writes in place — the deployed entry must match the live seam's real signature (that is what - > `seam_contract.py --mode entry` generates and `--mode bind` checks). An in-place live seam is served + > `seam_contract.py --mode entry` generates and `--mode binding` checks). An in-place live seam is served > by an entry that writes into the caller's `out` and returns what the live seam returns; the harness > still gets its fresh-output wrapper around it. 5. **Finalize `meta.json`**: set `build` (false for pure-Triton; true + a build cmd for HIP/CK/asm @@ -392,33 +395,57 @@ freeze an out-of-regime oracle nobody should trust. > genuine baseline-bind / correctness failure (exit 1). Only after 3 failed regenerations set > `unittest_smoke:"fail"` with `reason="harness_incomplete_unrecoverable"`. 7. **🔴 MANDATORY — machine-check the baseline and the seam binding. Do not hand-write these verdicts.** - The orchestrator gates head admission on the OUTPUT of this script, not on your prose. Run it and - paste the three objects it prints back VERBATIM into your Return JSON: + First write/reconcile `/meta.json` with the final `baseline_callable`, `target_callable`, + `baseline_origin`, `baseline_capture_evidence`, and complete `seam_runtime_evidence` (including an + explicit `hidden_context`, even when it is `[]`). The validator consumes that file; running it before + the evidence is persisted is not a valid check. Then run it and paste the three objects it prints back + VERBATIM into your Return JSON: ```bash python3 "$SKILL_DIR/scripts/seam_contract.py" \ --task-dir "" --eval-dir "$EVAL_DIR" \ - --spec "" \ + --baseline-spec "" \ + --target-spec "" \ --mode both --json ``` - - It imports the spec from the LIVE site-packages, checks the module file actually lives under a real + **🔴 `--baseline-spec` and `--target-spec` are DISTINCT seams — do not conflate them.** `--baseline-spec` + is the callable whose numbers you validate against; `--target-spec` is the live seam the entry must + rebind, and it is what `binding_descriptor`/`binding_check` are built from. The old single `--spec` is a + DEPRECATED alias for `--target-spec` only: passing the baseline through it makes the binding descriptor + describe the baseline, not the seam you must rebind, so a mismatched candidate passes silently. If you + omit either flag the script falls back to `meta.baseline_callable`/`meta.target_callable`; the explicit + flags prevent accidental spec substitution, but they do not replace the evidence that must already be + present in meta.json. + - It imports each spec from the LIVE site-packages, checks the module file actually lives under a real install root (not a directory you just created), and rejects a synthesized `baseline_src.*` strawman. `baseline_validation.ok=false` ⇒ the head is NOT admissible: return `editable:false` / `baseline_frozen:false` with the reported reason. Do not "fix" it by pointing at a file you wrote. - `binding_descriptor` is DERIVED from `inspect.signature` of the live callable — parameter names, - kinds, defaults, and which parameters are written in place. It is a fact about the seam, not a - guess; never edit it by hand. + kinds, and defaults. It is a fact about the seam, not a guess; never edit it by hand. + **`inspect.signature` sees ONLY the declared parameters.** It CANNOT see what a callable reads that + is not a parameter: module globals, forward/attention context, registries, env, captured closure + state, or which parameters it writes in place. Those are NOT inspect-derived — they come from + `seam_runtime_evidence` below, which YOU must fill from reading the source, and the descriptor + merely copies them through. A seam that reads hidden state is not a pure function of its arguments + and cannot be soundly rebound as if it were. - `binding_check` compares your candidate entry point against that descriptor. If it fails, the candidate cannot be rebound at the seam and the isolated speedup is unbankable — regenerate the entry from the descriptor instead of arguing with it: ```bash python3 "$SKILL_DIR/scripts/seam_contract.py" --task-dir "" --eval-dir "$EVAL_DIR" \ - --spec "" --mode entry --entry-name --out "/entry_contract.py" + --target-spec "" --mode entry --entry-name --out "/entry_contract.py" ``` and report that path as `entry_contract_path`. - `seam_runtime_evidence.inplace_params` MUST list the parameter names the live callable writes through (e.g. an `output=` buffer). `op_bench.py` uses this list — and ONLY this list, never a name heuristic — to decide where a replayed in-place seam's result is read from. Getting it wrong makes correctness unmeasurable, not merely inaccurate. + - `seam_runtime_evidence.hidden_context` MUST list EVERY non-parameter input the live callable reads: + module globals, forward/attention context, registry lookups, env, and captured closure state. This + is NOT discoverable from `inspect.signature` — you establish it by reading the source. An omitted + `hidden_context` is `unknown` and fails the strict binding gate; it is never inferred as `[]`. Do not + declare an empty list unless you verified that the seam is pure. If a seam depends on hidden context + it cannot be rebound at the seam by arguments alone: report it (so the entry can supply that context, + or the head is dropped) rather than silently admitting an impure seam as pure. - `num_cases` MUST be the real number of records in `reference_io.pt`. `0` means the oracle recorded no calls, so correctness is unfalsifiable, and the orchestrator will reject the head. Report the true count; do not round it up. @@ -440,7 +467,7 @@ Return JSON: "binding_descriptor": { "...": "verbatim from seam_contract.py --mode both" }, "binding_check": { "...": "verbatim from seam_contract.py --mode both" }, "entry_contract_path": "/entry_contract.py or \"\" if the candidate already matches", - "seam_runtime_evidence": { "inplace_params": ["output"] }, + "seam_runtime_evidence": { "inplace_params": ["output"], "returns_none": true, "hidden_context": [] }, "num_cases": 0, "regimes_captured": ["prefill","decode"], "candidate_backends": ["triton","hip","ck"], @@ -809,7 +836,7 @@ Return JSON: "binding_descriptor": { "...": "verbatim from seam_contract.py --mode both" }, "binding_check": { "...": "verbatim from seam_contract.py --mode both" }, "entry_contract_path": "/entry_contract.py or \"\"", - "seam_runtime_evidence": { "inplace_params": [] }, + "seam_runtime_evidence": { "inplace_params": [], "returns_none": false, "hidden_context": [] }, "smoke": "pass|fail", "notes": "transpose/bias inference, regime, whether oracle was synthesized vs captured" } diff --git a/e2e_workflow/roles/profiler.md b/e2e_workflow/roles/profiler.md index 3b8715dbb..392cea62b 100644 --- a/e2e_workflow/roles/profiler.md +++ b/e2e_workflow/roles/profiler.md @@ -103,6 +103,10 @@ An upstream orchestrator may already have profiled the SAME baseline workload wi `profile_topN.json` + `.md` via your own Write and set `source:"tracelens"`. **🔴 Then you MUST annotate the hand-assembled rows — do not hand-write `entity_kind`:** ```bash + # Select the trace to cross-check against FIRST — the annotate command below needs $TLT set. Prefer + # the top-level rank0 serving trace; never recurse into capture_traces/. + TLT=$(ls -1 "$TRACELENS_TRACE_FILE"/*rank0*.pt.trace.json.gz 2>/dev/null | head -1) + [ -z "$TLT" ] && TLT=$(ls -1 "$TRACELENS_TRACE_FILE"/*.pt.trace.json.gz "$TRACELENS_TRACE_FILE"/*.json.gz "$TRACELENS_TRACE_FILE"/*.json 2>/dev/null | head -1) python3 "$EVAL_DIR/parse_profile.py" --annotate "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" \ --torch-trace "$TLT" --annotate-out "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" ``` @@ -130,7 +134,10 @@ An upstream orchestrator may already have profiled the SAME baseline workload wi that matches** (this is the mandatory shape double-check, since `analysis.md` shapes may be inaccurate). Keep the TraceLens ranking/`%gpu` as the primary impact signal, but cross-check that the same heads top both views; note any disagreement in `notes`. Emit the final reconciled `profile_topN.json`/`.md` with - `source:"tracelens+trace"`. + `source:"tracelens+trace"`. **🔴 Reconciliation rewrites `profile_topN.json`, so re-run the `--annotate` + step above on the reconciled file (or carry the `entity_kind`/`entity_evidence`/`entity_kind_*` fields + forward onto it) — the final emitted file MUST still carry the annotator's evidence-backed kinds, never + hand-written ones.** - **If `TRACELENS_ANALYSIS_MD` is empty/missing (or the file does not exist) → ignore TraceLens entirely and run the normal collection (steps 1–5) unchanged.** Likewise, for ANY reprofile round the TraceLens prior is stale (it reflects the baseline config) — ignore it and re-collect. diff --git a/e2e_workflow/scripts/op_bench.py b/e2e_workflow/scripts/op_bench.py index 10cc07142..1868ce527 100644 --- a/e2e_workflow/scripts/op_bench.py +++ b/e2e_workflow/scripts/op_bench.py @@ -24,7 +24,7 @@ Exit 0 always (unless the task dir is unreadable); per-backend failures are captured in the JSON so an unavailable backend on this image is a recorded "skipped", not a crash. """ -import argparse, hashlib, json, math, os, sys, time, traceback +import argparse, hashlib, json, math, os, re, sys, time, traceback # Shared harness measurement library (single source of truth for timing + correctness + Amdahl). # op_bench.py lives in scripts/ so a plain import resolves; keep a guarded fallback so an old @@ -166,6 +166,15 @@ def resolve_denominator(meta): tgt = str(meta.get("target_callable") or "").strip() v = meta.get("baseline_validation") verified = (isinstance(v, dict) and v.get("contract") == "baseline_identity" and v.get("ok") is True) + # C5: a verdict certifies ONLY the spec it names. A stale/foreign verdict (recorded for a + # different callable in a prior retry or another task dir) must not upgrade THIS baseline to + # verified. Require the verdict's baseline_callable to match; if it also names a target, that + # must match too. The verdict carries these fields (seam_contract.py writes them), so the check + # is feasible — it was simply omitted before. Both fields are mandatory for an ok verdict: + # accepting an omitted target would let a verdict from another seam certify this task. + verdict_matches = (verified + and str(v.get("baseline_callable") or "").strip() == bl + and str(v.get("target_callable") or "").strip() == tgt) def out(spec, prov, why): return {"spec": spec, "provenance": prov, "severity": DENOM_SEVERITY[prov], "why": why} @@ -174,9 +183,14 @@ def out(spec, prov, why): return out(bl or tgt, "synthesized_reference", "meta.synthesized=true: the reference was written by the extractor, not captured " "from the live seam, so it is an oracle for CORRECTNESS only, never a perf denominator") - if bl and verified: + if bl and verdict_matches: return out(bl, "verified_baseline", "seam_contract.py --mode baseline reported ok=true for this spec") + if bl and verified and not verdict_matches: + return out(bl, "unverified_baseline", + f"baseline_validation.ok=true but it certifies " + f"{str(v.get('baseline_callable') or '')!r}/{str(v.get('target_callable') or '')!r}, " + f"not this spec {bl!r}/{tgt!r} — a stale or foreign verdict cannot bank a speedup") if bl: return out(bl, "unverified_baseline", "meta.baseline_callable is declared but carries no baseline_validation verdict " @@ -237,12 +251,44 @@ def _has_unreplayable(obj): return False -def load_oracle_records(task, torch, device): +def verify_oracle_sha(task, meta): + """Verify that captured oracle bytes match a complete SHA-256 committed by capture_shapes.py.""" + if meta.get("synthesized") is True: + return True, "" + declared = meta.get("reference_io_sha256") + if not (isinstance(declared, str) and declared.strip()): + return False, "reference_io_sha256 is missing; captured oracle bytes are not identity-pinned" + declared = declared.strip().lower() + if not re.fullmatch(r"[0-9a-f]{64}", declared): + return False, "reference_io_sha256 is not a complete 64-character lowercase hex digest" + iopath = os.path.join(task, "reference_io.pt") + if not os.path.exists(iopath): + return False, "reference_io.pt is missing; cannot verify its declared checksum" + h = hashlib.sha256() + try: + with open(iopath, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + except Exception as e: + return False, f"reference_io.pt unreadable for checksum: {e!r}" + got = h.hexdigest() + if got != declared: + return False, (f"reference_io_sha256 MISMATCH: meta declares {declared[:16]}… but the " + f"oracle bytes hash to {got[:16]}… — the oracle was altered after capture, so its " + f"correctness verdict cannot be trusted") + return True, "" + + +def load_oracle_records(task, torch, device, meta=None): """Return (records, err). Each record: {sig, regime, args, kwargs, output} with tensors on `device`. `err` is a human string when the oracle cannot be replayed; records is then [].""" iopath = os.path.join(task, "reference_io.pt") if not os.path.exists(iopath): return [], "no reference_io.pt in the task dir" + if meta is not None: # C4: reject a tampered/forged oracle before replaying it + sha_ok, sha_err = verify_oracle_sha(task, meta) + if not sha_ok: + return [], sha_err try: blob = torch.load(iopath, map_location="cpu", weights_only=False) except Exception as e: @@ -250,6 +296,16 @@ def load_oracle_records(task, torch, device): recs = blob.get("records") if isinstance(blob, dict) else None if not recs: return [], "reference_io.pt carries no 'records' (not a capture_shapes oracle)" + if meta is not None and meta.get("synthesized") is not True: + try: + declared_cases = int(meta.get("num_cases")) + except (TypeError, ValueError): + return [], "num_cases is missing or invalid; captured oracle coverage is not identity-pinned" + if declared_cases <= 0: + return [], f"num_cases={declared_cases}; a captured oracle must contain at least one case" + if len(recs) != declared_cases: + return [], (f"num_cases mismatch: meta declares {declared_cases}, but reference_io.pt " + f"contains {len(recs)} record(s)") out = [] for r in recs: args = _rehydrate(r.get("args", ()), torch, device) @@ -267,12 +323,35 @@ def load_oracle_records(task, torch, device): return out, "" +def _param_index_by_name(meta): + """Map each parameter NAME of the live target callable to its positional index. Lets an in-place + output buffer passed POSITIONALLY (not as a kwarg) still be located. Returns {} on any failure.""" + import inspect + spec = str(meta.get("target_callable") or "").strip() + fn = _resolve_callable(spec) if spec else None + if fn is None: + return {} + try: + out, index = {}, 0 + for p in inspect.signature(fn).parameters.values(): + if p.kind in (inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD): + out[p.name] = index + index += 1 + elif p.kind is inspect.Parameter.VAR_POSITIONAL: + # Everything after *args is keyword-only and has no stable index in the args tuple. + break + return out + except (TypeError, ValueError): + return {} + + def _out_param_of(rec, meta): """For an IN-PLACE seam (returns None, writes into a caller-supplied buffer) the oracle's `output` snapshot is None and the golden values live in one of the INPUTS after the call. Which one is a fact the extractor records; we do not guess from names here beyond a last-resort default, and we say so. - Returns (kind, key) with kind in {"kwarg","arg",""}. + Returns (kind, key) with kind in {"kwarg","arg",""}; for "arg", key is the positional index. """ ev = meta.get("seam_runtime_evidence") or {} names = [str(n) for n in (ev.get("inplace_params") or []) if str(n)] @@ -280,6 +359,15 @@ def _out_param_of(rec, meta): if n in rec["kwargs"]: return "kwarg", n if names: + # C3: not a kwarg — the buffer may have been passed POSITIONALLY. Resolve each declared name to + # its positional index via the live signature and look in rec["args"]. Still a recorded fact + # (the name comes from meta), not a guess. + idx_by_name = _param_index_by_name(meta) + args = rec.get("args") or () + for n in names: + i = idx_by_name.get(n) + if i is not None and i < len(args): + return "arg", i return "", "" # declared, but not present in this record -> refuse, don't guess return "", "" @@ -314,7 +402,7 @@ def bench_captured_replay(args, meta): delegation = _delegation_status(meta) results = [] - recs, err = load_oracle_records(args.task, torch, device) + recs, err = load_oracle_records(args.task, torch, device, meta) if err: note_degradation("bench_captured_replay", "no replayable oracle", err) return [{"backend": "current", "available": False, "correct": False, "ms": None, @@ -346,6 +434,8 @@ def bench_captured_replay(args, meta): golden = rec["ref"] if ip_kind == "kwarg" and torch.is_tensor(rec["kwargs"].get(ip_key)): golden = rec["kwargs"][ip_key].clone() + elif ip_kind == "arg" and ip_key < len(rec["args"]) and torch.is_tensor(rec["args"][ip_key]): + golden = rec["args"][ip_key].clone() for name, spec in plan: fn = _resolve_callable(spec) @@ -360,8 +450,16 @@ def call_once(): a, k = _clone_inputs(rec, torch) if ip_kind == "kwarg" and torch.is_tensor(k.get(ip_key)): k[ip_key].zero_() + elif ip_kind == "arg" and ip_key < len(a) and torch.is_tensor(a[ip_key]): + a[ip_key].zero_() r = fn(*a, **k) - return r if r is not None else (k.get(ip_key) if ip_kind == "kwarg" else None) + if r is not None: + return r + if ip_kind == "kwarg": + return k.get(ip_key) + if ip_kind == "arg" and ip_key < len(a): + return a[ip_key] + return None try: out = call_once(); _sync(torch) @@ -376,8 +474,9 @@ def call_once(): if out is None or ref is None or not torch.is_tensor(out) or not torch.is_tensor(ref): # We CAN time it, but we cannot certify it. Say exactly that instead of asserting correct. ms, wall_ms = _time_call(call_once, args.warmup, args.repeats) - why = ("seam returns None and meta.seam_runtime_evidence.inplace_params does not name the " - "output buffer" if out is None else + why = ("seam returns None and its output buffer could not be located from " + "meta.seam_runtime_evidence.inplace_params (not present as a kwarg or as a " + "resolvable positional arg)" if out is None else "oracle recorded no comparable output tensor for this case") note_degradation("bench_captured_replay", f"{name}: timed but unverified", why) results.append({"backend": name, "available": True, "correct": None, "ms": round(ms, 4) if ms else None, @@ -613,6 +712,9 @@ def _load_or_synth_gemm(torch, task, meta, device, seed): use_bias = bool(meta.get("bias", False)) iopath = os.path.join(task, "reference_io.pt") if os.path.exists(iopath): + sha_ok, sha_err = verify_oracle_sha(task, meta) # C4: refuse a tampered oracle before trusting it + if not sha_ok: + raise ValueError(sha_err) blob = torch.load(iopath, map_location=device) # accept a few shapes of recorded blob A = blob.get("A") if isinstance(blob, dict) else None @@ -1098,8 +1200,12 @@ def main(): "isolated_speedup_unpublishable": round(speedup, 4) if withhold else None, "speedup_withheld": bool(withhold), "speedup_withheld_reason": speedup_reason, - "denominator": {"spec": denom["spec"], "provenance": prov, "severity": severity, - "why": denom_why, "strict": bool(a.denominator_strict)}, + # C1: the orchestrator's OPBENCH_SCHEMA + denominatorSound() require `denominator` to be a + # flat provenance-enum STRING (matched against BANKABLE_DENOMINATORS). Emit that at top level; + # carry the object detail alongside so nothing is lost. + "denominator": prov, + "denominator_detail": {"spec": denom["spec"], "provenance": prov, "severity": severity, + "why": denom_why, "strict": bool(a.denominator_strict)}, "delegated_config_track": _delegation_status(meta), "degradations": DEGRADATIONS, "reason_code": next((r.get("reason_code") for r in results if r.get("reason_code")), diff --git a/e2e_workflow/scripts/parse_profile.py b/e2e_workflow/scripts/parse_profile.py index f89db0e72..6cdcb3363 100644 --- a/e2e_workflow/scripts/parse_profile.py +++ b/e2e_workflow/scripts/parse_profile.py @@ -399,6 +399,22 @@ def norm_key(name): def classify_entity(d, source): """-> (entity_kind, evidence). `d` is an agg entry; `source` the profile source string.""" cats = d.get("cat_counts") or {} + evidence_sources = set(d.get("evidence_sources") or []) + if d.get("src"): + evidence_sources.add(d["src"]) + observed_kinds = {_CAT_ENTITY.get(cat, "unresolved") for cat, n in cats.items() if n} + if "rocprof_kernel_stats" in evidence_sources: + observed_kinds.add("gpu_kernel") + device_kinds = observed_kinds & {"gpu_kernel", "memory_op"} + host_kinds = observed_kinds & {"dispatcher_op", "python_launcher"} + if device_kinds and host_kinds: + return "unresolved", { + "basis": "exact_name_multiple_entity_kinds", + "categories": dict(sorted(cats.items())), + "evidence_sources": sorted(evidence_sources), + "why": "the exact name denotes both host and device events; refusing to guess which entity " + "the external Top-N row represents", + } if cats: # A name can appear under several categories; the device categories decide. kinds = {} @@ -445,24 +461,72 @@ def index_host_entities(path): return idx +def merge_entity_evidence(*aggregates): + """Merge profiler evidence without letting a same-name source overwrite another entity kind.""" + merged = {} + for agg in aggregates: + for name, raw in (agg or {}).items(): + d = dict(raw) + d["cat_counts"] = dict(raw.get("cat_counts") or {}) + sources = set(raw.get("evidence_sources") or []) + if raw.get("src"): + sources.add(raw["src"]) + d["evidence_sources"] = sorted(sources) + if name not in merged: + merged[name] = d + continue + cur = merged[name] + cur["calls"] = int(cur.get("calls") or 0) + int(d.get("calls") or 0) + cur["total_us"] = float(cur.get("total_us") or 0.0) + float(d.get("total_us") or 0.0) + for cat, count in d["cat_counts"].items(): + cur.setdefault("cat_counts", {})[cat] = cur.setdefault("cat_counts", {}).get(cat, 0) + count + cur_sources = set(cur.get("evidence_sources") or []) + cur_sources.update(d.get("evidence_sources") or []) + cur["evidence_sources"] = sorted(cur_sources) + return merged + + def annotate_rows(rows, agg, source): """Stamp entity_kind onto Top-N rows that were assembled OUTSIDE this script (the TraceLens fast path builds them by hand from a third-party report). A row survives as a gpu_kernel only if its name matches something this profiler actually observed being dispatched; otherwise it is `unresolved` and the head track will refuse it. Returns (rows, stats).""" - by_key = {} - for name, d in (agg or {}).items(): - by_key.setdefault(norm_key(name), (name, d)) + agg = agg or {} + # C9: resolve by EXACT name first. norm_key() is a lossy fold (short_name + strip non-alnum), so a + # host dispatcher span 'aten::mul' and a device kernel 'mul' collapse to the SAME key; the old + # `setdefault` let whichever was aggregated first stamp its kind + matched_profiled_kernel onto the + # other row, silently misclassifying it and defeating the INV-6 gate in EITHER direction (a + # dispatcher_op admitted as a gpu_kernel, or a real kernel downgraded). The normalized fold is kept + # only as a fallback, and only when the key maps to exactly ONE observed entity. + by_exact = dict(agg) + by_norm, norm_collisions = {}, set() + for name in agg: + k = norm_key(name) + if k in by_norm and by_norm[k][0] != name: + norm_collisions.add(k) + else: + by_norm.setdefault(k, (name, agg[name])) stats = {k: 0 for k in ENTITY_KINDS} for r in rows: - hit = by_key.get(norm_key(r.get("name") or r.get("short_name") or "")) + rname = r.get("name") or r.get("short_name") or "" + k = norm_key(rname) + hit = None + if rname in by_exact: + hit = (rname, by_exact[rname]) + elif k in by_norm and k not in norm_collisions: + hit = by_norm[k] if hit: kind, ev = classify_entity(hit[1], source) ev["matched_profiled_kernel"] = hit[0] + elif k in norm_collisions: + kind = "unresolved" # fail closed: refuse to guess which colliding entity + ev = {"basis": "ambiguous_name_match", + "why": f"'{rname}' normalizes to a key shared by multiple entities observed in " + f"{source}; refusing to guess its kind rather than misclassify it"} else: kind = "unresolved" ev = {"basis": "name_not_found_in_profile", - "why": f"not among the {len(by_key)} kernels observed in {source}"} + "why": f"not among the {len(by_exact)} kernels observed in {source}"} r["entity_kind"] = kind r["entity_evidence"] = ev stats[kind] = stats.get(kind, 0) + 1 @@ -724,11 +788,11 @@ def main(): if args.annotate: with open(args.annotate) as fh: doc = json.load(fh) - ev_agg = dict(rp_agg or {}) - # Host spans first, so a device row of the same name overrides them below. - if args.torch_trace: - ev_agg.update(index_host_entities(args.torch_trace)) - ev_agg.update(torch_agg or {}) # torch categories are richer evidence; let them win + # Preserve all same-name evidence. A host span and a dispatched kernel can legitimately share + # an exact name; overwriting either one fabricates certainty, so classify_entity marks that row + # unresolved and the head gate refuses it. + host_agg = index_host_entities(args.torch_trace) if args.torch_trace else {} + ev_agg = merge_entity_evidence(rp_agg, host_agg, torch_agg) ev_src = doc.get("source") or ("merged" if (rp_agg and torch_agg) else "rocprofv3" if rp_agg else "torch-trace") rows, stats = annotate_rows(doc.get("top_kernels") or [], ev_agg, ev_src) diff --git a/e2e_workflow/scripts/seam_contract.py b/e2e_workflow/scripts/seam_contract.py index 5ebf5df6d..e33f060b5 100644 --- a/e2e_workflow/scripts/seam_contract.py +++ b/e2e_workflow/scripts/seam_contract.py @@ -254,24 +254,9 @@ def add(cid, ok, detail): # -------------------------------------------------------------------------- INV-2 binding contract -def describe_binding(spec, meta=None): - """Capture the LIVE callable's call contract by reflection. This descriptor -- not an agent's - recollection of it -- is what an authored entry has to satisfy.""" +def _describe_signature(sig, spec, qualname, resolved_file=None, meta=None): + """Build a binding descriptor from an already-resolved inspect.Signature.""" meta = dict(meta or {}) - res = resolve_spec(spec) - if not res.get("ok"): - return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": spec, "error": res.get("error")} - obj = res["obj"] - if not callable(obj): - return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": spec, "error": "seam resolves to a non-callable"} - try: - sig = inspect.signature(obj) - except Exception as e: # noqa: BLE001 - return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": spec, "error": f"signature unavailable: {e!r}"} - params, required_positional, out_params = [], [], [] accepts_varargs = accepts_varkw = False for name, p in sig.parameters.items(): @@ -306,21 +291,43 @@ def describe_binding(spec, meta=None): # Inputs the callable reads that do NOT arrive through its parameters (forward-context, layer # registries, module globals). A non-empty list means the seam is NOT a pure function of its # arguments, so an out-of-tree rewrite cannot be given the same inputs -- see check_binding. - hidden_ctx = list((meta.get("seam_runtime_evidence") or {}).get("hidden_context") or []) + hidden_known = isinstance(ev, dict) and isinstance(ev.get("hidden_context"), list) + hidden_ctx = list(ev["hidden_context"]) if hidden_known else [] return { "contract": "binding", "contract_version": CONTRACT_VERSION, "ok": True, - "seam": spec, "qualname": res.get("qualname"), "resolved_file": res.get("file"), + "seam": spec, "qualname": qualname, "resolved_file": resolved_file, "params": params, "required_positional": required_positional, "accepts_varargs": accepts_varargs, "accepts_varkw": accepts_varkw, "arity_required": len(required_positional), "arity_total": len(params), "returns_annotation": ret_ann, "returns_none": returns_none, "out_params": out_params, "out_params_evidence": evidence, "hidden_context": hidden_ctx, - "signature": f"{res.get('qualname')}{sig}", + "hidden_context_evidence": "declared" if hidden_known else "unknown", + "signature": f"{qualname}{sig}", } +def describe_binding(spec, meta=None): + """Capture the LIVE callable's call contract by reflection. This descriptor -- not an agent's + recollection of it -- is what an authored entry has to satisfy.""" + meta = dict(meta or {}) + res = resolve_spec(spec) + if not res.get("ok"): + return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": spec, "error": res.get("error")} + obj = res["obj"] + if not callable(obj): + return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": spec, "error": "seam resolves to a non-callable"} + try: + sig = inspect.signature(obj) + except Exception as e: # noqa: BLE001 + return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, + "seam": spec, "error": f"signature unavailable: {e!r}"} + return _describe_signature(sig, spec, res.get("qualname"), res.get("file"), meta) + + def _ann_str(ann): if ann is None: return "None" @@ -329,6 +336,49 @@ def _ann_str(ann): return getattr(ann, "__name__", None) or str(ann) +def _signature_from_descriptor(desc): + """Reconstruct an inspect.Signature for call-compatibility checks.""" + params = [] + for p in desc.get("params") or []: + kind = getattr(inspect.Parameter, p["kind"]) + default = None if p.get("has_default") else inspect.Parameter.empty + params.append(inspect.Parameter(p["name"], kind, default=default)) + return inspect.Signature(params) + + +def _representative_calls(desc): + """Calls spanning the live signature's positional/keyword and optional surfaces.""" + calls = [] + for include_optional in (False, True): + for keyword_pok in (False, True): + args, kwargs = [], {} + for p in desc.get("params") or []: + kind, name = p["kind"], p["name"] + required = not p.get("has_default") and kind not in ("VAR_POSITIONAL", "VAR_KEYWORD") + if kind == "VAR_POSITIONAL": + if include_optional: + args.append(object()) + continue + if kind == "VAR_KEYWORD": + if include_optional: + kwargs["__geak_extra_kwarg__"] = object() + continue + if not required and not include_optional: + continue + value = object() + if kind == "POSITIONAL_ONLY": + args.append(value) + elif kind == "POSITIONAL_OR_KEYWORD": + if keyword_pok: + kwargs[name] = value + else: + args.append(value) + elif kind == "KEYWORD_ONLY": + kwargs[name] = value + calls.append((args, kwargs)) + return calls + + def check_binding(descriptor, candidate): """Can `candidate` be bound AT the seam described by `descriptor`? @@ -387,10 +437,46 @@ def check_binding(descriptor, candidate): m.append({"code": "return_contract_mismatch", "detail": "candidate writes in place but the seam's callers consume a returned value"}) + # C6: an OPTIONAL live parameter that the candidate DROPS is still passed by existing callers, so the + # bound call raises TypeError even though the REQUIRED names+arity line up. (Comparing only the + # required-name sets missed this.) A candidate that swallows extras via **kwargs is exempt. + if not cand_varkw: + cand_names = {p["name"] for p in (cand.get("params") or [])} + live_optional = [p["name"] for p in (descriptor.get("params") or []) + if p.get("has_default") and p["kind"] in ("POSITIONAL_OR_KEYWORD", "KEYWORD_ONLY")] + dropped = [n for n in live_optional if n not in cand_names] + if dropped: + m.append({"code": "optional_param_dropped", + "detail": f"seam accepts optional param(s) {dropped} that live callers may pass; the " + f"candidate entry omits them and would raise TypeError when they are"}) + + # Check actual Python binding behavior over the live signature's minimal/maximal and + # positional/keyword call surfaces. This catches positional->keyword-only changes, reordered + # positional parameters when keyword calls are also legal, and variadic incompatibilities without + # incorrectly rejecting a live POSITIONAL_ONLY parameter against an identical candidate. + try: + cand_sig = _signature_from_descriptor(cand) + for call_args, call_kwargs in _representative_calls(descriptor): + try: + cand_sig.bind(*call_args, **call_kwargs) + except TypeError as e: + if not any(x["code"] in ("arity_mismatch", "param_name_mismatch", + "optional_param_dropped", "param_kind_mismatch") for x in m): + m.append({"code": "param_kind_mismatch", + "detail": f"candidate rejects a call accepted by the live seam: {e}"}) + break + except (TypeError, ValueError, KeyError) as e: + m.append({"code": "param_kind_mismatch", + "detail": f"candidate signature descriptor is invalid: {e}"}) + # Hidden context. If the seam reads state that never crosses the parameter boundary, an authored # replacement cannot be handed the same inputs -- authoring it is wasted budget regardless of how # fast it is. This is the check that costs seconds and saves a kernel budget. - if descriptor.get("hidden_context"): + if descriptor.get("hidden_context_evidence") != "declared": + m.append({"code": "hidden_context_inputs", + "detail": "seam_runtime_evidence.hidden_context is missing; purity is unknown, so the " + "seam cannot be admitted under the fail-closed binding contract"}) + elif descriptor.get("hidden_context"): m.append({"code": "hidden_context_inputs", "detail": f"seam reads non-parameter inputs {descriptor['hidden_context']}; it is not a " f"pure function of its arguments -- rebind at an inner seam that is"}) @@ -418,6 +504,11 @@ def render_entry(descriptor, entry_name="entry"): parts.append(f"{n}=None") else: parts.append(n) + posonly = [p["name"] for p in (descriptor.get("params") or []) if p["kind"] == "POSITIONAL_ONLY"] + if posonly: + last = posonly[-1] + i = parts.index(last) if last in parts else parts.index(f"{last}=None") + parts.insert(i + 1, "/") has_star = any(p["kind"] == "VAR_POSITIONAL" for p in descriptor.get("params") or []) kwonly = [p["name"] for p in (descriptor.get("params") or []) if p["kind"] == "KEYWORD_ONLY"] if kwonly and not has_star: @@ -455,7 +546,13 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--task-dir", default="", help="op task dir containing meta.json") ap.add_argument("--eval-dir", default="", help="run EVAL_DIR (also disqualified as a baseline origin)") - ap.add_argument("--spec", default="", help="module:attr to describe (overrides meta.target_callable)") + # C10: the binding DESCRIPTOR must be built from the deployment TARGET, never the baseline. Two + # explicit flags make the intent unambiguous; --spec stays as a DEPRECATED alias for --target-spec so + # existing callers keep working (it used to feed the descriptor and, when a role piped the + # baseline_callable through it, silently described the denominator instead of the deployment seam). + ap.add_argument("--target-spec", default="", help="module:attr of the DEPLOYMENT seam to describe (the binding target)") + ap.add_argument("--baseline-spec", default="", help="module:attr of the baseline/denominator (overrides meta.baseline_callable for validation)") + ap.add_argument("--spec", default="", help="DEPRECATED alias for --target-spec") ap.add_argument("--mode", default="both", choices=["baseline", "binding", "both", "entry"]) ap.add_argument("--candidate", default="", help="module:attr of the authored entry, for --mode binding") ap.add_argument("--entry-name", default="entry") @@ -473,21 +570,59 @@ def main(argv=None): if args.task_dir: meta, meta_err = _load_meta(args.task_dir) + # Explicit CLI specs override the corresponding meta fields for this validation run. + if args.baseline_spec: + meta = dict(meta) + meta["baseline_callable"] = args.baseline_spec + if args.target_spec or args.spec: + meta = dict(meta) + meta["target_callable"] = args.target_spec or args.spec + result = {"contract_version": CONTRACT_VERSION, "task_dir": args.task_dir or None} if meta_err: result["meta_error"] = meta_err + if args.spec and not args.target_spec: + sys.stderr.write("seam_contract: --spec is deprecated; use --target-spec for the deployment seam " + "(--spec builds the binding DESCRIPTOR, never the baseline)\n") if args.mode in ("baseline", "both"): result["baseline_validation"] = validate_baseline(args.task_dir or None, meta, args.eval_dir or None) if args.mode in ("binding", "both", "entry"): - spec = args.spec or (meta.get("target_callable") or "") + # C10: the descriptor is the DEPLOYMENT target's contract. Precedence: --target-spec, then the + # deprecated --spec alias, then meta.target_callable. The baseline is NEVER what gets described. + spec = args.target_spec or args.spec or (meta.get("target_callable") or "") desc = describe_binding(spec, meta) if spec else { "contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": "", "error": "no target_callable / --spec given"} + "seam": "", "error": "no target_callable / --target-spec given"} result["binding_descriptor"] = desc if args.candidate: result["binding_check"] = check_binding(desc, args.candidate) + elif desc.get("ok") and args.mode in ("binding", "both"): + # Validate the ACTUAL signature emitted by render_entry(), not the descriptor against itself. + # This proves the generated immutable entry contract accepts every live call shape. + try: + ns = {} + exec(render_entry(desc, args.entry_name), ns) # generated source; no untrusted input runs + generated = ns[args.entry_name] + generated_meta = {"seam_runtime_evidence": { + "inplace_params": list(desc.get("out_params") or []), + "returns_none": bool(desc.get("returns_none")), + "hidden_context": [], + }} + cand_desc = _describe_signature(inspect.signature(generated), + "", + args.entry_name, None, generated_meta) + result["binding_check"] = check_binding(desc, cand_desc) + result["binding_check"]["candidate"] = "" + except Exception as e: # noqa: BLE001 + result["binding_check"] = { + "contract": "binding_check", "contract_version": CONTRACT_VERSION, + "bindable": False, "seam": desc.get("seam"), "candidate": "", + "mismatches": [{"code": "candidate_unresolvable", + "detail": f"generated entry could not be inspected: {e!r}"}], + "codes": ["candidate_unresolvable"], + } if args.mode == "entry": if not desc.get("ok"): result["entry_error"] = desc.get("error") diff --git a/e2e_workflow/scripts/tests/test_op_bench.py b/e2e_workflow/scripts/tests/test_op_bench.py index 0ae442bcb..b03ab5a22 100644 --- a/e2e_workflow/scripts/tests/test_op_bench.py +++ b/e2e_workflow/scripts/tests/test_op_bench.py @@ -33,6 +33,7 @@ tearDown so the other test modules in this directory keep seeing a torch-free image. """ import contextlib +import hashlib import importlib.util import io import json @@ -173,6 +174,9 @@ def float(self): def contiguous(self): return self._like() + def clone(self): + return self._like() + def reshape(self, *shape): if len(shape) == 1 and isinstance(shape[0], (tuple, list)): shape = tuple(shape[0]) @@ -221,6 +225,10 @@ def __getitem__(self, idx): def __setitem__(self, idx, value): self.val = value.val if isinstance(value, _T) else float(value) + def zero_(self): # in-place clear, mirroring torch's out-buffer zeroing + self.val = 0.0 + return self + # ---- reductions / elementwise def abs(self): return self._like(val=abs(self.val)) @@ -360,6 +368,7 @@ def _build(self): torch = types.ModuleType("torch") for dt in (BF16, FP16, FP32, INT8, UINT8, FP8_E4M3FNUZ, FP8_E5M2FNUZ, FP8_E4M3FN, FP8_E5M2): setattr(torch, dt.name, dt) + torch.is_tensor = lambda value: isinstance(value, _T) class _Finfo: def __init__(self, dt): @@ -388,7 +397,7 @@ def make(*shape, **kw): return _T(shape, kw.get("dtype") or FP32, val, kw.get("device") or "cpu") return make - def _load_blob(path, map_location=None): + def _load_blob(path, map_location=None, **_kwargs): stack.calls.append(("torch.load", os.path.basename(path), map_location)) if stack.loaded_blob is None: raise RuntimeError("fake torch.load has no blob registered for %s" % path) @@ -1075,11 +1084,18 @@ def _task_with_io(self, blob): self.stack.loaded_blob = blob return d + @staticmethod + def _pinned_meta(task, **extra): + with open(os.path.join(task, "reference_io.pt"), "rb") as fh: + sha = hashlib.sha256(fh.read()).hexdigest() + return {"reference_io_sha256": sha, **extra} + def test_recorded_oracle_is_preferred_over_synthesis(self): blob = {"A": _T((8, 16), FP32, 0.25), "B": _T((32, 16), FP32, 0.5), "bias": None, "output": _T((8, 32), FP32, 4.0)} d = self._task_with_io(blob) - A, B, bias, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, {"dtype": "bf16"}, "cpu", 0) + A, B, bias, tb, ref = ob._load_or_synth_gemm( + self.stack.torch, d, self._pinned_meta(d, dtype="bf16"), "cpu", 0) self.assertIs(A.dtype, BF16) self.assertIs(B.dtype, BF16) self.assertIsNone(bias) @@ -1092,7 +1108,8 @@ def test_oracle_without_a_recorded_output_is_recomputed_with_bias(self): blob = {"A": _T((8, 16), FP32, 0.5), "B": _T((32, 16), FP32, 2.0), "bias": _T((32,), FP32, 1.0), "output": None} d = self._task_with_io(blob) - A, B, bias, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, {}, "cpu", 0) + A, B, bias, tb, ref = ob._load_or_synth_gemm( + self.stack.torch, d, self._pinned_meta(d), "cpu", 0) self.assertEqual(ref.shape, (8, 32)) self.assertEqual(ref.val, 0.5 * 2.0 * 16 + 1.0) self.assertIs(bias.dtype, BF16) @@ -1100,14 +1117,14 @@ def test_oracle_without_a_recorded_output_is_recomputed_with_bias(self): def test_recomputed_oracle_honours_transpose_b_false(self): blob = {"A": _T((8, 16), FP32, 1.0), "B": _T((16, 32), FP32, 1.0)} d = self._task_with_io(blob) - _, _, _, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, {"transpose_b": False}, - "cpu", 0) + _, _, _, tb, ref = ob._load_or_synth_gemm( + self.stack.torch, d, self._pinned_meta(d, transpose_b=False), "cpu", 0) self.assertFalse(tb) self.assertEqual(ref.shape, (8, 32)) def test_unrecognised_blob_falls_back_to_synthesis(self): d = self._task_with_io(["not", "a", "dict"]) - meta = {"a_shape": [4, 16], "b_shape": [32, 16]} + meta = self._pinned_meta(d, a_shape=[4, 16], b_shape=[32, 16]) A, B, bias, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, meta, "cpu", 3) self.assertEqual((A.shape, B.shape, ref.shape), ((4, 16), (32, 16), (4, 32))) @@ -1499,19 +1516,166 @@ def test_missing_capture_is_reported_as_unavailable(self): self.assertEqual(res[0]["backend"], "current") self.assertFalse(res[0]["available"]) self.assertIsNone(res[0]["ms"]) - self.assertIn("needs reference_io.pt", res[0]["note"]) + self.assertIn("no reference_io.pt", res[0]["note"]) def test_captured_oracle_is_validated_and_backend_swaps_are_delegated(self): d = self._task_dir() io_path = os.path.join(d, "reference_io.pt") open(io_path, "w").close() - res = ob.bench_attn(self._args(task=d), {"op_kind": "attn"}) + mod = types.ModuleType("fake_attn_seam") + mod.current = lambda x: x + sys.modules[mod.__name__] = mod + self.addCleanup(sys.modules.pop, mod.__name__, None) + self.stack.loaded_blob = {"records": [{ + "sig": "decode", "regime": "decode", "args": (_T((2, 4), FP32, 1.0),), + "kwargs": {}, "output": _T((2, 4), FP32, 1.0), + }]} + with open(io_path, "rb") as fh: + sha = hashlib.sha256(fh.read()).hexdigest() + meta = {"op_kind": "attn", "target_callable": "fake_attn_seam:current", + "reference_io_sha256": sha, "num_cases": 1} + res = ob.bench_attn(self._args(task=d), meta) self.assertTrue(res[0]["available"]) self.assertTrue(res[0]["correct"]) - self.assertIsNone(res[0]["ms"]) # op-level attention is not raced here - self.assertEqual(res[0]["artifact"], io_path) - self.assertIn("--attention-backend", res[0]["note"]) - self.assertIn("Config Tuner", res[0]["note"]) + self.assertIsNotNone(res[0]["ms"]) + self.assertIn("fake_attn_seam:current", res[0]["note"]) + + +# --------------------------------------------------------------------------- # +# bench_captured_replay -- POSITIONAL in-place output buffer, END-TO-END (C3) +# --------------------------------------------------------------------------- # +class TestCapturedReplayPositionalBuffer(_FakeStackMixin, unittest.TestCase): + """C3 END-TO-END: TestOutParamOf covers _out_param_of at the unit level; this drives the + positional ('arg') branch through bench_captured_replay itself. An in-place seam whose output + buffer is passed POSITIONALLY (entry(query,key,value,output,kv_cache)->None, output at index 3) + must have its golden taken from rec['args'][ip_key] (op_bench.py ~437-438), the copy handed to the + callee zeroed (~453), the written buffer returned (~460-461) and the result verified. A resolved + index that lands on a NON-tensor arg must degrade to the UNVERIFIED path (correct=None), never a + false-green.""" + + RAW = b"positional-inplace-oracle-bytes" + + def setUp(self): + super().setUp() + self.seen_buffer_vals = [] + seen = self.seen_buffer_vals + self.mod = types.ModuleType("fake_pos_seam") + + def attn_inplace(query, key, value, output, kv_cache=None): + # In-place seam: record what the callee was handed (must be zeroed), then write the answer. + seen.append(output.val) + output[:] = query + return None + + def attn_noop(query, key, value, output, kv_cache=None): + return None # never writes the buffer (output here is a non-tensor) + + self.mod.attn_inplace = attn_inplace + self.mod.attn_noop = attn_noop + sys.modules["fake_pos_seam"] = self.mod + self.addCleanup(sys.modules.pop, "fake_pos_seam", None) + + def _replay_task(self, records, target): + d = self._task_dir() + with open(os.path.join(d, "reference_io.pt"), "wb") as fh: + fh.write(self.RAW) + self.stack.loaded_blob = {"records": records} + return d, {"op_kind": "attn", "target_callable": target, + "reference_io_sha256": hashlib.sha256(self.RAW).hexdigest(), + "num_cases": len(records), + "seam_runtime_evidence": {"inplace_params": ["output"]}} + + def test_positional_output_buffer_is_zeroed_verified_and_returned(self): + # output snapshot is None (in-place seam); the golden lives in the recorded arg at index 3. + buf = _T((2, 4), FP32, 1.0) # captured AFTER the original ran -> holds the golden + rec = {"sig": "decode", "regime": "decode", + "args": (_T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), buf), + "kwargs": {}, "output": None} + d, meta = self._replay_task([rec], "fake_pos_seam:attn_inplace") + res = ob.bench_captured_replay(self._args(task=d), meta) + row = self._by_backend(res)["current"] + self.assertTrue(row["available"]) + # correct:True is only reachable if the golden was pulled from rec['args'][3] (output=None), + # the callee rewrote the returned buffer, and _correct verified it. + self.assertTrue(row["correct"]) + self.assertFalse(row["raised"]) + self.assertEqual(row["ms"], 1.5) + self.assertEqual(row["max_rel_err"], 0.0) + # every replay iteration handed the callee a ZEROED buffer (~453), not the pre-filled golden. + self.assertTrue(self.seen_buffer_vals) + self.assertTrue(all(v == 0.0 for v in self.seen_buffer_vals), self.seen_buffer_vals) + + def test_positional_index_on_a_non_tensor_arg_is_unverified_not_false_green(self): + rec = {"sig": "decode", "regime": "decode", + "args": (_T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), 42), + "kwargs": {}, "output": None} # index 3 ('output') resolves onto a NON-tensor + d, meta = self._replay_task([rec], "fake_pos_seam:attn_noop") + res = ob.bench_captured_replay(self._args(task=d), meta) + row = self._by_backend(res)["current"] + self.assertTrue(row["available"]) + self.assertIsNone(row["correct"]) # cannot certify -> NOT correct:True + self.assertFalse(row["raised"]) + self.assertIsNotNone(row["ms"]) # still timed, just unverified + self.assertIn("UNVERIFIED", row["note"]) + + +# --------------------------------------------------------------------------- # +# load_oracle_records -- num_cases record-count verification FAILURE modes (C4) +# --------------------------------------------------------------------------- # +class TestLoadOracleRecordCount(_FakeStackMixin, unittest.TestCase): + """C4: after the SHA identity check passes, load_oracle_records must also verify the declared + num_cases matches the actual record count (op_bench.py ~299-308). A missing / non-numeric / + non-positive / mismatched count is a fail-closed reject (records=[]), never a bankable replay. + A real reference_io.pt with a correct recomputed SHA is written so the count check is actually + reached -- it runs AFTER the sha check.""" + + RAW = b"reference-io-count-oracle-bytes" + + def _task(self, n_records): + d = self._task_dir() + with open(os.path.join(d, "reference_io.pt"), "wb") as fh: + fh.write(self.RAW) + recs = [{"sig": "s%d" % i, "regime": "decode", "args": (_T((2, 2), FP32, 1.0),), + "kwargs": {}, "output": _T((2, 2), FP32, 1.0)} for i in range(n_records)] + self.stack.loaded_blob = {"records": recs} + return d, hashlib.sha256(self.RAW).hexdigest() + + def _meta(self, sha, **extra): + return dict({"op_kind": "attn", "reference_io_sha256": sha}, **extra) + + def test_matching_count_passes_the_gate(self): + # positive control: proves the sha check is passed and the count gate is actually reached. + d, sha = self._task(2) + recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha, num_cases=2)) + self.assertEqual(err, "") + self.assertEqual(len(recs), 2) + + def test_missing_num_cases_is_rejected(self): + d, sha = self._task(1) + recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha)) + self.assertEqual(recs, []) + self.assertIn("num_cases is missing or invalid", err) + + def test_non_numeric_num_cases_is_rejected(self): + d, sha = self._task(1) + recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", + self._meta(sha, num_cases="not-a-number")) + self.assertEqual(recs, []) + self.assertIn("num_cases is missing or invalid", err) + + def test_nonpositive_num_cases_is_rejected(self): + d, sha = self._task(1) + recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha, num_cases=0)) + self.assertEqual(recs, []) + self.assertIn("at least one case", err) + + def test_count_mismatch_is_rejected(self): + d, sha = self._task(2) + recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha, num_cases=5)) + self.assertEqual(recs, []) + self.assertIn("num_cases mismatch", err) + self.assertIn("declares 5", err) + self.assertIn("2 record", err) # --------------------------------------------------------------------------- # @@ -1529,6 +1693,7 @@ def fake_bench(args, m): return list(results or []) ob.bench_gemm = fake_bench ob.bench_attn = fake_bench + ob.bench_captured_replay = fake_bench argv = ["op_bench.py", "--task", d] + list(extra_argv) if out_path: argv += ["--out", out_path] @@ -1654,12 +1819,45 @@ def test_hipblaslt_winner_has_nothing_to_deploy(self): self.assertEqual(summary["isolated_speedup"], 1.0) # it is its own baseline self.assertIn("nothing to deploy", summary["deployable_note"]) - def test_library_winner_without_a_baseline_reports_a_neutral_speedup(self): + def test_measured_backend_default_is_a_bankable_flat_denominator(self): + # C1: a real, timed library baseline (hipblaslt) -> denominator is the BANKABLE flat string + # 'measured_backend_default'; the object travels under denominator_detail. + summary, _ = self._run_main( + {"op_kind": "gemm"}, + results=[self._res("hipblaslt", ms=2.0, correct=True), + self._res("aiter", ms=1.0, correct=True)]) + self.assertEqual(summary["denominator"], "measured_backend_default") + self.assertIsInstance(summary["denominator"], str) + self.assertEqual(ob.DENOM_SEVERITY[summary["denominator"]], "ok") # bankable + self.assertFalse(summary["speedup_withheld"]) + self.assertEqual(summary["isolated_speedup"], 2.0) + detail = summary["denominator_detail"] + self.assertIsInstance(detail, dict) + self.assertEqual(detail["provenance"], "measured_backend_default") + self.assertEqual(detail["severity"], "ok") + self.assertIn("spec", detail) + + def test_verified_baseline_row_banks_a_flat_verified_denominator(self): + # C1: an explicit 'baseline' row carrying a verified provenance -> the flat bankable string + # 'verified_baseline' at top level, still with the object under denominator_detail. + summary, _ = self._run_main( + {"op_kind": "gemm"}, + results=[self._res("baseline", ms=2.0, correct=True, + denominator_provenance="verified_baseline"), + self._res("current", ms=1.0, correct=True)]) + self.assertEqual(summary["denominator"], "verified_baseline") + self.assertEqual(ob.DENOM_SEVERITY[summary["denominator"]], "ok") # bankable + self.assertFalse(summary["speedup_withheld"]) + self.assertEqual(summary["isolated_speedup"], 2.0) + self.assertEqual(summary["denominator_detail"]["provenance"], "verified_baseline") + + def test_library_winner_without_a_baseline_withholds_the_speedup(self): summary, _ = self._run_main({"op_kind": "gemm"}, results=[self._res("flydsl", ms=1.0, correct=True)]) self.assertEqual(summary["winner_backend"], "flydsl") self.assertIsNone(summary["baseline_backend"]) - self.assertEqual(summary["isolated_speedup"], 1.0) + self.assertIsNone(summary["isolated_speedup"]) + self.assertTrue(summary["speedup_withheld"]) self.assertEqual(summary["winner_kind"], "none") self.assertIn("verify deployability", summary["deployable_note"]) @@ -1758,5 +1956,182 @@ def test_out_file_carries_the_task_dir_and_full_result_list(self): self.assertEqual(summary["apply_flags"], "") +# --------------------------------------------------------------------------- # +# resolve_denominator -- verdict-to-spec binding (C5) and the flat provenance (C1) +# --------------------------------------------------------------------------- # +class TestResolveDenominator(unittest.TestCase): + """A machine verdict certifies ONE spec. resolve_denominator must only bank a speedup when the + recorded verdict actually names the current baseline (and target, if it names one) -- a stale or + foreign verdict from a prior retry/task must fall through to unverified_baseline.""" + + def test_matching_verdict_upgrades_to_verified_baseline(self): + meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:fast", + "baseline_validation": {"contract": "baseline_identity", "ok": True, + "baseline_callable": "pkg.new:fast", + "target_callable": "pkg.new:fast"}} + d = ob.resolve_denominator(meta) + self.assertEqual(d["provenance"], "verified_baseline") + self.assertEqual(d["severity"], "ok") + + def test_verdict_for_a_different_callable_does_not_verify(self): + # ok:true, but the verdict certifies pkg.OLD:other -- not this baseline. Must NOT bank. + meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:fast", + "baseline_validation": {"contract": "baseline_identity", "ok": True, + "baseline_callable": "pkg.OLD:other", + "target_callable": "pkg.OLD:other"}} + d = ob.resolve_denominator(meta) + self.assertEqual(d["provenance"], "unverified_baseline") + self.assertEqual(d["severity"], "unverified") + self.assertIn("stale or foreign", d["why"]) + + def test_verdict_target_mismatch_alone_blocks_verification(self): + meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:v2", + "baseline_validation": {"contract": "baseline_identity", "ok": True, + "baseline_callable": "pkg.new:fast", + "target_callable": "pkg.new:OTHER"}} + d = ob.resolve_denominator(meta) + self.assertEqual(d["provenance"], "unverified_baseline") + + def test_verdict_without_a_target_is_not_identity_bound(self): + meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:fast", + "baseline_validation": {"contract": "baseline_identity", "ok": True, + "baseline_callable": "pkg.new:fast"}} + self.assertEqual(ob.resolve_denominator(meta)["provenance"], "unverified_baseline") + + +# --------------------------------------------------------------------------- # +# _out_param_of -- the in-place output buffer, kwarg OR positional (C3) +# --------------------------------------------------------------------------- # +class TestOutParamOf(unittest.TestCase): + """An in-place seam writes into a caller-supplied buffer. The extractor records its NAME; the + driver must locate it whether callers pass it as a kwarg or POSITIONALLY -- otherwise the real + target path (e.g. entry(query,key,value,output,kv_cache)->None) can never be verified.""" + + def setUp(self): + self.mod = types.ModuleType("fake_inplace_seam") + + def entry(query, key, value, output, kv_cache=None): + return None + self.mod.entry = entry + sys.modules["fake_inplace_seam"] = self.mod + self.addCleanup(sys.modules.pop, "fake_inplace_seam", None) + self.meta = {"target_callable": "fake_inplace_seam:entry", + "seam_runtime_evidence": {"inplace_params": ["output"]}} + + def test_kwarg_output_is_found_as_a_kwarg(self): + rec = {"args": (1, 2, 3), "kwargs": {"output": object()}} + self.assertEqual(ob._out_param_of(rec, self.meta), ("kwarg", "output")) + + def test_positional_output_is_resolved_to_its_index(self): + rec = {"args": (1, 2, 3, object()), "kwargs": {}} # output is the 4th positional + self.assertEqual(ob._out_param_of(rec, self.meta), ("arg", 3)) + + def test_declared_but_absent_buffer_refuses_rather_than_guessing(self): + rec = {"args": (1, 2, 3), "kwargs": {}} # only 3 positionals -> not present + self.assertEqual(ob._out_param_of(rec, self.meta), ("", "")) + + def test_no_declared_inplace_param_returns_empty(self): + rec = {"args": (1, 2, 3, 4), "kwargs": {}} + self.assertEqual(ob._out_param_of(rec, {"target_callable": "fake_inplace_seam:entry"}), ("", "")) + + def test_param_index_map_is_empty_for_an_unresolvable_target(self): + self.assertEqual(ob._param_index_by_name({"target_callable": "no_mod:none"}), {}) + + def test_keyword_only_output_is_never_mapped_into_the_args_tuple(self): + def variadic(query, *rest, output=None): + return None + self.mod.variadic = variadic + meta = {"target_callable": "fake_inplace_seam:variadic", + "seam_runtime_evidence": {"inplace_params": ["output"]}} + rec = {"args": (1, 2, 3), "kwargs": {}} + self.assertNotIn("output", ob._param_index_by_name(meta)) + self.assertEqual(ob._out_param_of(rec, meta), ("", "")) + + +class TestDenominatorIsFlatString(_ObStateMixin, unittest.TestCase): + """C1: the orchestrator's OPBENCH_SCHEMA + denominatorSound() match `denominator` as a flat + provenance-enum STRING. main() must emit that at top level, with the object under + denominator_detail.""" + + def _run(self, meta): + d = self._task_dir(meta) + out_path = os.path.join(d, "result.json") + ob.bench_gemm = lambda a, m: [{"backend": "hipblaslt", "available": True, + "correct": True, "ms": 1.0}] + ob.bench_attn = ob.bench_gemm + old = sys.argv + sys.argv = ["op_bench.py", "--task", d, "--out", out_path] + try: + with contextlib.redirect_stdout(io.StringIO()): + ob.main() + finally: + sys.argv = old + with open(out_path) as fh: + return json.load(fh) + + def test_denominator_is_a_string_and_detail_is_the_object(self): + s = self._run({"op_kind": "gemm"}) + self.assertIsInstance(s["denominator"], str) + self.assertIn(s["denominator"], set(ob.DENOM_SEVERITY)) + self.assertIsInstance(s["denominator_detail"], dict) + self.assertEqual(s["denominator_detail"]["provenance"], s["denominator"]) + self.assertIn("spec", s["denominator_detail"]) + + +class TestVerifyOracleSha(unittest.TestCase): + """C4: a declared reference_io_sha256 used to be trusted verbatim -- even a fabricated 'abc123' + passed, so an oracle altered after capture could still certify a 'correct' verdict. The verifier + now requires a complete digest and the referenced bytes, then recomputes and compares the identity.""" + + def _task(self, data=b"golden-oracle-bytes"): + d = tempfile.mkdtemp(prefix="op_sha_") + self.addCleanup(shutil.rmtree, d, True) + p = os.path.join(d, "reference_io.pt") + with open(p, "wb") as fh: + fh.write(data) + return d, hashlib.sha256(data).hexdigest() + + def test_matching_digest_passes(self): + d, sha = self._task() + ok, err = ob.verify_oracle_sha(d, {"reference_io_sha256": sha}) + self.assertTrue(ok, err) + self.assertEqual(err, "") + + def test_mismatched_digest_is_rejected_as_tamper(self): + d, _ = self._task() + ok, err = ob.verify_oracle_sha(d, {"reference_io_sha256": "a" * 64}) + self.assertFalse(ok) + self.assertIn("MISMATCH", err) + + def test_fabricated_short_hash_no_longer_passes(self): + """The exact fail-open the reviewer named: 'abc123' certified anything.""" + d, _ = self._task() + ok, _ = ob.verify_oracle_sha(d, {"reference_io_sha256": "abc123"}) + self.assertFalse(ok) + + def test_synthesized_oracle_is_exempt(self): + d, _ = self._task() + ok, err = ob.verify_oracle_sha(d, {"synthesized": True, "reference_io_sha256": "abc123"}) + self.assertTrue(ok, err) + + def test_absent_declared_sha_fails_closed(self): + d, _ = self._task() + self.assertFalse(ob.verify_oracle_sha(d, {})[0]) + self.assertFalse(ob.verify_oracle_sha(d, {"reference_io_sha256": ""})[0]) + + def test_absent_file_cannot_verify_the_digest(self): + d = tempfile.mkdtemp(prefix="op_sha_") + self.addCleanup(shutil.rmtree, d, True) + ok, err = ob.verify_oracle_sha(d, {"reference_io_sha256": "a" * 64}) + self.assertFalse(ok) + self.assertIn("missing", err) + + def test_loader_refuses_a_tampered_oracle(self): + d, _ = self._task(b"records-blob") + recs, err = ob.load_oracle_records(d, None, "cpu", {"reference_io_sha256": "a" * 64}) + self.assertEqual(recs, []) + self.assertIn("MISMATCH", err) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/e2e_workflow/scripts/tests/test_parse_profile.py b/e2e_workflow/scripts/tests/test_parse_profile.py index 153902292..627542c2f 100644 --- a/e2e_workflow/scripts/tests/test_parse_profile.py +++ b/e2e_workflow/scripts/tests/test_parse_profile.py @@ -253,6 +253,79 @@ def test_norm_key_joins_hw_name_to_torch_name(self): self.assertEqual(pp.norm_key("at::native::Fill_Kernel"), "fillkernel") +class TestEntityEvidence(unittest.TestCase): + def test_exact_name_host_device_collision_is_unresolved(self): + host = {"shared_name": {"calls": 2, "total_us": 5.0, + "cat_counts": {"cpu_op": 2}}} + device = {"shared_name": {"calls": 3, "total_us": 7.0, + "cat_counts": {"kernel": 3}}} + merged = pp.merge_entity_evidence(host, device) + rows, stats = pp.annotate_rows([{"name": "shared_name"}], merged, "torch-trace") + self.assertEqual(rows[0]["entity_kind"], "unresolved") + self.assertEqual(rows[0]["entity_evidence"]["basis"], + "exact_name_multiple_entity_kinds") + self.assertEqual(stats["unresolved"], 1) + + def test_same_name_device_sources_remain_a_gpu_kernel(self): + rocprof = {"kernel_name": {"calls": 2, "total_us": 5.0, + "src": "rocprof_kernel_stats"}} + torch = {"kernel_name": {"calls": 3, "total_us": 7.0, + "cat_counts": {"kernel": 3}}} + merged = pp.merge_entity_evidence(rocprof, torch) + rows, _ = pp.annotate_rows([{"name": "kernel_name"}], merged, "merged") + self.assertEqual(rows[0]["entity_kind"], "gpu_kernel") + + def test_c9_norm_key_collision_resolves_each_row_by_exact_name(self): + # C9 (reviewer's LITERAL scenario): a host dispatcher span 'aten::mul' (cpu_op) and a device + # kernel 'mul' (cat="kernel") are DIFFERENT exact names that norm_key() folds to the SAME key + # 'mul' (short_name drops the aten:: namespace, then non-alnum is stripped). The old + # setdefault(first-collision-wins) fold let whichever was aggregated first stamp its kind onto + # the other row. Exact-name resolution must win for BOTH, and a third row that only matches via + # the lossy fold must fail closed as ambiguous rather than borrow either neighbor's kind. + self.assertEqual(pp.norm_key("aten::mul"), pp.norm_key("mul")) # the collision premise + self.assertEqual(pp.norm_key("aten::Mul"), pp.norm_key("mul")) + host = {"aten::mul": {"calls": 2, "total_us": 5.0, "cat_counts": {"cpu_op": 2}}} + device = {"mul": {"calls": 3, "total_us": 7.0, "cat_counts": {"kernel": 3}}} + merged = pp.merge_entity_evidence(host, device) + rows, stats = pp.annotate_rows( + [{"name": "mul"}, {"name": "aten::mul"}, {"name": "aten::Mul"}], merged, "torch-trace") + by_name = {r["name"]: r for r in rows} + # device kernel keeps gpu_kernel — EXACT name wins over the shared fold + self.assertEqual(by_name["mul"]["entity_kind"], "gpu_kernel") + self.assertEqual(by_name["mul"]["entity_evidence"]["matched_profiled_kernel"], "mul") + # host dispatcher keeps dispatcher_op — EXACT name wins (not stamped as a kernel) + self.assertEqual(by_name["aten::mul"]["entity_kind"], "dispatcher_op") + self.assertEqual(by_name["aten::mul"]["entity_evidence"]["matched_profiled_kernel"], + "aten::mul") + # a NON-exact row whose norm_key hits the shared collision is stamped unresolved, refusing to + # guess which colliding entity it is — never silently misclassified as either kind. + self.assertEqual(by_name["aten::Mul"]["entity_kind"], "unresolved") + self.assertEqual(by_name["aten::Mul"]["entity_evidence"]["basis"], "ambiguous_name_match") + self.assertNotIn("matched_profiled_kernel", by_name["aten::Mul"]["entity_evidence"]) + self.assertEqual(stats["gpu_kernel"], 1) + self.assertEqual(stats["dispatcher_op"], 1) + self.assertEqual(stats["unresolved"], 1) + + def test_c9_collision_resolution_is_merge_order_independent(self): + # The setdefault bug made the collision winner depend on which aggregate was merged first. + # Reversing the merge input order must yield IDENTICAL kinds for every row: neither the + # dispatcher nor the kernel may stamp its kind onto the other in EITHER direction. + host = {"aten::mul": {"calls": 2, "total_us": 5.0, "cat_counts": {"cpu_op": 2}}} + device = {"mul": {"calls": 3, "total_us": 7.0, "cat_counts": {"kernel": 3}}} + rowspec = [{"name": "mul"}, {"name": "aten::mul"}, {"name": "aten::Mul"}] + + def kinds(*aggs): + merged = pp.merge_entity_evidence(*aggs) + rows, _ = pp.annotate_rows([dict(r) for r in rowspec], merged, "torch-trace") + return {r["name"]: r["entity_kind"] for r in rows} + + forward = kinds(host, device) + reverse = kinds(device, host) # merge the device kernel FIRST this time + self.assertEqual(forward, reverse) + self.assertEqual(forward, {"mul": "gpu_kernel", "aten::mul": "dispatcher_op", + "aten::Mul": "unresolved"}) + + # --------------------------------------------------------------------------- # # serving-phase step windows # --------------------------------------------------------------------------- # @@ -627,7 +700,8 @@ def test_blank_cells_default_to_zero_duration_and_one_call(self): d = self._rocprof_dir("Name,Calls,TotalDurationNs\nqux_kernel,,\n") agg, total_us, launches = pp.parse_rocprof_dir(d) self.assertEqual(agg["qux_kernel"], {"calls": 1, "total_us": 0.0, - "shapes": set(), "dtypes": set()}) + "shapes": set(), "dtypes": set(), + "src": "rocprof_kernel_stats"}) self.assertEqual((total_us, launches), (0.0, 1)) def test_repeated_kernel_rows_accumulate(self): diff --git a/e2e_workflow/scripts/tests/test_seam_contract.py b/e2e_workflow/scripts/tests/test_seam_contract.py new file mode 100644 index 000000000..c3c8c3ff0 --- /dev/null +++ b/e2e_workflow/scripts/tests/test_seam_contract.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""Regression tests for the live-seam binding contract.""" +import contextlib +import importlib.util +import inspect +import io +import json +import os +import shutil +import sys +import tempfile +import types +import unittest + + +SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SPEC = importlib.util.spec_from_file_location( + "seam_contract", os.path.join(SCRIPTS_DIR, "seam_contract.py")) +sc = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(sc) + + +def _descriptor(fn, seam="fixture:entry", evidence=True): + runtime = {"inplace_params": [], "returns_none": False} + if evidence: + runtime["hidden_context"] = [] + return sc._describe_signature( + inspect.signature(fn), seam, fn.__name__, None, + {"seam_runtime_evidence": runtime}) + + +class TestBindingCompatibility(unittest.TestCase): + def test_identical_positional_only_signature_is_bindable(self): + def entry(a, b, /, c=None): + return a + + desc = _descriptor(entry) + verdict = sc.check_binding(desc, desc) + self.assertTrue(verdict["bindable"], verdict["mismatches"]) + + def test_rendered_entry_preserves_positional_only_marker(self): + def entry(a, b, /, c=None): + return a + + namespace = {} + exec(sc.render_entry(_descriptor(entry)), namespace) + self.assertEqual(str(inspect.signature(namespace["entry"])), "(a, b, /, c=None)") + + def test_missing_hidden_context_evidence_fails_closed(self): + def entry(x): + return x + + desc = _descriptor(entry, evidence=False) + verdict = sc.check_binding(desc, _descriptor(entry)) + self.assertFalse(verdict["bindable"]) + self.assertIn("hidden_context_inputs", verdict["codes"]) + + def test_null_hidden_context_is_not_explicit_purity_evidence(self): + def entry(x): + return x + + desc = sc._describe_signature( + inspect.signature(entry), "fixture:entry", "entry", None, + {"seam_runtime_evidence": {"hidden_context": None}}) + self.assertEqual(desc["hidden_context_evidence"], "unknown") + self.assertFalse(sc.check_binding(desc, _descriptor(entry))["bindable"]) + + def test_dropped_optional_parameter_is_rejected(self): + def live(a, optional=None): + return a + + def candidate(a): + return a + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertFalse(verdict["bindable"]) + self.assertIn("optional_param_dropped", verdict["codes"]) + + # ---- C6 case #2: POSITIONAL_OR_KEYWORD live param declared POSITIONAL_ONLY on candidate. + # The reviewer's own second false-positive. live f(a, b) accepts a keyword call f(a=.., b=..); + # a candidate f(a, b, /) rejects that call, so it CANNOT be rebound at a keyword call site even + # though required names + arity line up. The representative-call bind simulation must catch it. + # Descriptors carry hidden_context=[] (via _descriptor) so the strict hidden_context gate does + # not mask the param-kind logic under test. + def test_positional_or_keyword_bound_as_positional_only_is_rejected(self): + def live(a, b): + return a + + def candidate(a, b, /): + return a + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertFalse(verdict["bindable"], verdict["mismatches"]) + self.assertIn("param_kind_mismatch", verdict["codes"]) + # It is specifically the keyword-call surface (not arity/name) that fails. + self.assertNotIn("arity_mismatch", verdict["codes"]) + self.assertNotIn("param_name_mismatch", verdict["codes"]) + + def test_reordered_positional_parameters_are_accepted(self): + # live f(a, b) vs candidate f(b, a): every representative live call (positional AND keyword) + # still binds against the candidate, so this is harmless and PASSES. + def live(a, b): + return a + + def candidate(b, a): + return b + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertTrue(verdict["bindable"], verdict["mismatches"]) + + def test_candidate_varargs_varkw_swallows_live_signature(self): + # A candidate that accepts (*args, **kwargs) can absorb every representative live call, so the + # varargs/varkw candidate binds. Exercises the varargs/varkw short-circuits in check_binding. + def live(a, b): + return a + + def candidate(*args, **kwargs): + return args + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertTrue(verdict["bindable"], verdict["mismatches"]) + + def test_live_varargs_not_absorbed_by_fixed_candidate_is_rejected(self): + # live f(a, *args) accepts an extra positional; the representative-call simulation appends one + # (include_optional leg), and a fixed-arity candidate f(a) rejects it -> param_kind_mismatch. + def live(a, *args): + return a + + def candidate(a): + return a + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertFalse(verdict["bindable"], verdict["mismatches"]) + self.assertIn("param_kind_mismatch", verdict["codes"]) + + def test_live_varkw_not_absorbed_by_fixed_candidate_is_rejected(self): + # live f(a, **kwargs) accepts an extra keyword; the representative-call simulation adds one, and + # a fixed candidate f(a) rejects it -> param_kind_mismatch via the bind simulation. + def live(a, **kwargs): + return a + + def candidate(a): + return a + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertFalse(verdict["bindable"], verdict["mismatches"]) + self.assertIn("param_kind_mismatch", verdict["codes"]) + + def test_positional_to_keyword_only_is_rejected_deliberate_safe_direction(self): + # DOCUMENTED CURRENT BEHAVIOR: live f(a, b) vs candidate f(a, *, b). The reviewer flagged this + # as harmless (a keyword-only candidate param can still receive the live positional value by + # name), but the revised code intentionally FAILS CLOSED: a representative live call passes b + # positionally, which the keyword-only candidate rejects -> param_kind_mismatch. This is a + # deliberate safe-direction over-rejection (better to reject a bindable candidate than to admit + # an unbindable one); this test pins that intended behavior, not an accidental one. + def live(a, b): + return a + + def candidate(a, *, b): + return a + + verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) + self.assertFalse(verdict["bindable"], verdict["mismatches"]) + self.assertIn("param_kind_mismatch", verdict["codes"]) + + # ---- C7 edge: hidden_context that is neither a list nor None. Only None was covered; a dict or a + # str must ALSO be treated as unknown (not "declared") and therefore fail closed as not bindable. + def test_hidden_context_dict_is_unknown_and_fails_closed(self): + def entry(x): + return x + + desc = sc._describe_signature( + inspect.signature(entry), "fixture:entry", "entry", None, + {"seam_runtime_evidence": {"inplace_params": [], "returns_none": False, + "hidden_context": {"forward_ctx": "layer"}}}) + self.assertEqual(desc["hidden_context_evidence"], "unknown") + verdict = sc.check_binding(desc, _descriptor(entry)) + self.assertFalse(verdict["bindable"]) + self.assertIn("hidden_context_inputs", verdict["codes"]) + + def test_hidden_context_str_is_unknown_and_fails_closed(self): + def entry(x): + return x + + desc = sc._describe_signature( + inspect.signature(entry), "fixture:entry", "entry", None, + {"seam_runtime_evidence": {"inplace_params": [], "returns_none": False, + "hidden_context": "forward_ctx"}}) + self.assertEqual(desc["hidden_context_evidence"], "unknown") + verdict = sc.check_binding(desc, _descriptor(entry)) + self.assertFalse(verdict["bindable"]) + self.assertIn("hidden_context_inputs", verdict["codes"]) + + +class TestCliBinding(unittest.TestCase): + def test_target_override_updates_descriptor_identity_and_checks_rendered_entry(self): + module = types.ModuleType("seam_contract_fixture") + + def entry(a, /, b=None): + return a + + module.entry = entry + sys.modules[module.__name__] = module + self.addCleanup(sys.modules.pop, module.__name__, None) + task = tempfile.mkdtemp(prefix="seam_contract_") + self.addCleanup(shutil.rmtree, task, True) + with open(os.path.join(task, "meta.json"), "w") as fh: + json.dump({ + "target_callable": "wrong.module:entry", + "seam_runtime_evidence": { + "inplace_params": [], + "returns_none": False, + "hidden_context": [], + }, + }, fh) + + output = io.StringIO() + with contextlib.redirect_stdout(output): + rc = sc.main([ + "--task-dir", task, + "--target-spec", "seam_contract_fixture:entry", + "--mode", "binding", + "--json", + ]) + result = json.loads(output.getvalue()) + self.assertEqual(rc, 0) + self.assertEqual(result["binding_descriptor"]["seam"], "seam_contract_fixture:entry") + self.assertEqual(result["binding_check"]["candidate"], "") + self.assertTrue(result["binding_check"]["bindable"]) + + def _install_fixture_module(self, name, **members): + module = types.ModuleType(name) + for attr, fn in members.items(): + setattr(module, attr, fn) + sys.modules[name] = module + self.addCleanup(sys.modules.pop, name, None) + return module + + def _write_meta(self, **overrides): + task = tempfile.mkdtemp(prefix="seam_contract_") + self.addCleanup(shutil.rmtree, task, True) + meta = { + "seam_runtime_evidence": { + "inplace_params": [], "returns_none": False, "hidden_context": [], + }, + } + meta.update(overrides) + with open(os.path.join(task, "meta.json"), "w") as fh: + json.dump(meta, fh) + return task + + # ---- C10 NEGATIVE: --baseline-spec must NEVER leak into the binding descriptor. The descriptor is + # built from the deployment TARGET; the baseline only steers baseline_validation. (The positive + # direction -- --target-spec drives the descriptor -- is covered above.) + def test_baseline_spec_does_not_leak_into_binding_descriptor(self): + def entry(a, /, b=None): + return a + + def baseline(x, y, z): # deliberately a DIFFERENT callable / different arity + return x + + self._install_fixture_module("seam_c10_target", entry=entry) + self._install_fixture_module("seam_c10_baseline", baseline=baseline) + task = self._write_meta( + target_callable="wrong.module:entry", baseline_callable="wrong.module:baseline") + + output = io.StringIO() + with contextlib.redirect_stdout(output): + sc.main([ + "--task-dir", task, + "--target-spec", "seam_c10_target:entry", + "--baseline-spec", "seam_c10_baseline:baseline", + "--mode", "both", + "--json", + ]) + result = json.loads(output.getvalue()) + # The descriptor is the TARGET seam, never the baseline. + self.assertEqual(result["binding_descriptor"]["seam"], "seam_c10_target:entry") + self.assertNotEqual(result["binding_descriptor"]["seam"], "seam_c10_baseline:baseline") + # Descriptor is built from the target's real signature, not the baseline's (x, y, z). + self.assertTrue(result["binding_descriptor"]["signature"].endswith("(a, /, b=None)")) + # --baseline-spec only steers baseline_validation. + self.assertEqual( + result["baseline_validation"]["baseline_callable"], "seam_c10_baseline:baseline") + self.assertEqual(result["baseline_validation"]["target_callable"], "seam_c10_target:entry") + + # ---- C2: '--mode both' with NO --candidate must still emit a binding_check against the rendered + # entry (candidate == ""). The existing coverage used '--mode binding'. + def test_mode_both_without_candidate_checks_rendered_entry(self): + def entry(a, /, b=None): + return a + + self._install_fixture_module("seam_c2_target", entry=entry) + task = self._write_meta(target_callable="seam_c2_target:entry") + + output = io.StringIO() + with contextlib.redirect_stdout(output): + sc.main([ + "--task-dir", task, + "--target-spec", "seam_c2_target:entry", + "--mode", "both", + "--json", + ]) + result = json.loads(output.getvalue()) + self.assertIn("binding_check", result) + self.assertEqual(result["binding_check"]["candidate"], "") + self.assertTrue(result["binding_check"]["bindable"], result["binding_check"]["mismatches"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From d5e132cb4d3df565237b7c6c5beb1a132449a607 Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Wed, 19 Aug 2026 21:06:30 +0000 Subject: [PATCH 3/8] replace rejection gates with live kernel discovery Co-authored-by: Cursor --- e2e_workflow/e2e_workflow.js | 936 ++++-------------- e2e_workflow/roles/e2e_integrator.md | 34 +- e2e_workflow/roles/kernel_extractor.md | 163 +-- e2e_workflow/roles/op_benchmarker.md | 29 - e2e_workflow/roles/profiler.md | 44 +- e2e_workflow/roles/system_architect.md | 35 +- e2e_workflow/scripts/capture_shapes.py | 49 +- .../scripts/check_schema_consumption.py | 204 ---- e2e_workflow/scripts/kernel_selection.py | 360 +++++++ e2e_workflow/scripts/op_bench.py | 479 +-------- e2e_workflow/scripts/overlay_setup.py | 35 +- e2e_workflow/scripts/parse_profile.py | 177 +++- e2e_workflow/scripts/seam_contract.py | 651 ------------ e2e_workflow/scripts/seam_trace.py | 186 ++++ .../scripts/tests/test_capture_shapes.py | 11 +- .../scripts/tests/test_kernel_selection.py | 317 ++++++ e2e_workflow/scripts/tests/test_op_bench.py | 403 +------- .../scripts/tests/test_overlay_setup.py | 23 +- .../scripts/tests/test_parse_profile.py | 72 ++ .../scripts/tests/test_seam_contract.py | 311 ------ e2e_workflow/scripts/tests/test_seam_trace.py | 138 +++ 21 files changed, 1642 insertions(+), 3015 deletions(-) delete mode 100644 e2e_workflow/scripts/check_schema_consumption.py create mode 100644 e2e_workflow/scripts/kernel_selection.py delete mode 100644 e2e_workflow/scripts/seam_contract.py create mode 100644 e2e_workflow/scripts/seam_trace.py create mode 100644 e2e_workflow/scripts/tests/test_kernel_selection.py delete mode 100644 e2e_workflow/scripts/tests/test_seam_contract.py create mode 100644 e2e_workflow/scripts/tests/test_seam_trace.py diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index 262141d8c..e637cc8c8 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -178,21 +178,6 @@ const HEAD_CORRECTIVE_MAX = parseInt(A.head_corrective_max != null ? A.head_corr // the heavy re-author only when the surgical patch fails. surgical_fix=false => old heavy-only behavior. const SURGICAL_FIX = String(A.surgical_fix != null ? A.surgical_fix : 'true') === 'true'; const FIXABLE_REJECT_RX = /cuda_graph_capture_unsafe|no[_ ]?binary|NO_BINARY_FOR_GPU|hipErrorNoBinaryForGpu|capture[_ ]?(unsafe|hang)|host[_ ]?sync|graph[_ ]?capture|no[_ ]?rebind[_ ]?seam|no[_ ]?engagement|not[_ ]?engaged|signature[_ ]?mismatch|wrong[_ ]?seam/i; -// ---- DEGRADATION LEDGER (INV-3: no silent degradation) ---------------------------------------------- -// Every place the orchestrator accepts a WEAKER input than the contract asks for — a missing machine -// verdict, a fallback field, an unrecognized code — records here instead of quietly carrying on. The -// ledger is emitted with the run report, so "we ran with 3 contracts unenforced" is visible in the -// artifact rather than inferable only by reading the source. GENERIC: no op/kernel/model specifics. -const DEGRADATIONS = []; -// Heads rejected by an admission gate that runs BEFORE `flaggedHeads` exists (Strategize-time entity-kind -// admission). Merged into flaggedHeads so they are surfaced by the same "never silently skipped" path. -const PRE_FLAGGED_HEADS = []; -function noteDegradation(where, what, detail) { - const d = { where, what, detail: detail || '' }; - DEGRADATIONS.push(d); - log(` [degraded] ${where}: ${what}${detail ? ` — ${detail}` : ''}`); - return d; -} // ---- CORRECTNESS-class reject (auto-correct) -------------------------------------------------------- // A SECOND fix-and-retryable class: the candidate ENGAGED and beat the isolated oracle but produces the // WRONG output on the LIVE path (parity/accuracy failure) — OR posts an IMPLAUSIBLE e2e speedup (faster @@ -234,94 +219,18 @@ function isImplausibleSpeedup(pct_gpu_time, isolated, integ) { if (!Number.isFinite(ceilPct)) return false; return ((integ && integ.e2e_delta_pct) || 0) > ceilPct * (1 + IMPLAUSIBLE_SPEEDUP_MARGIN) + 1e-9; } -// ---- STRUCTURED REJECT CODES (INV-4) + OWNER-STAGE ROUTING (INV-5) --------------------------------- -// A reject carries a CODE from a closed set, not a sentence. Two things are read off that code: -// cls — which corrective instruction applies ('' = terminal, not auto-correctable); -// stage — WHICH PIPELINE STAGE OWNS THE DEFECT, i.e. where a fix has to re-enter. -// The stage column is the part that was missing. Every corrective used to re-enter at `author` -// (kernel_workflow mode:'optimize' on the SAME task dir), but a task dir's unittest.py is immutable by -// contract — so any defect whose fix requires a different call contract or a different seam is -// literally unfixable there, and re-authoring burns hours to arrive back at the same reject. Routing by -// owner stage means an extract-owned defect re-extracts and an author-owned defect re-authors. -// GENERIC — the table is keyed on failure MODE, never on op kind, backend, model or kernel. -const REJECT_CODES = { - // --- owned by EXTRACT: the task itself encodes the wrong seam / wrong contract / wrong denominator - no_rebind_seam: { cls: 'integration', stage: 'extract' }, - signature_mismatch: { cls: 'integration', stage: 'extract' }, - arity_mismatch: { cls: 'integration', stage: 'extract' }, - param_name_mismatch: { cls: 'integration', stage: 'extract' }, - return_contract_mismatch: { cls: 'integration', stage: 'extract' }, - optional_param_dropped: { cls: 'integration', stage: 'extract' }, - param_kind_mismatch: { cls: 'integration', stage: 'extract' }, - hidden_context_inputs: { cls: 'integration', stage: 'extract' }, - seam_mismatch: { cls: 'integration', stage: 'extract' }, - candidate_unresolvable: { cls: 'integration', stage: 'extract' }, - no_seam_descriptor: { cls: 'integration', stage: 'extract' }, - no_engagement: { cls: 'integration', stage: 'extract' }, - wrong_seam: { cls: 'integration', stage: 'extract' }, - invalid_denominator: { cls: 'integration', stage: 'extract' }, - // --- owned by AUTHOR: the kernel is right about the seam, wrong about posture or numerics - cuda_graph_capture_unsafe: { cls: 'integration', stage: 'author' }, - no_binary_for_gpu: { cls: 'integration', stage: 'author' }, - capture_hang: { cls: 'integration', stage: 'author' }, - host_sync_in_hot_path: { cls: 'integration', stage: 'author' }, - oom: { cls: 'integration', stage: 'author' }, - parity_regression: { cls: 'correctness', stage: 'author' }, - accuracy_regression: { cls: 'correctness', stage: 'author' }, - output_corruption: { cls: 'correctness', stage: 'author' }, - implausible_speedup: { cls: 'correctness', stage: 'author' }, - // --- owned UPSTREAM of the kernel track: no amount of kernel work fixes these - wrong_head_granularity: { cls: '', stage: 'profile' }, - delegated_track_disabled: { cls: '', stage: 'strategize' }, - // --- terminal by construction: a correct kernel with no headroom is not a defect - no_win: { cls: '', stage: '' }, - do_no_harm: { cls: '', stage: '' }, -}; -const normalizeRejectCode = (c) => String(c || '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, ''); -// Last-resort recovery of a code from free prose, used ONLY when the integrator returned no -// reason_code at all (older role revision). Scans for an explicit code token FIRST — this is the fix -// for the precedence bug where CORRECTNESS_REJECT_RX's bare `mismatch` swallowed `signature_mismatch` -// and made that FIXABLE_REJECT_RX branch permanently unreachable. -function rejectCodeFromProse(reason) { - const r = String(reason || ''); - for (const code of Object.keys(REJECT_CODES)) { - if (new RegExp(`(^|[^a-z0-9])${code.replace(/_/g, '[_ ]?')}([^a-z0-9]|$)`, 'i').test(r)) return code; - } - return ''; -} -// Resolve {cls, stage, code, provenance} for a reject. `code` is the integrator's structured -// reason_code when present. Fails closed on an UNKNOWN structured code (we cannot route what we cannot -// name) and records the degradation when it has to fall back to prose. -function rejectVerdict(reason, code) { - const c = normalizeRejectCode(code); - if (c) { - if (REJECT_CODES[c]) return { ...REJECT_CODES[c], code: c, provenance: 'structured' }; - noteDegradation('rejectVerdict', `unknown reason_code '${c}' — not in REJECT_CODES`, - 'treating as terminal; add the code to the table and to INTEGRATE_SCHEMA.reason_code'); - return { cls: '', stage: '', code: c, provenance: 'unknown_code' }; - } - const fromProse = rejectCodeFromProse(reason); - if (fromProse) { - noteDegradation('rejectVerdict', 'integrator returned no reason_code', - `recovered '${fromProse}' from the prose reason`); - return { ...REJECT_CODES[fromProse], code: fromProse, provenance: 'prose_code' }; - } +// Classify a reject reason into a fix-and-retry class ('' = terminal, not auto-correctable). +function rejectClass(reason) { const r = reason || ''; - const cls = CORRECTNESS_REJECT_RX.test(r) ? 'correctness' : (FIXABLE_REJECT_RX.test(r) ? 'integration' : ''); - if (r) noteDegradation('rejectVerdict', 'no reason_code and no recognizable code token in the prose', - `regex fallback -> class='${cls || 'terminal'}', owner stage UNKNOWN (corrective will re-enter at author)`); - return { cls, stage: cls ? 'author' : '', code: '', provenance: 'prose_regex' }; + if (CORRECTNESS_REJECT_RX.test(r)) return 'correctness'; + if (FIXABLE_REJECT_RX.test(r)) return 'integration'; + return ''; } -// Classify a reject reason into a fix-and-retry class ('' = terminal, not auto-correctable). -function rejectClass(reason, code) { return rejectVerdict(reason, code).cls; } -// Which pipeline stage owns this defect — i.e. where a corrective must re-enter to have any chance. -function rejectStage(reason, code) { return rejectVerdict(reason, code).stage; } // A gate 'accept'/'stack' only counts as a REAL win if the measured e2e delta is not an implausible // (corruption) speedup. Centralizes the guard so every integrate site treats a too-good-to-be-true // delta as a reject instead of banking it. GENERIC (uses only pct_gpu_time + isolated speedup). function integAccepted(integ, pct_gpu_time, isolated) { return !!(integ && (integ.gate === 'accepted' || integ.gate === 'stack') - && provenanceOk(integ, 'integrate') // INV-3: consume the integrator's own provenance claim && !isImplausibleSpeedup(pct_gpu_time, isolated, integ)); } // The reason string to feed the corrective loop: if the gate "passed" but the delta is impossible, emit @@ -561,28 +470,16 @@ const EXTRACT_OP_SCHEMA = obj({ target_callable: { type: 'string' }, // module:attr rebind seam for an authored kernel ('' if none) baseline_callable: { type: 'string' }, // module:attr of the FROZEN real online kernel (the speedup denominator) baseline_frozen: { type: 'boolean' }, // true only when baseline_src/ was frozen OR baseline_callable resolves - // MACHINE VERDICTS from scripts/seam_contract.py — pasted verbatim, never hand-written. These are the - // fields hasFrozenBaseline()/seamBindable() actually consume; without them the head fails closed. - baseline_validation: { type: 'object', additionalProperties: true }, // --mode baseline verdict (INV-1) - binding_descriptor: { type: 'object', additionalProperties: true }, // --mode binding descriptor (INV-2) - binding_check: { type: 'object', additionalProperties: true }, // entry-vs-seam bindability (INV-2) - entry_contract_path: { type: 'string' }, // --mode entry generated contract + device_kernel: { type: 'string' }, + seam_candidates: arrObj, + selection_validation: { type: 'object', additionalProperties: true }, smoke: { type: 'string' }, notes: { type: 'string' }, }, ['op_kind', 'task_dir', 'smoke']); const OPBENCH_SCHEMA = obj({ short_name: { type: 'string' }, op_kind: { type: 'string' }, provenance_ok: { type: 'boolean' }, winner_backend: { type: 'string' }, winner_kind: { type: 'string' }, - // null, NOT a number, when op_bench.py withheld the ratio: the denominator was not the live path, so - // there is no speedup to report. A withheld measurement must not be representable as `1.0` or as the - // unpublishable figure — both read as claims. - isolated_speedup: { type: ['number', 'null'] }, winner_editable: { type: 'boolean' }, - // INV-1. What the candidate was divided BY, and whether op_bench.py refused to publish the ratio. - // Copied from opbench_result.json; the gate below will not bank a win on an unsound denominator. - denominator: { type: 'string', enum: [ - 'measured_backend_default', 'verified_baseline', 'unverified_baseline', - 'target_fallback', 'synthesized_reference', 'none'] }, - speedup_withheld: { type: 'boolean' }, + isolated_speedup: { type: 'number' }, winner_editable: { type: 'boolean' }, best_known_ms: { type: 'number' }, recommend_tier_c: { type: 'boolean' }, author_plan: arrObj, tuning_artifact: { type: 'string' }, apply_env: { type: 'string' }, apply_flags: { type: 'string' }, code_patch: { type: 'string' }, @@ -596,14 +493,11 @@ const EXTRACT_SCHEMA = obj({ source_path_in_sglang: { type: 'string' }, target_callable: { type: 'string' }, num_cases: { type: 'number' }, regimes_captured: arrStr, candidate_backends: arrStr, build: { type: 'boolean' }, unittest_smoke: { type: 'string' }, - synthesized: { type: 'boolean' }, // true when the oracle was fabricated -> can never be the denominator baseline_callable: { type: 'string' }, // module:attr of the FROZEN real online kernel (the speedup denominator) baseline_frozen: { type: 'boolean' }, // true only when baseline_src/ was frozen OR baseline_callable resolves - // Same machine verdicts as EXTRACT_OP_SCHEMA — see the note there. - baseline_validation: { type: 'object', additionalProperties: true }, - binding_descriptor: { type: 'object', additionalProperties: true }, - binding_check: { type: 'object', additionalProperties: true }, - entry_contract_path: { type: 'string' }, + device_kernel: { type: 'string' }, + seam_candidates: arrObj, + selection_validation: { type: 'object', additionalProperties: true }, reference_io_sha256: { type: 'string' }, notes: { type: 'string' }, }, ['editable', 'task_dir', 'unittest_smoke']); @@ -629,18 +523,6 @@ const INTEGRATE_SCHEMA = obj({ // implausible-speedup guard only distrusts an 'accuracy'/soft accept; a byte_exact accept is trusted. parity_kind: { type: 'string' }, gate: { type: 'string', enum: ['accepted', 'stack', 'rejected', 'incomplete'] }, - // STRUCTURED reject code (INV-4). REQUIRED whenever gate='rejected'. The orchestrator routes the - // corrective off THIS, not off the prose in `reason` — classifying a reject by regex over a sentence - // an LLM wrote is not a decision procedure (it made the `signature_mismatch` branch unreachable, - // because the bare `mismatch` token in the correctness regex matched first). `reason` stays as the - // human-readable detail. The enum is the closed set in REJECT_CODES. - reason_code: { type: 'string', enum: [ - 'no_rebind_seam', 'signature_mismatch', 'arity_mismatch', 'param_name_mismatch', - 'return_contract_mismatch', 'optional_param_dropped', 'param_kind_mismatch', - 'hidden_context_inputs', 'seam_mismatch', 'no_engagement', 'wrong_seam', - 'invalid_denominator', 'cuda_graph_capture_unsafe', 'no_binary_for_gpu', 'capture_hang', - 'host_sync_in_hot_path', 'oom', 'parity_regression', 'accuracy_regression', 'output_corruption', - 'implausible_speedup', 'wrong_head_granularity', 'delegated_track_disabled', 'no_win', 'do_no_harm'] }, accepted_overlay: { type: 'string' }, reason: { type: 'string' }, }, ['gate', 'e2e_throughput_tok_s']); @@ -831,346 +713,144 @@ async function ensureFlydslGate() { } } -// ---- INV-1: DENOMINATOR IDENTITY ------------------------------------------------------------------ -// A frozen baseline is the speedup DENOMINATOR, so the only question that matters is whether the thing -// it names is the code the live server actually runs. That is not decidable from the string: a -// task-local `baseline_src.xxx_ref:forward` scaffold the extractor wrote itself is a perfectly -// well-formed non-empty string, and against it any authored kernel posts a large, meaningless win that -// cannot carry end-to-end. So the predicate now consumes the MACHINE VERDICT from the vendored -// validator (scripts/seam_contract.py --mode baseline), which resolves the callable and checks that it -// (a) imports from outside the task/eval dir, (b) lives in an installed distribution, (c) is the seam -// itself or an OBSERVED callee of it, and (d) was not flagged `synthesized`. Op-kind agnostic — the -// validator never asks what kind of op this is. -// Strict by default: no machine verdict => not a frozen baseline (fail closed). baseline_contract_strict -// =false restores the old string check for a legacy role revision, and records the degradation. -const BASELINE_CONTRACT_STRICT = String(A.baseline_contract_strict != null ? A.baseline_contract_strict : 'true') === 'true'; -// An oracle with zero recorded cases, or one whose bytes nobody hashed, cannot certify anything the -// task later claims. Both facts are already REQUESTED in EXTRACT_SCHEMA (num_cases, -// reference_io_sha256) and were, until now, read by nothing — see scripts/check_schema_consumption.py. -function oracleProvenance(ext) { - const problems = []; - // A fabricated oracle (synthesized:true) is handled as its OWN case downstream (baselineDefect -> - // 'synthesized': the RATIO is withheld, but the kernel can still be accepted on a measured e2e A/B — - // ten such wins live in the 88-run archive). It has no captured live calls to count or bytes to pin, - // so the num_cases/sha256 provenance requirements do not apply to it. - const synth = ext.synthesized === true; - // A claimed captured oracle must carry complete identity evidence. Partial/failed extractions never - // reach this acceptance predicate, so missing fields must fail closed here rather than be treated as - // harmless schema degradation. op_bench.py independently recomputes the digest from the bytes. - if (!synth) { - if (!Number.isFinite(Number(ext.num_cases)) || Number(ext.num_cases) <= 0) - problems.push('num_cases missing/zero (the oracle recorded no provable calls)'); - const sha = typeof ext.reference_io_sha256 === 'string' - ? ext.reference_io_sha256.trim().toLowerCase() : ''; - if (!/^[0-9a-f]{64}$/.test(sha)) - problems.push('reference_io_sha256 missing/invalid (oracle bytes are not identity-pinned)'); - } - return { ok: problems.length === 0, problems }; +// A profile identity is a device symbol; the replacement target is a live Python callable. Keep the +// two machine fields separate and require runtime evidence before bake-off or authoring. +const CALLABLE_SPEC_RX = /^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$/; +const validCallableSpec = (s) => CALLABLE_SPEC_RX.test(String(s || '').trim()); +const canonicalDeviceKernel = (s) => String(s || '').trim().toLowerCase() + .replace(/\[clone[^\]]*\]/g, '').replace(/<.*>/g, '').split('(', 1)[0] + .split('::').pop().replace(/[^a-z0-9_]+/g, ''); +function kernelIdentitiesMatch(a, b) { + const x = canonicalDeviceKernel(a), y = canonicalDeviceKernel(b); + return !!x && !!y && (x === y || + (x.length >= 6 && new RegExp(`(?:^|_)${x.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:$|_)`).test(y)) || + (y.length >= 6 && new RegExp(`(?:^|_)${y.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:$|_)`).test(x))); } - -// `provenance_ok` is the agent's own assertion that it re-hashed the oracle / confirmed the baseline -// before reporting a number. It is REQUIRED by both OPBENCH_SCHEMA and INTEGRATE_SCHEMA and was read -// nowhere: an agent could truthfully report provenance_ok=false and still have its speedup banked. -// An explicit false is now disqualifying; an absent value is a recorded degradation, not a pass. -function provenanceOk(res, where) { - if (!res || typeof res !== 'object' || res.provenance_ok !== true) { - log(` ⚠️ ${where}: provenance_ok is not true — the number is unsourced; not banking it`); - return false; - } - return true; +function requiredDeviceKernel(h) { + if (!h || h.entity_kind !== 'gpu_kernel') return ''; + return String(h.device_kernel || h.profile_kernel || + (h.entity_evidence && h.entity_evidence.matched_profiled_kernel) || + h.short_name || h.name || '').trim(); } - -// INV-1 at the ORCHESTRATOR boundary. op_bench.py already withholds a ratio it cannot stand behind, but -// the orchestrator must not depend on the script having been the thing that produced this JSON — an -// agent can hand-assemble the object, and the 4.47x-with-zero-e2e case is precisely a number that was -// divided by something the server never calls. So re-check the declared denominator here and fail -// closed. GENERIC: keyed on the provenance of the measurement, never on op kind or kernel name. -const BANKABLE_DENOMINATORS = ['measured_backend_default', 'verified_baseline']; -function denominatorSound(bake, where) { - if (!bake || typeof bake !== 'object') return false; - if (bake.speedup_withheld === true) { - log(` ⚠️ ${where}: op_bench WITHHELD the speedup (denominator=${bake.denominator || 'unknown'}) — not banking a win`); - return false; +const candidateSpec = (c) => + String((c && (c.target_callable || c.callable || c.spec)) || '').trim(); +function selectionCandidatesForHead(h, ext) { + const required = requiredDeviceKernel(h); + const fused = !!(h && (h.is_fused_kernel === true || h.op_kind === 'moe')); + const bySpec = new Map(); + for (const candidate of [ + ...((h && h.seam_candidates) || []), + ...((ext && ext.seam_candidates) || []), + ]) { + const spec = candidateSpec(candidate); + // Architect classifications win for an existing spec; Extractor may append missing inner launchers. + if (spec && !bySpec.has(spec)) bySpec.set(spec, candidate); } - const d = bake.denominator; - if (d == null) { - log(` ⚠️ ${where}: bake-off result declares no denominator — not banking an unauditable ratio`); - return false; - } - if (!BANKABLE_DENOMINATORS.includes(d)) { - log(` ⚠️ ${where}: denominator='${d}' is not the live path — the ratio is unbankable`); - return false; - } - return true; + return Array.from(bySpec.values()).filter((candidate) => { + if (!candidate || typeof candidate !== 'object' || !validCallableSpec(candidateSpec(candidate))) + return false; + const role = String(candidate.role || '').toLowerCase(); + if (role === 'kernel_entry') return false; // native/JIT object identity is unsafe to monkeypatch + const kernels = Array.isArray(candidate.device_kernels) ? candidate.device_kernels : []; + if (required && !kernels.some((kernel) => kernelIdentitiesMatch(required, kernel))) return false; + return (fused + ? ['outer_wrapper', 'dispatcher', 'op_seam'] + : ['outer_wrapper', 'dispatcher', 'op_seam', 'inner_launcher']).includes(role); + }); } - -// A denominator fails in two DIFFERENT ways and they must not share a consequence: -// 'missing' -- nothing to divide by at all (no baseline_callable, or an oracle with zero cases / -// unpinned bytes). Neither the ratio nor the correctness verdict means anything. -// 'synthesized' -- the reference was written by the extractor. The RATIO is worthless; the KERNEL is -// not. Whether it is actually faster is settled downstream by the live e2e A/B, -// which divides by the real server and by nothing the extractor authored. -// Replaying the 88 archived runs settles which consequence belongs to which: 113 of 210 real -// extractions declared synthesized:true, and TEN entries those runs went on to ACCEPT on a MEASURED -// e2e A/B came from one (+23.3%, +12.6%, +6.5%, +5.0%, ... see geak_inv_verify/corpus_gates.py). -// Killing the extraction on 'synthesized' would have thrown all ten away. Withholding the ratio costs -// nothing, because the ratio was never what those wins were banked on. -function baselineDefect(ext) { - if (!ext) return 'missing'; - if (!oracleProvenance(ext).ok) return 'missing'; - const v = ext.baseline_validation; - if (v && typeof v === 'object' && v.contract === 'baseline_identity') - return v.ok === true ? 'none' : 'missing'; - if (ext.synthesized === true) return 'synthesized'; - return 'none'; +function prepareHeadSelection(h) { + if (!h || typeof h !== 'object') return h; + const out = { ...h }; + const required = requiredDeviceKernel(out); + if (required && !out.device_kernel) out.device_kernel = required; + // live_call_seam and malformed target_callable values are prose, never machine targets. + if (!validCallableSpec(out.target_callable)) out.target_callable = ''; + const fused = out.is_fused_kernel === true || out.op_kind === 'moe'; + const deployable = selectionCandidatesForHead(out).filter((candidate) => { + const role = String(candidate.role || '').toLowerCase(); + return fused ? role === 'op_seam' : ['inner_launcher', 'op_seam'].includes(role); + }); + deployable.sort((a, b) => Number(b.depth || 0) - Number(a.depth || 0)); + out.target_callable = deployable.length ? candidateSpec(deployable[0]) : ''; + out.selection_status = out.target_callable + ? 'candidate_selected_needs_runtime_verification' + : 'extractor_must_discover_live_launcher'; + return out; } - -const hasFrozenBaseline = (ext) => { - if (!ext) return false; - const op = oracleProvenance(ext); - if (!op.ok) { - log(` [inv-1] oracle provenance FAILED: ${op.problems.join('; ')}`); - return false; - } - const v = ext.baseline_validation; - if (v && typeof v === 'object' && v.contract === 'baseline_identity') { - if (v.ok === true) { - // C5: the verdict certifies the spec IT names. A stale/foreign verdict (from a prior retry or - // another task) whose callable differs must NOT certify this baseline. Cross-check the callables. - const vb = String(v.baseline_callable || '').trim(), eb = String(ext.baseline_callable || '').trim(); - const vt = String(v.target_callable || '').trim(), et = String(ext.target_callable || '').trim(); - if (!vb || !eb || vb !== eb) { - log(` [inv-1] baseline_validation ok=true but certifies baseline '${vb}', not this task's '${eb}' — stale/foreign verdict, not a frozen baseline`); - return false; - } - if (!vt || !et || vt !== et) { - log(` [inv-1] baseline_validation ok=true but certifies target '${vt}', not this task's '${et}' — stale/foreign verdict, not a frozen baseline`); - return false; - } - return true; - } - log(` [inv-1] baseline_validation FAILED: ${(v.failed || []).join(', ') || 'unknown'} ` + - `(baseline_callable=${v.baseline_callable || "''"}, origin=${(v.baseline_origin || {}).kind || '?'})`); - return false; - } - // No machine verdict at all. - if (ext.synthesized === true) { // INV-3: a declared field that is now CONSUMED - noteDegradation('hasFrozenBaseline', 'extraction reports synthesized:true and no baseline_validation', - 'a fabricated oracle can never be the denominator — rejecting the extraction'); - return false; - } - const legacyOk = !!(ext.baseline_frozen === true || - (typeof ext.baseline_callable === 'string' && ext.baseline_callable.trim() !== '')); - noteDegradation('hasFrozenBaseline', 'extraction returned no baseline_validation (seam_contract.py not run)', - BASELINE_CONTRACT_STRICT - ? 'baseline_contract_strict=true -> treating as NO frozen baseline (fail closed)' - : `baseline_contract_strict=false -> falling back to the string check (=${legacyOk}); the ` + - 'denominator is UNVERIFIED and any isolated speedup from this task is unaudited'); - return BASELINE_CONTRACT_STRICT ? false : legacyOk; -}; - -// ---- INV-2: BINDING CONTRACT ---------------------------------------------------------------------- -// Whether an authored replacement can be bound at the live seam is decided by inspect.signature of the -// LIVE callable, not by an agent's recollection of it. seam_contract.py --mode binding emits the -// descriptor and (given the entry) the bindability verdict; the extractor returns both. A head whose -// seam is not bindable must never enter the author fan-out: the kernel that comes out is unusable no -// matter how fast it is, and the reject only surfaces hours later at the e2e gate. -// Strict by default; seam_contract_strict=false degrades to "unverified but allowed" and records it. -const SEAM_CONTRACT_STRICT = String(A.seam_contract_strict != null ? A.seam_contract_strict : 'true') === 'true'; -function seamBindable(ext, seam) { - if (!ext) return { ok: false, why: 'no extraction' }; - const chk = ext.binding_check, desc = ext.binding_descriptor; - // C5: a binding verdict is only valid for the seam it was computed against. seam_contract.py records - // that seam on both binding_check and binding_descriptor (field `seam`). If the selected deployment - // seam differs, the verdict is stale/foreign and must not admit the head. - const want = String(seam || '').trim(); - const seamMismatch = (obj, kind) => { - // Pre-selection contract-PRESENCE checks (extractWithBaseline's re-extract loop) call seamBindable - // with no seam yet: there is no selected seam to match against, so skip the seam check here and only - // verify a usable binding contract exists. The real admission gate (authoringAdmission) always - // passes the selected seam, so C5's stale/foreign-verdict rejection still fires there. Without this - // guard an empty `want` made every valid extraction look seam-mismatched, forcing pointless - // re-extractions with an unsatisfiable "not the selected seam ''" corrective. - if (!want) return null; - const got = String((obj && obj.seam) || '').trim(); - if (!got || got !== want) - return { ok: false, why: `${kind} certifies seam '${got}', not the selected seam '${want}'`, - codes: ['seam_mismatch'] }; - return null; - }; - // A binding_check cannot replace the descriptor: hidden-context/purity evidence belongs to the live - // seam descriptor and must be present even when the generated entry's signature is bindable. - if (!desc || typeof desc !== 'object' || desc.contract !== 'binding' || desc.ok !== true) { - if (SEAM_CONTRACT_STRICT) - return { ok: false, why: 'missing/unusable binding_descriptor', codes: ['no_seam_descriptor'] }; - } else { - const dm = seamMismatch(desc, 'binding_descriptor'); if (dm) return dm; - if (desc.hidden_context_evidence !== 'declared') - return { ok: false, why: 'hidden-context purity evidence is missing', - codes: ['hidden_context_inputs'] }; - if ((desc.hidden_context || []).length) - return { ok: false, why: `seam reads non-parameter inputs ${JSON.stringify(desc.hidden_context)}`, - codes: ['hidden_context_inputs'] }; - } - if (chk && typeof chk === 'object' && chk.contract === 'binding_check') { - const mm = seamMismatch(chk, 'binding_check'); if (mm) return mm; - const entryPath = typeof ext.entry_contract_path === 'string' ? ext.entry_contract_path.trim() : ''; - return chk.bindable === true - ? { ok: true, why: `binding_check.bindable${entryPath ? `; generated contract=${entryPath}` : ''}` } - : { ok: false, why: `binding_check: ${(chk.codes || []).join(', ') || 'not bindable'}`, - codes: chk.codes || [] }; - } - if (chk && SEAM_CONTRACT_STRICT) - return { ok: false, why: 'malformed binding_check', codes: ['no_seam_descriptor'] }; - if (desc && typeof desc === 'object' && desc.contract === 'binding') { - if (desc.ok !== true) return { ok: false, why: `binding_descriptor unusable: ${desc.error || 'unknown'}` }; - const mm = seamMismatch(desc, 'binding_descriptor'); if (mm) return mm; - if (SEAM_CONTRACT_STRICT) - return { ok: false, why: 'binding_check missing; generated entry was not verified', - codes: ['no_seam_descriptor'] }; - return { ok: true, why: 'legacy mode: seam described but generated entry unchecked' }; - } - noteDegradation('seamBindable', 'extraction returned no binding_descriptor (seam_contract.py not run)', - SEAM_CONTRACT_STRICT - ? 'seam_contract_strict=true -> head is NOT admitted to the author fan-out (fail closed)' - : 'seam_contract_strict=false -> authoring an UNVERIFIED seam; a contract mismatch will only ' + - 'surface at the e2e gate, after the budget is spent'); - return { ok: !SEAM_CONTRACT_STRICT, why: 'no binding contract (unverified)' }; +function kernelSelectionVerified(h, ext) { + const required = requiredDeviceKernel(h); + if (!required) return { ok: true, why: 'not a profiled GPU-head extraction' }; + const target = String((ext && ext.target_callable) || '').trim(); + if (!validCallableSpec(target)) + return { ok: false, why: `target_callable '${target}' is not an exact module:attr spec` }; + const verdict = ext && ext.selection_validation; + if (!verdict || verdict.contract !== 'kernel_selection') + return { ok: false, why: 'kernel_selection.py verdict is missing' }; + if (verdict.ok !== true) + return { ok: false, + why: `kernel_selection.py failed: ${(verdict.failed || []).join(', ') || 'unknown'}` }; + if (String(verdict.target_callable || '').trim() !== target) + return { ok: false, + why: `selection verdict certifies '${verdict.target_callable || ''}', not '${target}'` }; + if (!kernelIdentitiesMatch(required, verdict.device_kernel || ext.device_kernel)) + return { ok: false, + why: `selection verdict certifies '${verdict.device_kernel || ext.device_kernel || ''}', ` + + `not profiled kernel '${required}'` }; + if (Number(verdict.total_calls_observed || 0) <= 0 || + Number(verdict.target_marker_calls || 0) <= 0 || + Number(verdict.matched_kernel_calls || 0) <= 0) + return { ok: false, why: 'selection verdict lacks positive call/marker/kernel evidence' }; + const candidates = selectionCandidatesForHead(h, ext); + if (!candidates.length) + return { ok: false, why: 'no relevant structured seam_candidates were returned' }; + const tested = new Set((verdict.candidate_targets_tested || []).map((value) => String(value || '').trim())); + const omitted = candidates.map(candidateSpec).filter((spec) => !tested.has(spec)); + if (omitted.length) + return { ok: false, why: `selection probe omitted candidate(s): ${omitted.join(', ')}` }; + const selected = candidates.find((candidate) => candidateSpec(candidate) === target); + const selectedRole = String((selected && selected.role) || '').toLowerCase(); + const fused = h.is_fused_kernel === true || h.op_kind === 'moe'; + if (!selected || (fused ? selectedRole !== 'op_seam' + : !['inner_launcher', 'op_seam'].includes(selectedRole))) + return { ok: false, + why: fused + ? `fused head must select the whole-operation op_seam, got '${selectedRole || 'unknown'}'` + : `selected target role '${selectedRole || 'unknown'}' is not a deployable inner/op seam` }; + const live = new Set((verdict.live_candidate_targets || []).map((value) => String(value || '').trim())); + const selectedDepth = Number(selected.depth || 0); + const deeper = candidates.filter((candidate) => + candidateSpec(candidate) !== target && live.has(candidateSpec(candidate)) && + Number(candidate.depth || 0) > selectedDepth); + if (deeper.length) + return { ok: false, + why: `deeper live candidate(s) observed across calls/ranks: ${deeper.map(candidateSpec).join(', ')}` }; + if (verdict.deepest_verified !== true) + return { ok: false, why: 'selected callable is not machine-verified as deepest' }; + return { ok: true, why: `${target} launches profiled kernel ${required}` }; } -// ---- INV-6: ENTITY-KIND ADMISSION (applied at EVERY headQueue (re)assignment) --------------------- -// The head track optimizes GPU KERNELS. A torch DISPATCHER op is a Python-side span whose pct_gpu_time -// is the SUM of the kernels it dispatches; routing one as a head double-counts its Amdahl share and -// points extraction at a wrapper. The filter used to run ONCE, right after the initial strategy — but -// headQueue is ALSO (re)assigned from carried state on a phase resume and from the post-config -// re-strategize, and neither of those was re-admitted. A dispatcher_op arriving by either path reached -// the author fan-out unchecked. admitHeads() is now called at all three sites. -// A TYPE check, not a name blacklist: it needs no aten::/vllm:: prefix knowledge. -const HEAD_ENTITY_KIND_STRICT = String(A.head_entity_kind_strict != null ? A.head_entity_kind_strict : 'true') === 'true'; -function admitHeads(q, stage, flagged) { - q = (q || []).filter(Boolean); - const bad = q.filter((c) => c.entity_kind && c.entity_kind !== 'gpu_kernel'); - const noKind = q.filter((c) => !c.entity_kind); - let admitted = q.filter((c) => !(c.entity_kind && c.entity_kind !== 'gpu_kernel')); - for (const c of bad) { - log(` ⚠️ FLAG ${c.short_name || c.name}: profile row is a ${c.entity_kind}, not a gpu_kernel ` + - `(${c.pct_gpu_time || '?'}% is the sum of the kernels it dispatches) — NOT routed to the head ` + - `track (${stage}); the Architect must name the underlying kernel instead.`); - flagged.push({ short_name: c.short_name || c.name, pct_gpu_time: c.pct_gpu_time, - stage, gate: 'wrong_head_granularity', reason_code: 'wrong_head_granularity', - reason: `entity_kind=${c.entity_kind}; head track requires gpu_kernel` }); - } - if (noKind.length) { - if (HEAD_ENTITY_KIND_STRICT) { - admitted = admitted.filter((c) => c.entity_kind); - for (const c of noKind) { - log(` ⚠️ FLAG ${c.short_name || c.name}: profile row carries no entity_kind — ` + - `head_entity_kind_strict=true rejects it (cannot confirm it is a gpu_kernel).`); - flagged.push({ short_name: c.short_name || c.name, pct_gpu_time: c.pct_gpu_time, - stage, gate: 'wrong_head_granularity', reason_code: 'wrong_head_granularity', - reason: 'entity_kind missing; head track requires a confirmed gpu_kernel (strict)' }); - } - } else { - noteDegradation('head-admission', `${noKind.length} head candidate(s) carry no entity_kind at ${stage}`, - 'profile rows predate the entity_kind contract — a dispatcher op could still be routed as a kernel head'); +const PRE_FLAGGED_HEADS = []; +function admitHeads(queue, stage) { + const admitted = []; + for (const head of (queue || []).filter(Boolean)) { + if (head.entity_kind !== 'gpu_kernel') { + log(` ⚠️ FLAG ${head.short_name || head.name}: entity_kind=${head.entity_kind || 'missing'}; ` + + `the head track requires a profiler-confirmed gpu_kernel (${stage}).`); + PRE_FLAGGED_HEADS.push({ short_name: head.short_name || head.name, + pct_gpu_time: head.pct_gpu_time, stage, gate: 'wrong_head_granularity', + reason: `entity_kind=${head.entity_kind || 'missing'}; gpu_kernel required` }); + continue; } + admitted.push(prepareHeadSelection(head)); } return admitted; } -// ---- INV-2/INV-3: AUTHORING ADMISSION ------------------------------------------------------------- -// The ONLY precondition on spending the author budget used to be `isolated_speedup > 1.0` — a number -// produced INSIDE the task dir, which says nothing about whether the result can ever reach the live -// path. A head can clear it while having no seam to bind to at all; the kernel is then authored, -// optimized for hours, and rejected at the e2e gate for a reason that was decidable in seconds. -// This gate answers "if this kernel turns out to be fast, can we actually deploy it?" BEFORE the spend. -// Three questions, none of them op-kind specific: -// 1. is there a seam to rebind at? (target_callable / live_call_seam) -// 2. is the denominator real? (not a fabricated oracle) -// 3. does the live signature admit a swap? (seam_contract binding verdict) -// Returns { ok, reason_code, why }; reason_code is a REJECT_CODES key so the caller can route it. -function authoringAdmission(h, ext) { - const seam = String((ext && ext.target_callable) || (h && h.target_callable) || (h && h.live_call_seam) || '').trim(); - if (!seam) return { ok: false, reason_code: 'no_rebind_seam', - why: 'neither the extraction nor the Architect record names a target_callable/live_call_seam — an ' + - 'authored kernel would have nowhere to bind' }; - // NOT a rejection. Deployability is decided by the seam, not by the oracle's provenance: a - // fabricated reference makes the isolated RATIO unusable (withheld upstream), while whether the - // kernel is faster is decided by the live A/B. Rejecting here would have discarded ten measured - // wins in the 88-run archive — see baselineDefect above for the counts. - if (baselineDefect(ext) === 'synthesized') - noteDegradation('authoringAdmission', `${(h && h.short_name) || seam}: authoring a head whose oracle is synthesized`, - 'admitted on the strength of the seam alone; its isolated speedup is unbankable and only a ' + - 'measured e2e A/B can accept it'); - const b = seamBindable(ext, seam); // C5: verify the binding verdict is for THIS seam, not a stale one - if (!b.ok) { - const c = (b.codes || []).find((x) => REJECT_CODES[x]) || 'signature_mismatch'; - return { ok: false, reason_code: c, why: b.why }; - } - return { ok: true, reason_code: '', why: b.why, seam }; -} - -// ---- THE RE-EXTRACT CORRECTIVE (INV-5 at the CHEAPEST detection point) ---------------------------- -// A corrective has to name the ACTUAL defect. Telling an extractor "your denominator is invalid" when -// what failed is the BINDING verdict makes it re-run the same validator against the same task dir and -// hand back the same verdict; the retry burns budget and changes nothing. -// -// The two defects do share a root cause often enough to say so out loud, and saying it is what turns -// this from a retry into a fix. When the chosen seam is an OUTER WRAPPER that is not a pure function of -// its arguments — a torch custom-op reading a global layer/KV registry, writing into a caller-owned -// buffer and returning None — it can be neither captured as an oracle NOR rebound. DESCENDING to the -// inner launcher it dispatches to fixes both at once, because when target == the live launcher, the -// frozen baseline IS that launcher. -// 0720: target = baseline = `...ops.chunked_prefill_paged_decode` (the live launcher) -> real -// denominator, kernel bound at the live call site, +18.079% e2e. -// 0802: target = `...attention:unified_attention_with_output` (the custom-op wrapper) -> oracle -// synthesized, entry invented as `attention_forward(args:dict)->fresh_tensor`, rejected -// no_rebind_seam, 0-byte overlay. The run's OWN strategize record already named the inner -// launcher in `live_call_seam`; nothing ever asked the extractor to use it. -// Generic by construction: it describes the SHAPE of the defect (wrapper vs launcher, in-place vs -// fresh-return, hidden context) and points at fields the record already carries. No op, kernel, model -// or backend is named. -function reextractCorrective(needBaseline, bind, codes) { - const DESCEND = - '\n\nSEAM DESCENT — usually the whole fix, and it fixes BOTH defects at once. If the seam you chose ' + - 'is an OUTER WRAPPER (a torch custom-op / dispatcher that reads state which never crosses the ' + - 'parameter boundary, e.g. a global layer or KV-cache registry, and/or writes into a caller-owned ' + - 'buffer and returns None) then it can be neither captured as an oracle nor rebound, and re-running ' + - 'the validator against it will keep returning the same verdict. DESCEND to the innermost launcher ' + - 'that wrapper dispatches to which IS a pure function of its arguments. `KERNEL.live_call_seam` ' + - 'usually already names it. If it does not, read the candidate server.log to see which entry the ' + - 'server ACTUALLY dispatched — backends get overridden at startup, so do NOT trust the env var you ' + - 'set — and grep that module. Then set BOTH `target_callable` and `meta.baseline_callable` to that ' + - 'launcher: when the target IS the live launcher, the frozen baseline is the launcher itself and the ' + - 'denominator defect disappears together with the binding defect.'; - const ENTRY = - '\n\nENTRY CONTRACT — GENERATE IT, DO NOT INVENT IT:\n' + - ' python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --mode entry ' + - '--out /entry_contract.py\n' + - 'This renders the unittest entry FROM the live signature: exact parameter names and order, and ' + - 'whether the seam writes into an out-param and returns None. Bind the unittest to THAT entry and ' + - 'return its path as `entry_contract_path`. The `fn(args) -> FRESH out` shape in the harness ' + - 'anti-exploit rules is the ORACLE\'s timing/correctness contract — it is NOT licence to hand the ' + - 'authored kernel a dict-taking, fresh-returning entry when the live seam is positional and in-place.'; - const BASELINE = - '\n\nDENOMINATOR: set `meta.baseline_callable` to the REAL online kernel — a module:attr that imports ' + - 'from the INSTALLED package, never anything under the task dir and never a reference implementation ' + - 'you wrote — and bind the unittest\'s baseline leg to it.'; - const VERIFY = - '\n\nTHEN RUN THE VENDORED VALIDATOR AND PASTE ITS VERDICT VERBATIM:\n' + - ' python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --eval-dir $EVAL_DIR --mode both\n' + - 'Return its `baseline_validation` and `binding_descriptor`/`binding_check` objects as your fields of ' + - 'the same name, plus baseline_frozen:true. The orchestrator reads the VERDICT, not your description ' + - 'of it: an extraction whose baseline_validation.ok is not true, or whose binding_check.bindable is ' + - 'not true, is INVALID. If after descending there is still no seam that is both capturable and ' + - 'bindable, return editable:false and say why — that routes the op to the config/tune track. NEVER ' + - 'substitute a reference implementation you wrote, and never keep a seam the binding check refused.'; - const head = !bind.ok - ? ' PRIOR ATTEMPT PRODUCED AN UNBINDABLE SEAM' + (codes ? ` (${codes})` : '') + ': ' + bind.why + '.' + - (needBaseline ? ' It also produced no valid speedup denominator.' : '') - : ' PRIOR ATTEMPT DID NOT PRODUCE A VALID SPEEDUP DENOMINATOR.'; - return head + DESCEND + (bind.ok ? '' : ENTRY) + (needBaseline ? BASELINE : '') + VERIFY; -} +// A FROZEN baseline is resolvable when the extractor either froze baseline_src/ (baseline_frozen) +// OR set an importable meta.baseline_callable. That is the language-independent speedup denominator. +const hasFrozenBaseline = (ext) => + !!(ext && (ext.baseline_frozen === true || + (typeof ext.baseline_callable === 'string' && ext.baseline_callable.trim() !== ''))); // Run a kernel_extractor agent and GUARANTEE it froze a real baseline. safeAgent already retries // transient failures; this wraps it to ALSO re-extract when the extraction succeeds (smoke passed, @@ -1182,40 +862,49 @@ function reextractCorrective(needBaseline, bind, codes) { // site (deep, opt-A, milestone/head extract_op, and the non-op milestone extract). async function extractWithBaseline(role, phase, intro, inputs, opts) { const smokeOk = (e) => !!(e && e.task_dir && (e.smoke === 'pass' || e.unittest_smoke === 'pass')); + const head = (inputs && inputs.KERNEL) || {}; let ext = await safeAgent(roleAgent(role, phase, intro, inputs), opts); let tries = 0; - // Re-extract for a missing BINDING contract too, not only a missing baseline. The corrective below - // already asks for both verdicts in one seam_contract.py run; without this condition an extraction - // that froze a baseline but skipped --mode binding is never asked again and dies silently at - // authoringAdmission with signature_mismatch, which is a gate firing on an absent field rather than - // on a real defect. Replaying the 88-run archive, that absence alone accounts for every one of the - // 22 accepted-and-measured heads the strict regime would otherwise drop. - const contractOk = (e) => hasFrozenBaseline(e) && seamBindable(e).ok; - while (smokeOk(ext) && !contractOk(ext) && tries < BASELINE_EXTRACT_RETRIES) { + const attemptedTargets = []; + const complete = (e) => hasFrozenBaseline(e) && kernelSelectionVerified(head, e).ok; + while (smokeOk(ext) && !complete(ext) && tries < BASELINE_EXTRACT_RETRIES) { tries++; + const selection = kernelSelectionVerified(head, ext); const needBaseline = !hasFrozenBaseline(ext); - const bind = seamBindable(ext); - const codes = (bind.codes || []).join(', '); - const what = needBaseline && !bind.ok ? 'a frozen baseline AND the seam BINDING contract' - : needBaseline ? 'a frozen baseline (the speedup denominator would fall back to the candidate\'s ' + - 'own scaffold — a fake win)' - : `the seam BINDING contract (${bind.why}) — an authored kernel could not be proven bindable, so ` + - 'its win could never reach the server'; - log(` ${(opts && opts.label) || role}: extraction is missing ${what}. ` + + const priorTarget = String((ext && ext.target_callable) || '').trim(); + if (priorTarget && !attemptedTargets.includes(priorTarget)) attemptedTargets.push(priorTarget); + const selectionCorrective = selection.ok ? '' : + ` PRIOR ATTEMPT DID NOT SELECT THE PROFILED GPU KERNEL: ${selection.why}. ` + + 'Treat KERNEL.live_call_seam as prose only. Merge KERNEL.seam_candidates with any missing inner ' + + 'launcher found from source/runtime inspection; preserve existing candidate classifications. ' + + 'Install safe markers for every relevant candidate, never native/JIT kernel_entry objects. Run ' + + 'kernel_selection.py over every process-local capture and all root-call traces. Return its JSON ' + + 'verbatim as selection_validation. Select the deepest live inner_launcher/op_seam across all ' + + 'calls/ranks; a fused head must select the whole-op op_seam. Rejecting the previous outer wrapper ' + + 'is not success. Do not return any ATTEMPTED_TARGET_CALLABLES value again.'; + const baselineCorrective = needBaseline ? + ' PRIOR ATTEMPT DID NOT FREEZE A BASELINE. Freeze the real online kernel into immutable ' + + 'baseline_src/, set meta.baseline_callable, bind the unittest baseline leg to it, and return ' + + 'baseline_frozen:true.' : ''; + log(` ${(opts && opts.label) || role}: extraction contract incomplete ` + + `(${selection.ok ? 'kernel selected' : selection.why}; ` + + `${needBaseline ? 'baseline missing' : 'baseline frozen'}). ` + `RE-EXTRACTING (retry ${tries}/${BASELINE_EXTRACT_RETRIES}).`); ext = await safeAgent( - roleAgent(role, phase, intro + reextractCorrective(needBaseline, bind, codes), inputs), + roleAgent(role, phase, intro + selectionCorrective + baselineCorrective, { + ...(inputs || {}), + PRIOR_TARGET_CALLABLE: priorTarget, + PRIOR_SELECTION_VALIDATION: (ext && ext.selection_validation) || {}, + ATTEMPTED_TARGET_CALLABLES: attemptedTargets.slice(), + }), opts); } - // Retries exhausted. Abort only when there is NO usable denominator at all; a synthesized reference - // proceeds with its ratio withheld (see baselineDefect) so the live A/B still gets to decide. - if (smokeOk(ext) && !hasFrozenBaseline(ext) && baselineDefect(ext) === 'synthesized') { - noteDegradation('extractWithBaseline', - `${(opts && opts.label) || role}: baseline is a SYNTHESIZED reference after ${BASELINE_EXTRACT_RETRIES} re-extractions`, - 'keeping the extraction but its isolated speedup is UNBANKABLE (op_bench withholds it and ' + - 'denominatorSound refuses it) — only a measured e2e A/B can accept this head'); - return { ...ext, denominator_invalid: true, - notes: `denominator is a synthesized reference (isolated speedup withheld) — ${ext.notes || ''}` }; + const finalSelection = kernelSelectionVerified(head, ext); + if (smokeOk(ext) && !finalSelection.ok) { + log(` ${(opts && opts.label) || role}: kernel selection still unverified after ` + + `${BASELINE_EXTRACT_RETRIES} re-extractions — ABORTING (${finalSelection.why}).`); + return { ...ext, smoke: 'fail', unittest_smoke: 'fail', selection_failed: true, + notes: `kernel selection failed: ${finalSelection.why} — ${ext.notes || ''}` }; } if (smokeOk(ext) && !hasFrozenBaseline(ext)) { log(` ${(opts && opts.label) || role}: STILL no frozen baseline after ${BASELINE_EXTRACT_RETRIES} ` + @@ -1304,111 +993,13 @@ async function trySurgicalFix(spec, reason, fixClass, attempt) { // isolated, base_inputs (the integrate inputs template, carries KERNEL_RESULT), reason, // cur:{overlay,flags,env,tput} }. Returns { banked, integ, isolated } (banked=false if // ineligible or still rejected). See knowledge/learned/method-cudagraph-safe-integration. -// EXTRACT-STAGE RE-ENTRY (INV-5). The reject says the TASK is wrong — wrong seam, wrong call contract, -// or a denominator that was never the online kernel. Fixing that means building a NEW task (new seam, -// new immutable unittest generated FROM the live signature), then authoring against it — not editing a -// kernel that is pinned to the old contract. The caller supplies `spec.reextract(reason, code)`, which -// re-runs extraction with the diagnosis attached and returns a fresh extraction record; without it we -// refuse rather than fall through to a re-author we know cannot work. -// Bounded by HEAD_REEXTRACT_MAX (default 1) and, like every corrective, not charged to HEAD_BUDGET. -const HEAD_REEXTRACT_MAX = parseInt(A.head_reextract_max != null ? A.head_reextract_max : 1, 10); -async function tryExtractReentry(spec, reason, verdict) { - if (typeof spec.reextract !== 'function' || HEAD_REEXTRACT_MAX < 1) { - log(` ${spec.short_name}: reject '${verdict.code || reason}' is EXTRACT-owned (the task's seam/contract ` + - `is wrong, and its unittest is immutable) — re-authoring in place cannot fix it. No re-extract hook ` + - `available here; NOT spending a corrective.`); - return { banked: false, blocked_stage: 'extract', reason_code: verdict.code }; - } - log(` ${spec.short_name}: reject '${verdict.code || reason}' is EXTRACT-owned — RE-EXTRACTING at a ` + - `bindable seam (up to ${HEAD_REEXTRACT_MAX}) instead of re-authoring against the immutable old contract.`); - for (let attempt = 1; attempt <= HEAD_REEXTRACT_MAX; attempt++) { - const ext2 = await spec.reextract(reason, verdict.code, attempt); - if (!ext2 || ext2.smoke !== 'pass' || !ext2.task_dir) { - log(` ${spec.short_name}: re-extract ${attempt}/${HEAD_REEXTRACT_MAX} produced no usable task ` + - `(${ext2 ? ext2.notes || ext2.smoke : 'null'}).`); - continue; - } - const adm = authoringAdmission(spec.head || {}, ext2); - if (!adm.ok) { - log(` ${spec.short_name}: re-extract ${attempt} still not admissible (${adm.reason_code}) — ${adm.why}.`); - continue; - } - if (ext2.task_dir === spec.task_dir) - noteDegradation('tryExtractReentry', 're-extract returned the SAME task_dir', - 'the seam may not have changed; the re-author may reproduce the original reject'); - log(` ${spec.short_name}: re-extracted at ${adm.seam} (task ${ext2.task_dir}); authoring against the ` + - `regenerated contract.`); - let al; - try { - al = await fastBoundedWorkflow({ scriptPath: KERNEL_WF_SCRIPT }, { - kernel_path: ext2.task_dir, workflow_dir: KERNEL_WF_DIR, - mode: 'author', target_language: spec.language || 'triton', - op_spec: { op_kind: ext2.op_kind || spec.op_kind, shapes: ext2.shapes || spec.shapes || {}, - dtype: ext2.dtype || spec.dtype || 'bf16', regime: spec.regime || '', cuda_graph_safe: true, - ...(ext2.workload_path ? { workload_path: ext2.workload_path } : {}) }, - perf_knowledge_dir: KERNEL_KNOWLEDGE_DIR, - use_expert_skills: USE_EXPERT_SKILLS ? 'true' : 'false', expert_skills_dir: EXPERT_SKILLS_DIR, - budget: KERNEL_BUDGET, gpu_ids: spec.gpu_id, exp_root: `${EVAL_DIR}/kernels/_exp`, - task: `RE-EXTRACTED SEAM. The previous attempt was rejected at the e2e gate with reason_code ` + - `'${verdict.code}' ("${reason}"): the kernel was correct and fast in isolation but could not be ` + - `bound at the live call site. This task dir targets a DIFFERENT, verified-bindable seam and its ` + - `unittest entry is GENERATED FROM THE LIVE SIGNATURE — implement exactly that entry contract ` + - `(same parameter names/order, same in-place-vs-return convention). Do not invent a new one. ` + - GRAPH_REQ + (TASK || ''), - apply_to_original: 'false', - }, `${spec.short_name}:reextract`); - } catch (e) { al = { authored: false, validation_status: 'error', reason: String(e) }; } - const iso2 = al && (al.final_weighted != null ? al.final_weighted : al.final_geomean); - if (!al || al.authored === false || !(iso2 > 1.0) || !al.final_patch) { - log(` ${spec.short_name}: re-extract author produced no usable kernel (${al ? al.reason || al.validation_status : 'null'}).`); - continue; - } - const base = spec.base_inputs || {}; - const inputs2 = { ...base, - KERNEL_RESULT: { ...(base.KERNEL_RESULT || {}), - task_dir: ext2.task_dir, target_callable: adm.seam, - code_patch: al.final_patch, final_patch: al.final_patch, - authored_kernel_eval_dir: al.eval_dir || '', verified_isolated_speedup: iso2, - corrective_fix_of: `${verdict.code} (re-extracted seam)` } }; - if (spec.cur) { - inputs2.CURRENT_OVERLAY = spec.cur.overlay; inputs2.CURRENT_FLAGS = spec.cur.flags; - inputs2.CURRENT_ENV = spec.cur.env; inputs2.CURRENT_THROUGHPUT = spec.cur.tput; - } - const integ2 = await runIntegrateBothLegs( - 'Apply the kernel authored against the RE-EXTRACTED seam; gate on e2e throughput.', inputs2, - `integrate ${spec.short_name} reextract`, spec.phase_name || 'HeadKernel'); - const curTput = (spec.cur && spec.cur.tput) || 0; - if (abDone(integ2) && integAccepted(integ2, spec.pct_gpu_time, iso2) && integ2.e2e_throughput_tok_s > curTput) - return { banked: true, integ: integ2, isolated: iso2, via: 'reextract' }; - const r2 = gateRejectReason(integ2, spec.pct_gpu_time, iso2); - log(` ${spec.short_name}: re-extracted candidate still rejected (${r2}).`); - // If the NEW seam is also extract-owned we are going in circles; stop rather than loop. - if (rejectStage(r2, integ2 && integ2.reason_code) === 'extract') break; - } - return { banked: false, blocked_stage: 'extract', reason_code: verdict.code }; -} - async function tryCorrectiveReauthor(spec) { let reason = spec.reason || ''; - // Which fix-and-retry class is this reject, and WHICH STAGE OWNS IT? '' = terminal. - const verdict0 = rejectVerdict(reason, spec.reason_code); - let fixClass = spec.fix_class || verdict0.cls; + // Which fix-and-retry class is this reject? '' = terminal (not auto-correctable). + let fixClass = spec.fix_class || rejectClass(reason); const eligible = HEAD_CORRECTIVE_MAX > 0 && !((FAST_MODE && FAST_DEADLINE_HIT) || TIME_DEADLINE_HIT) && (spec.kernel_eval_dir || spec.task_dir) && (spec.isolated || 0) > 1.0 && fixClass !== ''; if (!eligible) return { banked: false }; - // ---- INV-5: RE-ENTER AT THE STAGE THAT OWNS THE DEFECT -------------------------------------------- - // The loop below re-runs the kernel workflow on spec.task_dir. That dir's unittest.py is IMMUTABLE by - // the extractor's contract, so it pins the entry signature — which means a defect whose fix REQUIRES a - // different call contract or a different seam (no_rebind_seam, signature/arity/return mismatch, hidden - // context, a fabricated denominator) cannot be fixed there, by construction. Re-authoring anyway costs - // hours and lands on the same reject. Route those to the EXTRACT stage instead; route defects the - // kernel track cannot touch at all (profile/strategize) to nobody, loudly. - if (verdict0.stage && verdict0.stage !== 'author') { - if (verdict0.stage === 'extract') return await tryExtractReentry(spec, reason, verdict0); - log(` ${spec.short_name}: reject '${verdict0.code || reason}' is owned by the ${verdict0.stage.toUpperCase()} ` + - `stage — no amount of kernel re-authoring can fix it. NOT spending a corrective; flagging instead.`); - return { banked: false, blocked_stage: verdict0.stage, reason_code: verdict0.code }; - } const curTput = (spec.cur && spec.cur.tput) || 0; // The corrective instruction is CLASS-SPECIFIC. `integration` = the posture is wrong (JIT/capture/ // host-sync); `correctness` = the output is wrong on the live path (parity/accuracy fail or an @@ -1509,23 +1100,7 @@ async function tryCorrectiveReauthor(spec) { break; } log(` ${spec.short_name}: corrective still rejected (${reason}).`); - if (implausible2) { fixClass = 'correctness'; } - else { - const v2 = rejectVerdict(reason, integ2 && integ2.reason_code); - // If the NEW reject is owned by a different stage, hand off there instead of grinding the - // author loop against a defect it cannot reach (INV-5). - if (v2.stage && v2.stage !== 'author') { - if (v2.stage === 'extract') { - const re = await tryExtractReentry({ ...spec, reason, reason_code: v2.code }, reason, v2); - if (re.banked) return re; - } else { - log(` ${spec.short_name}: corrective surfaced a ${v2.stage.toUpperCase()}-owned defect ` + - `('${v2.code || reason}') — stopping the author loop.`); - } - break; - } - fixClass = v2.cls; - } + fixClass = implausible2 ? 'correctness' : rejectClass(reason); if (fixClass === '') break; // new failure not auto-correctable -> stop retrying // Progressive: the NEXT attempt builds on this attempt's (partially) fixed kernel, not the original. spec.kernel_eval_dir = fix.eval_dir || spec.kernel_eval_dir; @@ -1693,36 +1268,20 @@ if (want('setup')) { // OP-IDENTITY GUARD — a fused-MoE / grouped-expert GEMM must be optimized AS the fused op at its live // dispatcher seam, never decomposed into standalone dense GEMMs (a dense candidate has no live call site // → no_rebind_seam). So force op_kind='moe' (the grouped-GEMM branch; gemmSynthFor keys on this to keep - // dense synth OFF) and preserve the live seam as target_callable, so ANY lever (backend-swap / tune / - // author-fused) binds. The head is never SKIPPED — editability is irrelevant, since a non-editable fused + // dense synth OFF). The head is never SKIPPED — editability is irrelevant, since a non-editable fused // kernel is still backend-swapped at its (editable) dispatcher. GENERIC: detects via the Architect's // is_fused_kernel OR the profile class/name; never keys on a backend name. const _isFusedOp = (c) => (c && c.is_fused_kernel === true) || /(?:^|[^a-z])moe(?:[^a-z]|$)|group(?:ed)?[_ ]?gemm|ck_moe|expert|fused[_ ]?moe|fmoe|asm_moe|fused_custom/i .test(`${(c && c.op_kind) || ''} ${(c && c.short_name) || ''} ${(c && c.name) || ''} ${(c && c.classification) || ''} ${(c && c.class) || ''} ${(c && c.backend) || ''}`); - let _fusedTagged = 0, _seamBackfilled = 0; + let _fusedTagged = 0; for (const c of headQueue) { - // SEAM BACKFILL applies to EVERY head, not just fused ones. It used to sit behind the fused test, - // so a non-fused head (attention, a standalone GEMM) whose Architect record carried a live_call_seam - // but no explicit target_callable went into extraction with target_callable=undefined — i.e. the - // extractor was handed the seam as PROSE and had to re-derive it, and any two runs could re-derive - // it differently. Nothing about "copy the seam we already identified into the field that names the - // seam" is specific to fused ops. See INV-2. - if (!c.target_callable && c.live_call_seam) { c.target_callable = c.live_call_seam; _seamBackfilled++; } if (!_isFusedOp(c)) continue; c.op_kind = 'moe'; // grouped-GEMM branch (gemmSynthFor → no dense synth) _fusedTagged++; } - if (_seamBackfilled) log(`[op-identity] ${_seamBackfilled} head(s): target_callable backfilled from the Architect's live_call_seam (all op kinds).`); + headQueue = admitHeads(headQueue, 'strategize'); if (_fusedTagged) log(`[op-identity] ${_fusedTagged} fused/grouped head(s): op_kind=moe (never dense-GEMM), bound at live seam — optimized as the fused op, never skipped.`); - // ---- INV-6: ENTITY-KIND ADMISSION ----------------------------------------------------------------- - // The head track optimizes GPU KERNELS. A profile row can also be a torch DISPATCHER op — a Python-side - // span whose `pct_gpu_time` is the SUM of the kernels it dispatches, so routing one as a head both - // double-counts its Amdahl share and points extraction at a wrapper rather than at code that runs on - // the device. parse_profile.py now labels every row `entity_kind`; this admits only gpu_kernel rows. - // Note this is a TYPE check, not a name blacklist: it does not need to recognize `aten::`/`vllm::` - // prefixes, or any future naming convention, to keep a dispatcher span out of the kernel track. - headQueue = admitHeads(headQueue, 'strategize', PRE_FLAGGED_HEADS); log(`Strategy: ${headQueue.length} head candidates, ${kernelQueue.length} kernel candidates, ${(strategy && strategy.config_directions || []).length} config directions.`); // strategize decided the backends -> if any candidate routed flydsl, provision it now (blocking). await ensureFlydslGate(); @@ -1738,7 +1297,7 @@ if (want('setup')) { profile = { profile_topN_json: ST.profile_topn_json || '' }; strategy = { config_directions: ST.config_directions || [] }; kernelQueue = ST.kernelQueue || []; - headQueue = admitHeads(ST.headQueue || [], 'resume', PRE_FLAGGED_HEADS); // C8: re-admit on resume + headQueue = admitHeads(ST.headQueue || [], 'resume'); log(`Loaded carried state: EVAL_DIR=${EVAL_DIR}, baseline ${BASELINE_TPUT}, flags='${curFlags}', env='${curEnv}', ${headQueue.length} head + ${kernelQueue.length} kernel candidates.`); } @@ -1778,7 +1337,7 @@ if (want('config') && CONFIG_TUNE_ENABLED && strategy && (strategy.config_direct { phase: 'Strategize', label: 'architect:re-strategize', schema: STRATEGY_SCHEMA }); if (restrat && restrat.kernel_candidates) kernelQueue = restrat.kernel_candidates.slice(); if (restrat && restrat.head_candidates) - headQueue = admitHeads(restrat.head_candidates.slice(), 're-strategize', PRE_FLAGGED_HEADS); // C8 + headQueue = admitHeads(restrat.head_candidates.slice(), 're-strategize'); // re-strategize may have (re)routed flydsl -> provision it (idempotent; no-op if already done). await ensureFlydslGate(); } else { @@ -1801,7 +1360,7 @@ const acceptedHeads = (ST.accepted_heads || []).slice(); // so Finalize can finish the best one's A/B (Fix C) and so a real isolated win // is surfaced (return.pending_integrations) instead of being silently dropped. const pendingIntegrations = (ST.pending_integrations || []).slice(); -const flaggedHeads = (ST.flagged_heads || []).slice().concat(PRE_FLAGGED_HEADS); // dominant heads that could NOT be optimized (loudly surfaced, never silently skipped) +const flaggedHeads = (ST.flagged_heads || []).concat(PRE_FLAGGED_HEADS); // heads that could NOT be optimized (loudly surfaced, never silently skipped) let headDispatched = 0; const history = ST.history || { insights: [], ledger: [], milestones: [], bottleneck_now: '', suggest_next: '' }; @@ -1809,34 +1368,6 @@ const history = ST.history || { insights: [], ledger: [], milestones: [], bottle // never decomposed into a standalone dense GEMM — so dense-GEMM synth is off for it. function gemmSynthFor(h) { return (h && h.op_kind === 'moe') ? 'false' : GEMM_SYNTH; } -// EXTRACT-STAGE RE-ENTRY HOOK (INV-5), one definition for every track. `tryExtractReentry` can only act -// when the caller hands it a way to rebuild the task; without one it logs "no re-extract hook available" -// and the extract-owned reject dies as a flag. That hook used to exist at exactly ONE of the five -// corrective call sites (fast-mode head integration), so the same defect was repairable or terminal -// depending only on which scheduling path the run happened to take — deep mode, the serial head path and -// the milestone track all fell through. Factored out here so all five behave identically. -// `rec` is the head/kernel record (carries live_call_seam + engagement_check), `priorExt` the extraction -// being replaced. Op-kind agnostic: everything specific to the failure is passed through as PRIOR_*. -function headReextractor(rec, priorExt, phaseName) { - return async (why, code) => await extractWithBaseline( - 'kernel_extractor', 'extract_op', - 'RE-EXTRACT at a BINDABLE seam. The previous task for this op was authored successfully but ' + - 'REJECTED at the live e2e gate with reason_code=' + (code || 'n/a') + ' ("' + why + '"): the kernel ' + - 'could not be bound at (or never ran on) the live call site, so its isolated win could never reach ' + - 'the server. Do NOT rebuild the same task. Pick a seam that the binding contract admits and prove it ' + - 'mechanically before returning.', - { EVAL_DIR, MODEL_PATH, GPU_ID: SERVING_GPU, WORKLOAD, KERNEL: rec, GEMM_SYNTH: gemmSynthFor(rec), - ...(profile && profile.profile_workload_json ? { PROFILE_WORKLOAD_JSON: profile.profile_workload_json } : {}), - CURRENT_FLAGS: curFlags, CURRENT_ENV: curEnv, SKILL_DIR: WORKFLOW_DIR, - REQUIRE_DECODE_BUCKET: true, DECODE_M_BUCKETS: [1, CONC], - PRIOR_TASK_DIR: (priorExt && priorExt.task_dir) || '', - PRIOR_TARGET_CALLABLE: (priorExt && priorExt.target_callable) || (rec && rec.target_callable) || '', - PRIOR_REJECT_CODE: code || '', PRIOR_REJECT_REASON: why || '', - REQUIRE_BINDING_CONTRACT: true }, - { phase: phaseName || 'HeadKernel', label: `re-extract ${(rec && rec.short_name) || 'op'}`, - schema: EXTRACT_OP_SCHEMA }); -} - // =========================================================================== // PHASE: HeadKernel — the highest-pct_gpu_time ops (GEMM / attention), optimized // regardless of edit flag, via the bake-off ladder. This is the lever the old @@ -1945,17 +1476,6 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { `Return {roofline_note, target_geomean}.`, { phase: 'HeadKernel', label: `roofline ${h.short_name}`, schema: ROOFLINE_SCHEMA }); const rooflineTarget = anchor && Number.isFinite(anchor.target_geomean) ? anchor.target_geomean : 0; - // PRE-AUTHOR ADMISSION (INV-2) — same gate as the fast path, before any lane is opened. Deep mode - // spends the most budget per head, so an unbindable seam is most expensive here. - const admD = authoringAdmission(h, ext); - if (!admD.ok) { - log(` ⚠️ [deep] ${h.short_name}: NOT admitted to authoring (${admD.reason_code}) — ${admD.why}. No lanes opened.`); - flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, stage: 'extract', - gate: 'author_not_admitted', reason_code: admD.reason_code, reason: admD.why }); - history.ledger.push({ direction: h.short_name, verdict: 'flagged', - lesson: `deep author route withheld: ${admD.reason_code} — ${admD.why}` }); - return null; - } const lanes = lanesSpec.map((b) => ({ uid: `${h.short_name}::${b.key || b.lang}`, key: b.key || b.lang, lang: b.lang, mode: b.mode, steer: b.steer || '', state_dir: `${deepDir}/state/${b.key || b.lang}`, best: 1.0, noImprove: 0, active: true, ran: 0, lastEval: '', patch: '', @@ -2090,8 +1610,6 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { short_name: c.head.short_name, op_kind: c.ext.op_kind, shapes: c.ext.shapes, dtype: c.ext.dtype, regime: c.head.regime, gpu_id: SERVING_GPU, kernel_eval_dir: c.lastEval, task_dir: c.ext.task_dir, language: c.lang, isolated: c.best, reason: dreason, fix_class: rejectClass(dreason), pct_gpu_time: c.head.pct_gpu_time, - reason_code: (integ && integ.reason_code) || '', head: c.head, phase_name: 'HeadKernel', - reextract: headReextractor(c.head, c.ext, 'HeadKernel'), base_inputs: { EVAL_DIR, MODEL_PATH, GPU_ID: SERVING_GPU, WORKLOAD, NOISE_BAND_PCT: NOISE_BAND, E2E_REPEATS, KERNEL_RESULT: { @@ -2311,28 +1829,10 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { } const st = { h, ext, cands: [] }; headState.set(h.short_name, st); - // INV-1/INV-3: a bake-off win is only a win if the benchmarker vouched for its own provenance - // (it re-hashed the oracle and confirmed the denominator). op_bench.py now WITHHOLDS - // isolated_speedup entirely when the denominator is unsourced, so a null here is also a no-win. - if (bake && bake.gate === 'have_winner' && bake.isolated_speedup > 1.0 && provenanceOk(bake, 'op_bench') - && denominatorSound(bake, 'op_bench')) + if (bake && bake.gate === 'have_winner' && bake.isolated_speedup > 1.0) st.cands.push({ kind: 'direct_light', source: bake.winner_backend, winner_kind: bake.winner_kind, apply_env: bake.apply_env || '', apply_flags: bake.apply_flags || '', code_patch: bake.code_patch || '', tuning_artifact: bake.tuning_artifact || '', isolated: bake.isolated_speedup, parity_note: bake.parity_note || 'expected_close' }); - // PRE-AUTHOR ADMISSION (INV-2): decide deployability BEFORE the author fan-out, not after it. - // A failing verdict does NOT kill the head — direct_light (env/flag/backend-swap) candidates need - // no rebind and stay in play; only the AUTHORED route, whose whole value depends on being able to - // bind at the seam, is withheld. - const adm = authoringAdmission(h, ext); - if (!adm.ok) { - log(` ⚠️ ${h.short_name}: NOT admitted to the author fan-out (${adm.reason_code}) — ${adm.why}. ` + - `Author budget withheld; ${st.cands.length ? 'the non-authored candidate(s) still proceed' : 'head flagged'}.`); - flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, stage: 'extract', - gate: 'author_not_admitted', reason_code: adm.reason_code, reason: adm.why }); - history.ledger.push({ direction: h.short_name, verdict: 'flagged', - lesson: `author route withheld: ${adm.reason_code} — ${adm.why}` }); - continue; - } for (const ap of (bake && bake.author_plan ? bake.author_plan.slice(0, HEAD_AUTHOR_MAX) : [])) authorJobs.push({ short_name: h.short_name, h, ext, ap, best_known_ms: bake.best_known_ms }); } @@ -2431,19 +1931,11 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { // gateRejectReason injects an implausible_speedup verdict when the gate "passed" but the delta is // impossible (corruption) — so a fake win routes to the correctness corrective instead of banking. const reason = gateRejectReason(integ, h.pct_gpu_time, cand.isolated); - // Route on the STRUCTURED code when the integrator supplied one (INV-4). `reason_code` also - // decides, inside the corrective, whether to re-author or re-extract (INV-5). - const rcode = (integ && integ.reason_code) || ''; - const rverdict = rejectVerdict(reason, rcode); - const corr = (cand.kind === 'authored' && rverdict.cls !== '') + const corr = (cand.kind === 'authored' && rejectClass(reason) !== '') ? await tryCorrectiveReauthor({ short_name: h.short_name, op_kind: st.ext.op_kind, shapes: st.ext.shapes, dtype: st.ext.dtype, regime: h.regime, gpu_id: SERVING_GPU, kernel_eval_dir: cand.kernel_eval_dir, task_dir: st.ext.task_dir, language: cand.language, - isolated: cand.isolated, reason, reason_code: rcode, head: h, - fix_class: rverdict.cls, pct_gpu_time: h.pct_gpu_time, phase_name: 'HeadKernel', - // EXTRACT-stage re-entry hook (INV-5): rebuild the task at a seam that is actually - // bindable, with a unittest entry GENERATED from the live signature, and re-author there. - reextract: headReextractor(h, st.ext, 'HeadKernel'), + isolated: cand.isolated, reason, fix_class: rejectClass(reason), pct_gpu_time: h.pct_gpu_time, phase_name: 'HeadKernel', base_inputs: { EVAL_DIR, MODEL_PATH, GPU_ID: SERVING_GPU, WORKLOAD, NOISE_BAND_PCT: NOISE_BAND, E2E_REPEATS, KERNEL_RESULT: { short_name: h.short_name, task_dir: st.ext.task_dir, op_kind: st.ext.op_kind, @@ -2465,16 +1957,7 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { history.ledger.push({ direction: h.short_name, isolated_speedup: corr.isolated, e2e_delta_pct: corr.integ.e2e_delta_pct, verdict: 'confirmed_corrective', lesson: `fixed: ${reason}` }); } else { log(` ${h.short_name}: REJECTED at e2e gate (${reason}).`); - // A defect the kernel track structurally cannot fix is FLAGGED with its owning stage, not - // filed as a plain dead end — the run report then says WHERE the pipeline has to change. - if (corr.blocked_stage) { - flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, - stage: corr.blocked_stage, gate: 'stage_owned_defect', - reason_code: corr.reason_code || rverdict.code, reason }); - log(` ⚠️ FLAG ${h.short_name}: defect owned by the ${corr.blocked_stage.toUpperCase()} stage ` + - `(${corr.reason_code || rverdict.code || 'unclassified'}) — surfaced, not silently dropped.`); - } - history.ledger.push({ direction: h.short_name, isolated_speedup: cand.isolated, e2e_delta_pct: integ ? integ.e2e_delta_pct : 0, verdict: 'dead_end', lesson: reason || 'no e2e gain', reason_code: rverdict.code || '', owner_stage: rverdict.stage || '' }); + history.ledger.push({ direction: h.short_name, isolated_speedup: cand.isolated, e2e_delta_pct: integ ? integ.e2e_delta_pct : 0, verdict: 'dead_end', lesson: reason || 'no e2e gain' }); } } } @@ -2548,8 +2031,7 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { // Build the candidate list: the cheap direct_light winner (if any) + any authored implementations. const headCands = []; - if (bake.gate === 'have_winner' && bake.isolated_speedup > 1.0 - && provenanceOk(bake, 'op_bench') && denominatorSound(bake, 'op_bench')) { + if (bake.gate === 'have_winner' && bake.isolated_speedup > 1.0) { headCands.push({ kind: 'direct_light', source: bake.winner_backend, winner_kind: bake.winner_kind, apply_env: bake.apply_env || '', apply_flags: bake.apply_flags || '', code_patch: bake.code_patch || '', tuning_artifact: bake.tuning_artifact || '', isolated: bake.isolated_speedup, @@ -2558,17 +2040,7 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { // Author/rewrite route: write (+optimize) a fresh impl per planned language via the recursive kernel // layer. mode=author writes a from-scratch baseline then optimizes it; mode=optimize rewrites an // existing editable impl. The immutable oracle in ext.task_dir is the judge for both. - // PRE-AUTHOR ADMISSION (INV-2) — same gate as the fast/deep paths. The direct_light candidate above - // needs no rebind and is unaffected; only the author route is withheld when the seam cannot take it. - const admS = authoringAdmission(h, ext); - const plan = admS.ok ? (bake.author_plan || []).slice(0, HEAD_AUTHOR_MAX) : []; - if (!admS.ok && (bake.author_plan || []).length) { - log(` ⚠️ ${h.short_name}: author route withheld (${admS.reason_code}) — ${admS.why}.`); - flaggedHeads.push({ short_name: h.short_name, pct_gpu_time: h.pct_gpu_time, stage: 'extract', - gate: 'author_not_admitted', reason_code: admS.reason_code, reason: admS.why }); - history.ledger.push({ direction: h.short_name, verdict: 'flagged', - lesson: `author route withheld: ${admS.reason_code} — ${admS.why}` }); - } + const plan = (bake.author_plan || []).slice(0, HEAD_AUTHOR_MAX); for (const ap of plan) { const lang = ap.language || 'triton'; let al; @@ -2712,8 +2184,6 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { short_name: h.short_name, op_kind: ext.op_kind, shapes: ext.shapes, dtype: ext.dtype, regime: h.regime, gpu_id: h.gpu_id, kernel_eval_dir: cand.kernel_eval_dir, task_dir: ext.task_dir, language: cand.language, isolated: cand.isolated, base_inputs: headIntegrateInputs, reason, - reason_code: (integ && integ.reason_code) || '', head: h, pct_gpu_time: h.pct_gpu_time, - reextract: headReextractor(h, ext, 'HeadKernel'), cur: { overlay: curOverlay, flags: curFlags, env: curEnv, tput: curTput }, }) : { banked: false }; @@ -2741,8 +2211,6 @@ if (want('head') && headQueue.length && HEAD_BUDGET > 0) { short_name: h.short_name, op_kind: ext.op_kind, shapes: ext.shapes, dtype: ext.dtype, regime: h.regime, gpu_id: h.gpu_id, kernel_eval_dir: cand.kernel_eval_dir, task_dir: ext.task_dir, language: cand.language, isolated: cand.isolated, base_inputs: headIntegrateInputs, reason, fix_class: rejectClass(reason), pct_gpu_time: h.pct_gpu_time, - reason_code: (integ && integ.reason_code) || '', head: h, - reextract: headReextractor(h, ext, 'HeadKernel'), cur: { overlay: curOverlay, flags: curFlags, env: curEnv, tput: curTput }, }) : { banked: false }; @@ -2913,8 +2381,6 @@ while (want('kernel') && !TIME_DEADLINE_HIT && dispatched < BUDGET && (dispatche short_name: c.short_name, op_kind: ext.op_kind, shapes: ext.shapes, dtype: ext.dtype, regime: c.regime, gpu_id: c.gpu_id, kernel_eval_dir: kl.kernel_eval_dir, task_dir: ext.task_dir, language: kl.language || '', isolated: kl.final_geomean, base_inputs: mileIntegrateInputs, reason, fix_class: rejectClass(reason), pct_gpu_time: c.pct_gpu_time, phase_name: 'Milestone', - reason_code: (integ && integ.reason_code) || '', head: c, - reextract: headReextractor(c, ext, 'Milestone'), cur: { overlay: curOverlay, flags: curFlags, env: curEnv, tput: curTput }, }) : { banked: false }; @@ -3126,24 +2592,6 @@ if (want('final')) { // carried best-accepted throughput so a real, parity-checked win is never // reported as 0 / no_gain downstream. validatedOk = !!(validation && validation.director_verified_throughput_tok_s > 0 && validation.throughput_speedup > 0); - // INV-3: `claimed_throughput_tok_s` and `applied_to_original` are asked for precisely so the pipeline - // can be caught overclaiming — and were read by nothing. A gap between what the run claimed and what - // the Director independently measured is the single most important number in the whole report, and a - // win that was never applied to the original tree is not a delivered win. Both are now recorded. - if (validatedOk && validation.claimed_throughput_tok_s > 0) { - const v = validation.director_verified_throughput_tok_s; - const gapPct = 100 * (validation.claimed_throughput_tok_s - v) / v; - if (gapPct > 2) { - noteDegradation('validate', 'the run OVERCLAIMED its throughput', - `claimed ${validation.claimed_throughput_tok_s} tok/s vs Director-verified ${v} tok/s ` + - `(+${gapPct.toFixed(1)}% overclaim) — the reported speedup is the verified one`); - log(` ⚠️ overclaim: claimed ${validation.claimed_throughput_tok_s} vs verified ${v} tok/s (+${gapPct.toFixed(1)}%)`); - } - } - if (validatedOk && !String(validation.applied_to_original || '').trim()) - noteDegradation('validate', 'no applied_to_original path', - 'the Director verified a number but did not state where the win was applied in the original ' + - 'tree — the result may not be reproducible outside the eval dir'); finalSpeedup = validatedOk ? validation.throughput_speedup : (BASELINE_TPUT ? finalTput / BASELINE_TPUT : finalSpeedup); log(`COMPLETE. ${MODEL_NAME}: ${BASELINE_TPUT} -> ${validatedOk ? validation.director_verified_throughput_tok_s : finalTput} tok/s ` + `(${finalSpeedup ? finalSpeedup.toFixed(3) : '?'}x, status ${validation ? validation.validation_status : '?'}` + @@ -3197,7 +2645,6 @@ const carryState = { // Carry pending (verified-isolated, A/B-incomplete) wins WITH their inputs so a // resumed phase run can finish their A/B instead of re-discovering them. pending_integrations: pendingIntegrations, - degradations: DEGRADATIONS, // carried so a resumed phase run inherits, not resets, the audit trail history, }; @@ -3235,13 +2682,6 @@ const wfReturn = { pct_gpu_time: p.pct_gpu_time, partial: p.partial || null, })), flagged_heads: flaggedHeads, // dominant heads surfaced but not optimized (harness/extract/no-candidate) — never silently dropped - // INV-3: every contract this run could NOT enforce, and what it fell back to. An empty list means - // every gate ran on a machine verdict; a non-empty one is the run telling you which conclusions are - // unaudited. Silence used to be indistinguishable from enforcement. - degradations: DEGRADATIONS, - contracts: { baseline_contract_strict: BASELINE_CONTRACT_STRICT, seam_contract_strict: SEAM_CONTRACT_STRICT, - head_entity_kind_strict: HEAD_ENTITY_KIND_STRICT, - head_reextract_max: HEAD_REEXTRACT_MAX }, config_tune_enabled: CONFIG_TUNE_ENABLED, head_budget: HEAD_BUDGET, head_used: headDispatched, diff --git a/e2e_workflow/roles/e2e_integrator.md b/e2e_workflow/roles/e2e_integrator.md index 050f6267e..e823cefe3 100644 --- a/e2e_workflow/roles/e2e_integrator.md +++ b/e2e_workflow/roles/e2e_integrator.md @@ -53,35 +53,10 @@ e2e_optimization.md` (measurement discipline + the Amdahl stop rule). waived) and the measured e2e delta BLOWS PAST that ceiling, the kernel is likely doing LESS / degenerate work (corruption) that squeaked past a small accuracy sample — a fast-but-wrong server (truncated / degenerate generations). Re-check accuracy on a LARGER sample vs the TRUE baseline; if it does not - genuinely hold, report `gate:"rejected"` with `reason_code:"implausible_speedup"`. (A byte-exact accept - is NOT subject to this — trust it.) - -### 🔴 `reason_code` — the reject vocabulary is a CLOSED ENUM, and it is what routes the fix -Whenever `gate:"rejected"`, you MUST set `reason_code` to exactly one of these tokens. The orchestrator -routes the corrective off this field, **not** off your prose in `reason`. Prose is for humans; a code is -a decision. (Classifying a reject by pattern-matching an English sentence is not a decision procedure — -it is how `signature_mismatch` became unreachable, because a bare "mismatch" matched the correctness -rule first, so a seam defect was retried as a numerics defect for hours.) - -| owning stage | `reason_code` | means | -|---|---|---| -| **extract** (the TASK encodes the wrong seam/contract/denominator — re-extraction, not re-authoring) | `no_rebind_seam`, `signature_mismatch`, `arity_mismatch`, `param_name_mismatch`, `optional_param_dropped`, `param_kind_mismatch`, `return_contract_mismatch`, `seam_mismatch`, `hidden_context_inputs`, `candidate_unresolvable`, `no_seam_descriptor`, `no_engagement`, `wrong_seam`, `invalid_denominator` | the kernel may be perfect; it cannot be bound where the server actually calls, or its speedup was measured against something that is not the live path | -| **author** (the seam is right; posture or numerics are wrong) | `cuda_graph_capture_unsafe`, `no_binary_for_gpu`, `capture_hang`, `host_sync_in_hot_path`, `oom`, `parity_regression`, `accuracy_regression`, `output_corruption`, `implausible_speedup` | re-authoring on the SAME task dir can fix it | -| **upstream** (no amount of kernel work fixes it) | `wrong_head_granularity`, `delegated_track_disabled` | | -| **terminal** (not a defect) | `no_win`, `do_no_harm` | a correct kernel with no headroom | - -Pick the code that names the FIRST thing that went wrong, not the symptom you noticed last: if the -candidate could not be rebound at the seam, that is `no_rebind_seam`/`signature_mismatch` even though -the visible outcome was "e2e delta was zero". An extract-owned code sent as an author-owned one causes a -guaranteed-futile re-author loop. If nothing in the enum fits, use the closest extract-owned code and -explain in `reason` — an unrecognised token is logged as a degradation and the head is dropped. - -### 🔴 `provenance_ok` -Set `provenance_ok:false` whenever any number you are reporting was not measured by you in this run -against a verified baseline — a carried-over figure, an estimate, a number read out of an earlier -report, or an A/B where the reference leg did not actually run. The orchestrator will not bank a result -with `provenance_ok:false`, which is the correct outcome; reporting `true` for an unsourced number is -the failure this field exists to prevent. Omitting the field entirely is recorded as a degradation. + genuinely hold, report `gate:"rejected"` with reason **`implausible_speedup`**. (A byte-exact accept is + NOT subject to this — trust it.) Use the reason vocabulary the orchestrator's auto-correct classifier + keys on — `parity_regression`, `accuracy_regression`, `implausible_speedup`, `output_corruption` — so a + fixable correctness reject is routed to a corrective re-author rather than dropped. If any fails, REJECT and record why (with the numbers) for the eval-dir timeline report — a real isolated speedup that doesn't show up e2e is an expected Amdahl outcome, not a bug. @@ -335,7 +310,6 @@ Return JSON: "output_parity": "pass|fail", "parity_kind": "byte_exact|accuracy|none", "gate": "accepted|stack|rejected|incomplete", - "reason_code": "", "accepted_overlay": "", "reason": "why accepted/rejected/incomplete (cite Amdahl + measured delta vs noise band)" } diff --git a/e2e_workflow/roles/kernel_extractor.md b/e2e_workflow/roles/kernel_extractor.md index b8ea9e259..aeea20d71 100644 --- a/e2e_workflow/roles/kernel_extractor.md +++ b/e2e_workflow/roles/kernel_extractor.md @@ -75,7 +75,12 @@ If the live regime genuinely cannot be reproduced offline (op only exists fused routing-dependent MoE token counts), say so in `notes` and report `editable:false`/drop rather than freeze an out-of-regime oracle nobody should trust. -1. **Locate the source.** **If `KERNEL.source_hint`/`KERNEL.launcher_hint` is provided (TraceLens +1. **Locate the source and select the live launcher.** `KERNEL.device_kernel` is the profiled GPU + symbol this extraction must reach. `KERNEL.target_callable` is only a hint, and + `KERNEL.live_call_seam` is prose that must never become a machine target. Start from + `KERNEL.seam_candidates[]`. If source/runtime inspection finds a missing inner launcher, append it + to the returned `seam_candidates[]` with its exact role, depth, matching device kernels, and evidence. + **If `KERNEL.source_hint`/`KERNEL.launcher_hint` is provided (TraceLens pre-resolved the file/seam), look there FIRST** — but always CONFIRM by importing the package + grepping the `short_name`/`module:attr` target; never trust the hint blindly (it may point at a launcher/wrapper rather than the true defining file). If no hint, resolve as usual @@ -89,46 +94,47 @@ freeze an out-of-regime oracle nobody should trust. NOT synthesize a standalone-GEMM proxy just to make it look extractable. - **FUSED / monolithic op** (fused-MoE, grouped-expert GEMM, asm/CK fused kernel — `KERNEL` arrives with `op_kind=moe` and `GEMM_SYNTH=false`): **extract the FUSED op** (capture its live I/O oracle), NOT its - constituent standalone GEMMs. Set `target_callable` to the **dispatcher** actually called at runtime — - use `KERNEL.target_callable`/`KERNEL.live_call_seam` if provided (e.g. the vLLM `fused_moe`/ - `fused_experts` dispatcher), which is editable Python EVEN WHEN the underlying kernel is a non-editable + constituent standalone GEMMs. Select the bindable whole-operation **`op_seam`** from + `KERNEL.seam_candidates` (or an exact `KERNEL.target_callable` hint), which is editable Python EVEN + WHEN the underlying kernel is a non-editable library/asm `.so`. That dispatcher seam is what lets a fused op be BACKEND-SWAPPED (aiter/flydsl/triton fused) or AUTHOR-fused-replaced regardless of the underlying kernel's editability. Report `editable=true` (the seam is rebindable). NEVER decompose it into a dense A·Bᵀ GEMM — no live call site. - - **OUTER WRAPPER that is NOT a pure function of its arguments** (a torch custom-op / dispatch entry that - reads a global registry or layer object, writes its result IN PLACE into a caller-owned buffer, and/or - returns `None`): it can be **neither captured as an oracle NOR rebound**, so do NOT try to force it and - do NOT synthesize a reference implementation to stand in for it. **DESCEND** to the innermost launcher - that wrapper dispatches to which IS a pure function of its arguments — - `KERNEL.live_call_seam` usually already names it. Confirm which entry the server ACTUALLY dispatched by - reading the candidate `server.log` (backends get overridden at startup — do NOT trust the env var you - set). Then set **BOTH** `target_callable` **and** `meta.baseline_callable` to that launcher, so the - authored kernel and the speedup denominator are the same live seam. Generate its deployable entry with - `python3 $SKILL_DIR/scripts/seam_contract.py --task-dir --mode entry` — never hand-write it — - and verify with `--mode both` that `baseline_validation.ok`, `binding_descriptor.ok`, AND - `binding_check.bindable` are ALL true. With no `--candidate`, `--mode both` checks the entry it just - rendered against its own descriptor, so a `bindable:false` here means the GENERATED entry does not fit - the live seam — regenerate it, do not paste the failing verdict. - A synthesized oracle plus an unbindable wrapper is the single failure mode this case exists to prevent: - it yields a kernel-level "speedup" against a number nothing in the server ever computed, and an overlay - that patches nothing. If after descending no seam is both capturable and bindable, report - `editable=false` — that is an honest stop, not a fallback to synthesis. + - **OUTER WRAPPER / dispatcher** → do not stop after rejecting it. Descend to the deepest safe Python + `inner_launcher` or `op_seam` that launches `KERNEL.device_kernel`. A native or Triton + `kernel_entry` remains source evidence, not a monkeypatch target. If no safe callable can be found, + report `editable=false`; never claim selection success from rejection alone. 2. **Capture shapes + oracle** from a live server using `scripts/capture_shapes.py` via a temporary capture overlay, driven by the SAME workload as the profile so shapes match the regime: ```bash TASK="$EVAL_DIR/kernels/_task"; mkdir -p "$TASK" - # write a tiny capture overlay sitecustomize that calls capture_shapes.install(...) - python3 "$SKILL_DIR/scripts/overlay_setup.py" monkeypatch \ + # Repeat add-marker for every relevant safe Python candidate, deepest first. + python3 "$SKILL_DIR/scripts/overlay_setup.py" add-marker \ + --overlay "$TASK/_capture_overlay" --target "" \ + --marker-file "$SKILL_DIR/scripts/seam_trace.py" + python3 "$SKILL_DIR/scripts/overlay_setup.py" add-capture \ --overlay "$TASK/_capture_overlay" \ - --target "" --impl-module capture_shapes --impl-attr _wrapper \ - --impl-file "$SKILL_DIR/scripts/capture_shapes.py" 2>/dev/null || true - # simpler/robust: drive via env so capture_shapes self-installs on import + --target "" --out "$TASK" --max 5 \ + --capture-file "$SKILL_DIR/scripts/capture_shapes.py" BACKEND="" OUT_DIR="$TASK/_capture" GPU="$GPU_ID" MODEL="$MODEL_PATH" \ ISL= OSL= CONC= REPEATS=0 PROFILE=0 \ - OVERLAY_PYTHONPATH="$SKILL_DIR/scripts" \ - EXTRA_ENV="CAPTURE_TARGET= CAPTURE_OUT=$TASK CAPTURE_MAX=5" \ + OVERLAY_PYTHONPATH="$TASK/_capture_overlay" \ + EXTRA_ENV="" \ bash "$EVAL_DIR/bench_e2e.sh" 2>&1 | tee "$EVAL_DIR/logs/capture_.log" + python3 "$SKILL_DIR/scripts/kernel_selection.py" \ + --target "" --device-kernel "" \ + --capture-meta "$TASK"/capture.pid-*.rank-*/meta.json \ + --torch-trace "$TASK"/selection_trace.pid-*.rank-*.call-*.json \ + --candidate-target "" \ + --out "$TASK/selection_validation.json" ``` + Repeat `--candidate-target` for every relevant candidate. `seam_trace` writes an atomic trace for + each PID/rank/root call, and `capture_shapes` writes atomic process-local artifacts. The verifier + merges all calls for each PID, isolates External ids between trace files, and requires every capture + PID to pass. Installation proof is distinct from execution: a marked mutually exclusive branch may + stay inactive, while a selected outer target fails if a deeper marked candidate launches the same + device kernel in any call/rank. Require `deepest_verified:true`, then copy the selected process's + `meta.json` and `reference_io.pt` into the task root. (REPEATS=0 → just warmup drives a short window; capture flushes incrementally + on server exit.) Verify `reference_io.pt` + `meta.json` exist and `num_cases` ≥ 1. For a head GEMM that serves both regimes you MUST capture/synthesize BOTH a decode case (M ≈ `WORKLOAD.conc`) and a prefill case (large M) — @@ -358,13 +364,6 @@ freeze an out-of-regime oracle nobody should trust. > caller. `h.check_correct_multi` catches it (later call overwrites the earlier return; distinct > `data_ptr` + no-mutation asserted). Never write a correctness check that reads each output right > after its own call — check them all together, as the shared lib does. - > **Scope: this is the ORACLE HARNESS's timing/correctness contract, not the deployable entry's.** It - > says the function the unittest times must not alias a static buffer across calls. It is NOT licence - > to give the authored kernel a dict-taking, fresh-returning entry when the LIVE seam is positional and - > writes in place — the deployed entry must match the live seam's real signature (that is what - > `seam_contract.py --mode entry` generates and `--mode binding` checks). An in-place live seam is served - > by an entry that writes into the caller's `out` and returns what the live seam returns; the harness - > still gets its fresh-output wrapper around it. 5. **Finalize `meta.json`**: set `build` (false for pure-Triton; true + a build cmd for HIP/CK/asm candidates), `candidate_backends`, `regime`, the source path in sglang, and re-confirm the `reference_io_sha256` checksum (the validator re-checks it to detect tampering). @@ -394,63 +393,6 @@ freeze an out-of-regime oracle nobody should trust. > Do **NOT** record `unittest_smoke:"fail"` or drop the head for exit 3 — that status is reserved for a > genuine baseline-bind / correctness failure (exit 1). Only after 3 failed regenerations set > `unittest_smoke:"fail"` with `reason="harness_incomplete_unrecoverable"`. -7. **🔴 MANDATORY — machine-check the baseline and the seam binding. Do not hand-write these verdicts.** - First write/reconcile `/meta.json` with the final `baseline_callable`, `target_callable`, - `baseline_origin`, `baseline_capture_evidence`, and complete `seam_runtime_evidence` (including an - explicit `hidden_context`, even when it is `[]`). The validator consumes that file; running it before - the evidence is persisted is not a valid check. Then run it and paste the three objects it prints back - VERBATIM into your Return JSON: - ```bash - python3 "$SKILL_DIR/scripts/seam_contract.py" \ - --task-dir "" --eval-dir "$EVAL_DIR" \ - --baseline-spec "" \ - --target-spec "" \ - --mode both --json - ``` - **🔴 `--baseline-spec` and `--target-spec` are DISTINCT seams — do not conflate them.** `--baseline-spec` - is the callable whose numbers you validate against; `--target-spec` is the live seam the entry must - rebind, and it is what `binding_descriptor`/`binding_check` are built from. The old single `--spec` is a - DEPRECATED alias for `--target-spec` only: passing the baseline through it makes the binding descriptor - describe the baseline, not the seam you must rebind, so a mismatched candidate passes silently. If you - omit either flag the script falls back to `meta.baseline_callable`/`meta.target_callable`; the explicit - flags prevent accidental spec substitution, but they do not replace the evidence that must already be - present in meta.json. - - It imports each spec from the LIVE site-packages, checks the module file actually lives under a real - install root (not a directory you just created), and rejects a synthesized `baseline_src.*` strawman. - `baseline_validation.ok=false` ⇒ the head is NOT admissible: return `editable:false` / - `baseline_frozen:false` with the reported reason. Do not "fix" it by pointing at a file you wrote. - - `binding_descriptor` is DERIVED from `inspect.signature` of the live callable — parameter names, - kinds, and defaults. It is a fact about the seam, not a guess; never edit it by hand. - **`inspect.signature` sees ONLY the declared parameters.** It CANNOT see what a callable reads that - is not a parameter: module globals, forward/attention context, registries, env, captured closure - state, or which parameters it writes in place. Those are NOT inspect-derived — they come from - `seam_runtime_evidence` below, which YOU must fill from reading the source, and the descriptor - merely copies them through. A seam that reads hidden state is not a pure function of its arguments - and cannot be soundly rebound as if it were. - - `binding_check` compares your candidate entry point against that descriptor. If it fails, the - candidate cannot be rebound at the seam and the isolated speedup is unbankable — regenerate the - entry from the descriptor instead of arguing with it: - ```bash - python3 "$SKILL_DIR/scripts/seam_contract.py" --task-dir "" --eval-dir "$EVAL_DIR" \ - --target-spec "" --mode entry --entry-name --out "/entry_contract.py" - ``` - and report that path as `entry_contract_path`. - - `seam_runtime_evidence.inplace_params` MUST list the parameter names the live callable writes - through (e.g. an `output=` buffer). `op_bench.py` uses this list — and ONLY this list, never a name - heuristic — to decide where a replayed in-place seam's result is read from. Getting it wrong makes - correctness unmeasurable, not merely inaccurate. - - `seam_runtime_evidence.hidden_context` MUST list EVERY non-parameter input the live callable reads: - module globals, forward/attention context, registry lookups, env, and captured closure state. This - is NOT discoverable from `inspect.signature` — you establish it by reading the source. An omitted - `hidden_context` is `unknown` and fails the strict binding gate; it is never inferred as `[]`. Do not - declare an empty list unless you verified that the seam is pure. If a seam depends on hidden context - it cannot be rebound at the seam by arguments alone: report it (so the entry can supply that context, - or the head is dropped) rather than silently admitting an impure seam as pure. - - `num_cases` MUST be the real number of records in `reference_io.pt`. `0` means the oracle recorded - no calls, so correctness is unfalsifiable, and the orchestrator will reject the head. Report the - true count; do not round it up. - - `reference_io_sha256` MUST be the checksum of the oracle bytes you actually shipped. An empty string - means the oracle is unpinned and tampering is undetectable — also a rejection. Return JSON: ```json @@ -459,15 +401,16 @@ Return JSON: "editable": true, "task_dir": "/kernels/_task", "source_path_in_sglang": "", + "device_kernel": "", "target_callable": "", + "seam_candidates": [ + {"target_callable": "", + "role": "outer_wrapper|dispatcher|op_seam|inner_launcher|kernel_entry", + "device_kernels": [""], "depth": 0, "evidence": "source/runtime evidence"} + ], + "selection_validation": {"contract": "kernel_selection", "ok": true, "deepest_verified": true}, "baseline_callable": "", "baseline_frozen": true, - "synthesized": false, - "baseline_validation": { "...": "verbatim from seam_contract.py --mode both" }, - "binding_descriptor": { "...": "verbatim from seam_contract.py --mode both" }, - "binding_check": { "...": "verbatim from seam_contract.py --mode both" }, - "entry_contract_path": "/entry_contract.py or \"\" if the candidate already matches", - "seam_runtime_evidence": { "inplace_params": ["output"], "returns_none": true, "hidden_context": [] }, "num_cases": 0, "regimes_captured": ["prefill","decode"], "candidate_backends": ["triton","hip","ck"], @@ -806,15 +749,6 @@ force real compact-operand compute: > those M values for every (N,K) — these are non-negotiable; the smoke-test and downstream gate depend on > them.** Combine with the prefill M per `PREFILL_M_NOTE`. -**🔴 Before returning, run the same machine check as `PHASE=extract` step 7** (`seam_contract.py ---mode both`) and paste `baseline_validation` / `binding_descriptor` / `binding_check` back verbatim. -It applies with FULL force here, because `PHASE=extract_op` is the path on which a synthesized oracle is -legal: `synthesized:true` says the reference IO was manufactured, not captured from the live server, and -the orchestrator treats a synthesized baseline as an INVALID speedup denominator. Report it honestly — -a head with `synthesized:true` can still be benchmarked, but its isolated speedup will be withheld -rather than banked, which is the correct outcome. Claiming `synthesized:false` for a manufactured oracle -is the single defect that produced a 4.47× "win" with zero end-to-end effect. - Return JSON: ```json { @@ -828,15 +762,16 @@ Return JSON: "regimes_captured": ["prefill"], "candidate_backends": ["aiter","hipblaslt","triton","ck"], "reference_io_sha256": "", - "num_cases": 0, + "device_kernel": "", "target_callable": "", + "seam_candidates": [ + {"target_callable": "", + "role": "outer_wrapper|dispatcher|op_seam|inner_launcher|kernel_entry", + "device_kernels": [""], "depth": 0, "evidence": "source/runtime evidence"} + ], + "selection_validation": {"contract": "kernel_selection", "ok": true, "deepest_verified": true}, "baseline_callable": "", "baseline_frozen": true, - "baseline_validation": { "...": "verbatim from seam_contract.py --mode both" }, - "binding_descriptor": { "...": "verbatim from seam_contract.py --mode both" }, - "binding_check": { "...": "verbatim from seam_contract.py --mode both" }, - "entry_contract_path": "/entry_contract.py or \"\"", - "seam_runtime_evidence": { "inplace_params": [], "returns_none": false, "hidden_context": [] }, "smoke": "pass|fail", "notes": "transpose/bias inference, regime, whether oracle was synthesized vs captured" } diff --git a/e2e_workflow/roles/op_benchmarker.md b/e2e_workflow/roles/op_benchmarker.md index 596e8e7e1..a583da50b 100644 --- a/e2e_workflow/roles/op_benchmarker.md +++ b/e2e_workflow/roles/op_benchmarker.md @@ -227,32 +227,6 @@ Inputs: `EVAL_DIR`, `OP_TASK_DIR` (from the Kernel Extractor `extract_op`), `OP_ NOTE: the experimental triton GEMM stub is NOT a real implementation — treat "no editable triton kernel for this op" as author-needed. FlyDSL DOES have a real importable GEMM (`flydsl_hgemm` / `flydsl_preshuffle_gemm_a8`), so a flydsl author baseline reuses it rather than starting from zero. -2a. **🔴 THE DENOMINATOR — a speedup is only as real as what it was divided by.** - `op_bench.py` now names the thing it timed as the baseline and reports it as `denominator` (also on - every row as `denominator_provenance`). It is one of: - | `denominator` | meaning | speedup | - |---|---|---| - | `measured_backend_default` | the default backend that the live server actually dispatches, timed here | **bankable** | - | `verified_baseline` | the frozen real online kernel, machine-verified by `seam_contract.py` | **bankable** | - | `unverified_baseline` | a declared baseline whose binding was never checked | withheld unless `--no-denominator-strict` | - | `target_fallback` | no baseline resolved; the candidate was timed against the target itself | **withheld** | - | `synthesized_reference` | the reference IO was manufactured, not captured from the live server | **withheld** | - | `none` | nothing to divide by | **withheld** | - - When the denominator is not sound the script sets `speedup_withheld:true` with - `speedup_withheld_reason`, moves the number to `isolated_speedup_unpublishable`, suppresses - `amdahl_ceiling_e2e_pct`, and prints `speedup=WITHHELD ()`. **In that case you MUST report - `isolated_speedup: null` and `reason_code:"invalid_denominator"` — do NOT copy the unpublishable - figure into `isolated_speedup`, and do NOT re-run with `--no-denominator-strict` to make the number - reappear.** A withheld speedup is not a missing measurement; it is a measurement of the wrong thing. - This is the exact failure being prevented: a kernel timed against a manufactured reference scored - 4.47× and moved end-to-end throughput by nothing at all, because the server never called it. The right - response is to fix the seam (send it back to extract), not to publish the ratio. - - Set `provenance_ok:false` whenever any number you report was not measured by you in this run against a - sound denominator — including a figure carried over from an earlier round or read out of a report. The - orchestrator will not bank a result with `provenance_ok:false`, which is the correct outcome. - 2b. **HARNESS SELF-CHECK + bounded self-repair (do NOT mistake a broken harness for "no win").** Distinguish two completely different outcomes in `opbench_result.json`: - a backend that **ran and produced a number** but was slower / not correct → a legitimate per-backend @@ -327,9 +301,6 @@ Return JSON: "winner_backend": "aiter|hipblaslt|triton|flydsl|ck|none", "winner_kind": "env|flag|patch|none", "isolated_speedup": 1.0, - "denominator": "measured_backend_default|verified_baseline|unverified_baseline|target_fallback|synthesized_reference|none", - "speedup_withheld": false, - "reason_code": "<'invalid_denominator' when speedup_withheld; else ''>", "winner_editable": false, "best_known_ms": 0.0, "recommend_tier_c": false, diff --git a/e2e_workflow/roles/profiler.md b/e2e_workflow/roles/profiler.md index 392cea62b..364282077 100644 --- a/e2e_workflow/roles/profiler.md +++ b/e2e_workflow/roles/profiler.md @@ -101,20 +101,20 @@ An upstream orchestrator may already have profiled the SAME baseline workload wi or triton per `kernel_kind`; attention→library_attn; etc.), `editable`←`op_to_source_patchable`. Carry `source_file`/`kernel_path` into each entry's `notes` (the Architect/Extractor reuse them). Write `profile_topN.json` + `.md` via your own Write and set `source:"tracelens"`. - **🔴 Then you MUST annotate the hand-assembled rows — do not hand-write `entity_kind`:** + **Then annotate the assembled rows from profiler evidence; never hand-write `entity_kind`:** ```bash - # Select the trace to cross-check against FIRST — the annotate command below needs $TLT set. Prefer - # the top-level rank0 serving trace; never recurse into capture_traces/. TLT=$(ls -1 "$TRACELENS_TRACE_FILE"/*rank0*.pt.trace.json.gz 2>/dev/null | head -1) - [ -z "$TLT" ] && TLT=$(ls -1 "$TRACELENS_TRACE_FILE"/*.pt.trace.json.gz "$TRACELENS_TRACE_FILE"/*.json.gz "$TRACELENS_TRACE_FILE"/*.json 2>/dev/null | head -1) - python3 "$EVAL_DIR/parse_profile.py" --annotate "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" \ - --torch-trace "$TLT" --annotate-out "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" + [ -z "$TLT" ] && TLT=$(ls -1 "$TRACELENS_TRACE_FILE"/*.pt.trace.json.gz \ + "$TRACELENS_TRACE_FILE"/*.json.gz "$TRACELENS_TRACE_FILE"/*.json 2>/dev/null | head -1) + python3 "$EVAL_DIR/parse_profile.py" \ + --annotate "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" \ + --torch-trace "$TLT" \ + --annotate-out "$EVAL_DIR/profile/round_${ROUND}/profile_topN.json" ``` - Annotation needs an evidence source (`--torch-trace` and/or `--rocprof-dir`): if `TLT` is empty there - is no trace to cross-check the TraceLens rows against, so **do not hand-stamp `entity_kind` — fall - back to the normal collection (steps 1–5)** and annotate from your own trace. The annotator exits - non-zero if any Top-N row is `unresolved`: resolve those before returning, because a row whose kind is - unknown may not be a GPU kernel at all. + If no trace is available, fall back to the normal collection below instead of guessing a row's + entity kind. Annotation expands an outer dispatcher/custom-op row through torch-profiler External-id + edges into its concrete device children. Preserve the resulting `device_kernel`, `profile_parent`, + and split GPU percentages: rejecting a dispatcher without discovering its children is not success. - **If `TRACELENS_TRACE_FILE` is also a non-empty path that EXISTS → run an ADDITIONAL trace-analysis pass on top of analysis.md to sharpen the picture** (this is required by contract when the trace is present). `TRACELENS_TRACE_FILE` is a `torch_trace` **directory** that holds one steady-state serving @@ -134,10 +134,7 @@ An upstream orchestrator may already have profiled the SAME baseline workload wi that matches** (this is the mandatory shape double-check, since `analysis.md` shapes may be inaccurate). Keep the TraceLens ranking/`%gpu` as the primary impact signal, but cross-check that the same heads top both views; note any disagreement in `notes`. Emit the final reconciled `profile_topN.json`/`.md` with - `source:"tracelens+trace"`. **🔴 Reconciliation rewrites `profile_topN.json`, so re-run the `--annotate` - step above on the reconciled file (or carry the `entity_kind`/`entity_evidence`/`entity_kind_*` fields - forward onto it) — the final emitted file MUST still carry the annotator's evidence-backed kinds, never - hand-written ones.** + `source:"tracelens+trace"`. - **If `TRACELENS_ANALYSIS_MD` is empty/missing (or the file does not exist) → ignore TraceLens entirely and run the normal collection (steps 1–5) unchanged.** Likewise, for ANY reprofile round the TraceLens prior is stale (it reflects the baseline config) — ignore it and re-collect. @@ -246,20 +243,6 @@ degrade to whatever is available, and if both analysis.md and trace are unusable skill errors out at any point, note it and return the Top-N anyway — a failed analysis skill must never fail or block the profile.** -### 🔴 `entity_kind` — every Top-N row must say WHAT KIND OF THING it is -`parse_profile.py` stamps each row with `entity_kind ∈ {gpu_kernel, memory_op, dispatcher_op, -python_launcher, unresolved}` plus `entity_evidence` (the profiler category the kind was DERIVED from). -This is a fact about how the work was observed, not a guess from the name — so never edit it, and never -substitute a name pattern for it. The existing `classification`/`backend_guess` fields ARE name guesses -and are labelled as such; they are not a substitute. - -Why it gates: only a `gpu_kernel` row is a rewrite target. A `dispatcher_op` row is a host span that -ENCLOSES the kernels it dispatched, so its time is already counted in them — routing a head at it -double-counts and the "optimization" cannot move e2e. A `memory_op` row is a copy (fix the allocation, -not the kernel). A `python_launcher` row is host overhead. An `unresolved` row is one this profiler -never saw dispatched at all, and the orchestrator's head-admission gate refuses it. Rows you have not -annotated are treated as `unresolved`, so run the annotator; do not fill the field in by hand. - Return JSON: ```json { @@ -271,8 +254,7 @@ Return JSON: "total_gpu_time_ms": 0.0, "top_kernels": [ {"rank": 1, "short_name": "...", "classification": "...", "pct_gpu_time": 0.0, - "calls": 0, "avg_us": 0.0, "shapes": [[...]], "editable": true, "regime_note": "prefill|decode|both", - "entity_kind": "gpu_kernel", "entity_evidence": ""} + "calls": 0, "avg_us": 0.0, "shapes": [[...]], "editable": true, "regime_note": "prefill|decode|both"} ], "shift_note": "for reprofile: how the bottleneck moved vs previous round", "notes": "resolved 'other' entries, rocprof availability, anything unusual" diff --git a/e2e_workflow/roles/system_architect.md b/e2e_workflow/roles/system_architect.md index 0c7b86744..a3b6fc1af 100644 --- a/e2e_workflow/roles/system_architect.md +++ b/e2e_workflow/roles/system_architect.md @@ -241,19 +241,19 @@ OPTIONAL upstream TraceLens prior (may be empty strings — treat empty/missing e2e by MORE than the noise band. Otherwise drop it — say so. 5. Write `EVAL_DIR/strategy.md` (human-readable plan) and return the routing. -> **🔴 Every `head_candidates` entry MUST carry `entity_kind` and `target_callable`, copied forward — -> not re-derived.** -> - `entity_kind` comes verbatim from the profile row (`parse_profile.py --annotate` stamped it from the -> profiler's own event category). Only `gpu_kernel` rows may be routed to the head track: the -> orchestrator drops and loudly flags anything else. A `dispatcher_op` row is a HOST span that encloses -> the kernels it dispatched, so its GPU time is already counted in them — scheduling a head there books -> Amdahl mass twice and guarantees the "win" cannot appear e2e. If a row's `entity_kind` is `unresolved`, -> send it back to the Profiler rather than routing it on a name that looks like a kernel. -> - `target_callable` is the `module:attr` the Extractor must actually bind — normally the callable part -> of `live_call_seam`. State it explicitly: the Extractor's speedup is only bankable if it is measured -> at the seam the server really dispatches, and leaving the seam implicit is what lets a plausible-looking -> reference implementation be frozen as "the baseline" and produce a large isolated speedup with zero -> end-to-end effect. +> **Every head candidate must identify a device kernel and a structured callable chain.** +> - Copy `entity_kind` and `device_kernel` from the profiled row. Route only `gpu_kernel` rows; if a +> dispatcher was expanded, route its device children rather than recreating the outer aggregate. +> - `live_call_seam` is prose context only. Never copy arrows, signatures, paths, or prose into +> `target_callable`. +> - Build `seam_candidates[]` from source and the baseline server log. Each entry has an exact importable +> `target_callable` (`module:attr`), `role` (`outer_wrapper|dispatcher|op_seam|inner_launcher|kernel_entry`), +> matching `device_kernels`, `depth`, and evidence. Include every plausible callable on the live path. +> - Keep native/JIT `kernel_entry` objects as source evidence only; replacing them can break their +> `.run`, `.warmup`, or cache protocols. For non-fused heads prefer the deepest safe +> `inner_launcher`/`op_seam`; fused heads must select the whole-operation `op_seam`. +> - `target_callable` is only an initial hint. The Extractor may add a missing inner launcher and must +> prove the final choice with runtime markers; merely rejecting an outer wrapper is not discovery. Return JSON: ```json @@ -268,7 +268,14 @@ Return JSON: {"id": "h0", "short_name": "...", "op_kind": "gemm|attn", "pct_gpu_time": 0.0, "shapes": "[[1024,5120],[5120,34816]]", "dtype": "bf16", "regime": "prefill|decode|both", "entity_kind": "gpu_kernel", - "target_callable": "", + "device_kernel": "", + "target_callable": "", + "seam_candidates": [ + {"target_callable": "", + "role": "outer_wrapper|dispatcher|op_seam|inner_launcher|kernel_entry", + "device_kernels": [""], "depth": 0, + "runtime_verified": false, "evidence": "source/log evidence"} + ], "transpose_b": true, "bias": false, "candidate_backends": ["aiter","hipblaslt","triton","ck"], "is_fused_kernel": false, diff --git a/e2e_workflow/scripts/capture_shapes.py b/e2e_workflow/scripts/capture_shapes.py index 5464de7ea..9261f7f1d 100755 --- a/e2e_workflow/scripts/capture_shapes.py +++ b/e2e_workflow/scripts/capture_shapes.py @@ -54,6 +54,22 @@ } +def _rank(): + return next((str(os.environ[key]) for key in + ("RANK", "LOCAL_RANK", "TP_RANK", "SLURM_PROCID") + if key in os.environ), "unknown") + + +def _process_out_dir(out_dir): + """Isolate selection-capture artifacts so TP workers cannot corrupt each other.""" + unique = (os.environ.get("CAPTURE_PROCESS_UNIQUE") == "1" + or bool(os.environ.get("GEAK_SELECTION_TRACE"))) + if not unique: + return out_dir + return os.path.join( + out_dir, f"capture.pid-{os.getpid()}.rank-{_rank()}") + + def _shapes_dtypes(args, kwargs): """Light shape/dtype walk (no clone) so we can catalog EVERY distinct shape cheaply, independent of the memory-bounded oracle capture.""" @@ -153,7 +169,20 @@ def _sig(args, kwargs): def _wrapper(*args, **kwargs): s = _STATE - out = s["orig"](*args, **kwargs) + # This marker is consumed by kernel_selection.py from the capture run's torch trace. It turns + # "the hook saw calls" into stronger evidence: the GPU kernel selected from the baseline profile + # must actually execute while THIS callable is active. record_function is effectively a no-op + # when no profiler is collecting, so existing non-profiled capture users keep the same behavior. + marker = f"GEAK_TARGET::{s['target']}" + try: + record_function = _torch().profiler.record_function + except Exception: + record_function = None + if record_function: + with record_function(marker): + out = s["orig"](*args, **kwargs) + else: + out = s["orig"](*args, **kwargs) s["calls"] += 1 in_graph = _capturing() try: @@ -239,7 +268,9 @@ def _flush(write_oracle=True): # workload capture (< max_cases distinct shapes) and a late regime-coverage case (appended past # max_cases) land on disk; records is bounded, so this rewrites only a handful of times. if write_oracle and records and len(records) > s["oracle_records"]: - torch.save({"target": s["target"], "records": records}, io_path) + tmp_io = f"{io_path}.tmp-{os.getpid()}-{threading.get_ident()}" + torch.save({"target": s["target"], "records": records}, tmp_io) + os.replace(tmp_io, io_path) import hashlib h = hashlib.sha256() with open(io_path, "rb") as fh: @@ -274,6 +305,8 @@ def walk(o): # not just single-shape h.check_correct_multi. meta = { "target": s["target"], + "process_id": os.getpid(), + "rank": _rank(), "module": s["mod"].__name__ if s["mod"] else None, "attr": s["attr"], "num_cases": len(records), @@ -291,8 +324,11 @@ def walk(o): "build": False, # default: pure-python/triton; Extractor flips to True for HIP/CK/asm tasks "note": "Oracle captured from baseline. Do NOT edit unittest.py or reference_io.pt during opt.", } - with open(os.path.join(out_dir, "meta.json"), "w") as fh: + meta_path = os.path.join(out_dir, "meta.json") + tmp_meta = f"{meta_path}.tmp-{os.getpid()}-{threading.get_ident()}" + with open(tmp_meta, "w") as fh: json.dump(meta, fh, indent=2) + os.replace(tmp_meta, meta_path) sys.stderr.write(f"[capture_shapes] flushed {len(records)} case(s) " f"(regimes={sorted(s['regime_seen'])}), " f"oracle_complete={s['oracle_written']} -> {out_dir}\n") @@ -365,8 +401,15 @@ def install(target, out_dir, max_cases=5): f"({type(orig).__module__}.{type(orig).__name__}): a plain-function stand-in for a native/" f"triton-JIT callable SIGSEGVs the server (e.g. mxfp4 matmul_ogs). Hook a Python-level seam " f"(its caller) instead, or set CAPTURE_WRAP_UNSAFE=1 to force.") + out_dir = _process_out_dir(out_dir) s.update(target=target, out_dir=out_dir, max_cases=int(max_cases), orig=orig, mod=mod, attr=attr, installed=True) + if os.environ.get("GEAK_SELECTION_TRACE"): + # Selection runs are intentionally short and server teardown may use SIGTERM, which skips + # Python atexit. Persist the first eager capture immediately. + s["flush_every"] = 1 + elif os.environ.get("CAPTURE_FLUSH_EVERY"): + s["flush_every"] = max(1, int(os.environ["CAPTURE_FLUSH_EVERY"])) setattr(mod, attr, _make_wrapper(orig)) atexit.register(_flush) sys.stderr.write(f"[capture_shapes] hooked {target}; recording up to {max_cases} cases -> {out_dir}\n") diff --git a/e2e_workflow/scripts/check_schema_consumption.py b/e2e_workflow/scripts/check_schema_consumption.py deleted file mode 100644 index 4bdd377ad..000000000 --- a/e2e_workflow/scripts/check_schema_consumption.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python3 -"""CI check: every field an agent is ASKED to return must be READ by something. - -Why this exists ---------------- -The orchestrator declares response schemas (`const X_SCHEMA = obj({...})`) that agents are contractually -required to fill in. A declared-but-never-read field is worse than a missing one: the agent spends -tokens computing it, the reviewer sees it in the JSON and assumes it gated something, and the pipeline -proceeds on a check that does not exist. `synthesized` was exactly this — declared in EXTRACT_OP_SCHEMA, -asked for in the role prompt, written truthfully by the extractor, and consumed by no line of code, so a -fabricated baseline sailed through the head gate. - -This check is deliberately syntactic and conservative: it flags a field only when NO plausible read of -that name occurs anywhere outside a schema literal. It cannot prove a field is used correctly; it can -prove a field is used nowhere, which is the failure mode above. - -Usage: - python3 check_schema_consumption.py [--file e2e_workflow.js] [--json] [--list] - exit 0 = every declared field has a reader; exit 1 = at least one does not. - -Waivers: add a field name to ALLOWED_UNCONSUMED below WITH a reason. A waiver is a statement that the -field is documentation for the agent, not an input to a decision. - -Stdlib only. -""" -import argparse -import json -import os -import re -import sys - -DEFAULT_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "e2e_workflow.js") - -# name -> why it is legitimately write-only. -ALLOWED_UNCONSUMED = { - "notes": "free-text rationale carried into the report/ledger for humans, not a gate input", - "smoke": "required evidence string the agent must produce; read by humans reviewing the ledger", - "why": "human-facing justification", - "note": "human-facing justification", - "summary": "human-facing narrative", - "rationale": "human-facing narrative", - "evidence": "human-facing narrative", - "risk": "human-facing narrative", - - # --- artifact PATHS: written so a human (or a later run) can open the file. Nothing branches on - # them; the orchestrator addresses those artifacts by its own EVAL_DIR convention, not by the path - # the agent reports. If one ever becomes an input to a decision, delete its waiver. - "baseline_summary_path": "artifact path for the reader; the orchestrator uses its own EVAL_DIR path", - "bench_script": "artifact path for reproduction; the orchestrator re-derives the bench invocation", - "profile_topN_md": "human-readable twin of profile_topN.json, which IS consumed", - "strategy_path": "artifact path for the reader; the strategy object itself is consumed", - - # --- telemetry that lands in the report/ledger. Reported, not gated. - "baseline_spread_pct": "run-quality telemetry surfaced in the report; the gate uses the A/B medians", - "total_gpu_time_ms": "profile telemetry; ranking uses pct_gpu_time, which IS consumed", - "throughput_speedup_vs_baseline": "sweep telemetry; the sweep winner is chosen on absolute tok/s", - "trials": "sweep search log kept for the report", - "workload": "the setup's echo of the workload config it was given; the orchestrator holds the source of truth", - "regime_summary": "prose summary of the regime split for the report; the regime object is consumed", - "order_of_work": "the strategist's suggested ordering, superseded by the orchestrator's own head queue", - "drop_list": "advisory 'do not pursue' list for the report; heads are dropped by measured gates", - "accepted_config": "config bundle passed through to the report writer verbatim", - "playbook_appended": "acknowledgement that the experience curator wrote its file", - "per_backend": "full bake-off table kept for the report; the winner fields are consumed", - "recommend_tier_c": "advisory routing hint; the actual route is decided by winner_kind + admission", - "winner_editable": "duplicate of winner_kind=='patch_candidate', which is what routes", - "arbitration_note": "the Director's prose when report and validation disagree", - "build": "extraction fact recorded for the surgeon's build step, which reads the task meta.json directly", - "regimes_captured": "recorded in the ledger; regime coverage is enforced inside the frozen unittest", -} - - -def find_schema_blocks(src): - """-> [(schema_name, body_text, start, end)] for `const NAME_SCHEMA = obj({ ... })` declarations.""" - out = [] - for m in re.finditer(r"const\s+([A-Za-z0-9_]*SCHEMA)\s*=\s*obj\(\s*\{", src): - i = m.end() - 1 # at the '{' - depth, j = 0, i - while j < len(src): - if src[j] == "{": - depth += 1 - elif src[j] == "}": - depth -= 1 - if depth == 0: - break - j += 1 - out.append((m.group(1), src[i + 1:j], m.start(), j + 1)) - return out - - -def _strip_comments(s): - s = re.sub(r"/\*.*?\*/", " ", s, flags=re.S) - return re.sub(r"//[^\n]*", " ", s) - - -def fields_of(body): - """Top-level property names of a schema body, ignoring nested object/array literals. - - Walks the text tracking brace/bracket depth so `properties` of an inline nested obj({...}) are not - mistaken for fields of the outer schema — those are checked as part of their own nested read. - """ - body = _strip_comments(body) - names, depth, i, tok_start = [], 0, 0, 0 - while i < len(body): - c = body[i] - if c in "{[(": - depth += 1 - elif c in "}])": - depth -= 1 - elif c == ":" and depth == 0: - seg = body[tok_start:i].strip().strip(",").strip() - m = re.search(r"([A-Za-z_][A-Za-z0-9_]*)\s*$", seg) - if m: - names.append(m.group(1)) - tok_start = i + 1 - elif c == "," and depth == 0: - tok_start = i + 1 - i += 1 - return names - - -def nested_field_names(body): - """Every property name at ANY depth — used to catch nested declarations too.""" - body = _strip_comments(body) - return set(re.findall(r"([A-Za-z_][A-Za-z0-9_]*)\s*:\s*(?:\{|arr|obj\()", body)) - - -def consumption_sites(code, field): - """Plausible reads of `field` in `code` (which already has schema literals removed).""" - pats = [ - rf"\.{re.escape(field)}\b", # o.field - rf"\[\s*['\"]{re.escape(field)}['\"]\s*\]", # o['field'] - rf"\b{re.escape(field)}\s*[,}}]", # const { field } = o / { field, x } - rf"['\"]{re.escape(field)}['\"]", # 'field' as a key/lookup string - ] - return sum(len(re.findall(p, code)) for p in pats) - - -def check(path): - with open(path) as fh: - src = fh.read() - blocks = find_schema_blocks(src) - if not blocks: - return {"error": f"no `const *SCHEMA = obj({{...}})` declarations found in {path}"}, 1 - - # Code = the file with every schema literal cut out, so a field's own declaration is not - # mistaken for a read of it. - code, last = [], 0 - for _, _, s, e in sorted(blocks, key=lambda b: b[2]): - code.append(src[last:s]) - last = e - code.append(src[last:]) - code = _strip_comments("".join(code)) - - declared, findings = {}, [] - for name, body, _, _ in blocks: - for f in fields_of(body) + sorted(nested_field_names(body)): - declared.setdefault(f, set()).add(name) - - for f in sorted(declared): - if f in ("type", "properties", "required", "items", "enum", "additionalProperties", - "description", "default", "format"): - continue # JSON-Schema vocabulary, not a payload field - n = consumption_sites(code, f) - if n == 0: - findings.append({"field": f, "schemas": sorted(declared[f]), - "waived": f in ALLOWED_UNCONSUMED, - "waiver_reason": ALLOWED_UNCONSUMED.get(f, "")}) - unwaived = [x for x in findings if not x["waived"]] - return {"file": os.path.relpath(path), "num_schemas": len(blocks), - "num_fields": len(declared), "unconsumed": findings, - "num_unconsumed_unwaived": len(unwaived)}, (1 if unwaived else 0) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--file", default=DEFAULT_FILE) - ap.add_argument("--json", action="store_true") - ap.add_argument("--list", action="store_true", help="also print every declared field") - a = ap.parse_args() - - rep, rc = check(os.path.abspath(a.file)) - if a.json: - print(json.dumps(rep, indent=2)) - return rc - if "error" in rep: - print("ERROR: " + rep["error"]) - return rc - print(f"{rep['file']}: {rep['num_schemas']} schemas, {rep['num_fields']} declared fields") - for x in rep["unconsumed"]: - tag = "WAIVED " if x["waived"] else "UNREAD " - print(f" {tag} {x['field']:<32} declared in {', '.join(x['schemas'])}" - + (f" ({x['waiver_reason']})" if x["waived"] else "")) - if rc: - print(f"\nFAIL: {rep['num_unconsumed_unwaived']} field(s) are requested from an agent and read " - f"by nothing. Either consume them or add a waiver with a reason in " - f"check_schema_consumption.ALLOWED_UNCONSUMED.") - else: - print("\nOK: every declared field has a reader (or a documented waiver).") - return rc - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/e2e_workflow/scripts/kernel_selection.py b/e2e_workflow/scripts/kernel_selection.py new file mode 100644 index 000000000..fe084d87c --- /dev/null +++ b/e2e_workflow/scripts/kernel_selection.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""Verify that an extracted callable is the live seam for a profiled GPU kernel. + +The extractor marks every structured candidate with ``seam_trace`` and captures +the selected callable with ``capture_shapes``. ``seam_trace`` profiles a bounded +warmup/call window and emits install proof plus nested live-call markers. This tool +verifies, from machine evidence, that: + +* the target is an exact import spec rather than a prose call chain; +* the target was called by the live server; +* the expected profiled GPU kernel ran while the target marker was active. +* no deeper marked live candidate launches the same kernel. + +The JSON verdict is consumed by the workflow. A rejected outer wrapper is not +selection success; the extractor must descend and repeat this probe until both +this contract and the binding contract pass. +""" + +import argparse +import gzip +import json +import os +import re +import sys + + +MARKER_PREFIX = "GEAK_TARGET::" +INSTALL_PREFIX = "GEAK_INSTALLED::" +_SPEC_RE = re.compile( + r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:" + r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$" +) + + +def valid_callable_spec(value): + return bool(_SPEC_RE.fullmatch(str(value or "").strip())) + + +def _open(path): + return gzip.open(path, "rt") if str(path).endswith(".gz") else open(path, "rt") + + +def load_events(path): + with _open(path) as fh: + doc = json.load(fh) + return doc.get("traceEvents", doc if isinstance(doc, list) else []) + + +def merge_process_traces(paths): + """Merge call traces without allowing pid or External-id collisions across files.""" + merged = [] + for trace_index, path in enumerate(paths): + prefix = f"trace-{trace_index}:" + for original in load_events(path): + if not isinstance(original, dict): + continue + event = dict(original) + event["pid"] = prefix + str(original.get("pid")) + event["tid"] = prefix + str(original.get("tid")) + args = dict(original.get("args") or {}) + if args.get("External id") is not None: + args["External id"] = prefix + str(args["External id"]) + event["args"] = args + merged.append(event) + return merged + + +def canonical_kernel_name(value): + """Return a stable token for matching mangled/demangled kernel symbols.""" + text = str(value or "").strip().lower() + text = re.sub(r"\[clone[^\]]*\]", "", text) + text = re.sub(r"<.*>", "", text) + text = text.split("(", 1)[0] + text = text.rsplit("::", 1)[-1] + return re.sub(r"[^a-z0-9_]+", "", text) + + +def kernel_matches(expected, observed): + want = canonical_kernel_name(expected) + got = canonical_kernel_name(observed) + if not want or not got: + return False + return want == got or ( + len(want) >= 6 + and re.search(r"(?:^|_)" + re.escape(want) + r"(?:$|_)", got) is not None + ) + + +def _complete_spans(events, name): + spans = [] + stacks = {} + for event in events: + if not isinstance(event, dict) or event.get("name") != name: + continue + phase = event.get("ph", "X") + if phase == "X" and event.get("ts") is not None: + start = float(event["ts"]) + spans.append((start, start + float(event.get("dur") or 0), event)) + elif phase == "B": + stacks.setdefault((event.get("pid"), event.get("tid")), []).append(event) + elif phase == "E": + key = (event.get("pid"), event.get("tid")) + if stacks.get(key): + begin = stacks[key].pop() + start = float(begin.get("ts") or 0) + spans.append((start, float(event.get("ts") or start), begin)) + return spans + + +def _within_any_span(event, spans, same_thread=False): + if event.get("ts") is None: + return False + ts = float(event["ts"]) + return any( + start <= ts <= end + and (not same_thread or ( + event.get("pid") == marker.get("pid") and event.get("tid") == marker.get("tid"))) + for start, end, marker in spans + ) + + +def _marker_kernel_evidence(trace_events, target_callable, device_kernel): + marker = MARKER_PREFIX + target_callable + spans = _complete_spans(trace_events or [], marker) + # Only CPU events on the marker's own thread can establish launch causality. Global timestamp + # overlap is unsafe under concurrent serving. The resulting External ids bridge to async GPU events. + related_external_ids = set() + for event in trace_events or []: + if not isinstance(event, dict) or not _within_any_span(event, spans, same_thread=True): + continue + if event.get("cat") == "kernel": + continue + ext = (event.get("args") or {}).get("External id") + if ext is not None: + related_external_ids.add(ext) + matched = [] + for event in trace_events or []: + if not isinstance(event, dict) or event.get("cat") != "kernel": + continue + ext = (event.get("args") or {}).get("External id") + if ext is not None and ext in related_external_ids and kernel_matches( + device_kernel, event.get("name")): + matched.append(str(event.get("name") or "")) + return { + "target": target_callable, + "marker": marker, + "spans": spans, + "related_external_ids": related_external_ids, + "matched": matched, + } + + +def _is_nested(inner_spans, outer_spans): + for inner_start, inner_end, inner_event in inner_spans: + for outer_start, outer_end, outer_event in outer_spans: + same_thread = ( + inner_event.get("pid") == outer_event.get("pid") + and inner_event.get("tid") == outer_event.get("tid") + ) + if same_thread and outer_start <= inner_start and inner_end <= outer_end and ( + outer_start < inner_start or inner_end < outer_end): + return True + return False + + +def verify(target_callable, device_kernel, capture_meta, trace_events, candidate_targets=None): + target_callable = str(target_callable or "").strip() + device_kernel = str(device_kernel or "").strip() + failed = [] + + if not valid_callable_spec(target_callable): + failed.append("invalid_target_callable") + if not device_kernel: + failed.append("missing_device_kernel") + + meta_target = "" + observed_calls = 0 + if isinstance(capture_meta, dict): + module = str(capture_meta.get("module") or "").strip() + attr = str(capture_meta.get("attr") or "").strip() + meta_target = f"{module}:{attr}" if module and attr else "" + observed_calls = int(capture_meta.get("total_calls_observed") or 0) + if meta_target != target_callable: + failed.append("capture_target_mismatch") + if observed_calls <= 0: + failed.append("target_not_observed") + + candidates = [] + for candidate in list(candidate_targets or []) + [target_callable]: + candidate = str(candidate or "").strip() + if candidate and candidate not in candidates: + candidates.append(candidate) + invalid_candidates = [candidate for candidate in candidates if not valid_callable_spec(candidate)] + if invalid_candidates: + failed.append("invalid_candidate_target") + evidence = { + candidate: _marker_kernel_evidence(trace_events, candidate, device_kernel) + for candidate in candidates if valid_callable_spec(candidate) + } + installed_candidates = [ + candidate for candidate in candidates + if _complete_spans(trace_events, INSTALL_PREFIX + candidate) + ] + missing_candidate_markers = sorted(set(candidates) - set(installed_candidates)) + if missing_candidate_markers: + failed.append("candidate_marker_not_installed") + selected_evidence = evidence.get(target_callable) or { + "marker": MARKER_PREFIX + target_callable, "spans": [], + "related_external_ids": set(), "matched": [], + } + marker = selected_evidence["marker"] + spans = selected_evidence["spans"] + if not spans: + failed.append("target_marker_missing") + + related_external_ids = selected_evidence["related_external_ids"] + matched = selected_evidence["matched"] + if not matched: + failed.append("device_kernel_not_under_target") + deeper = [ + candidate for candidate, candidate_evidence in evidence.items() + if candidate != target_callable and candidate_evidence["matched"] + and _is_nested(candidate_evidence["spans"], spans) + ] + if deeper: + failed.append("deeper_live_candidate_exists") + deepest_verified = bool( + spans and matched and not deeper and not invalid_candidates + and not missing_candidate_markers) + + return { + "contract": "kernel_selection", + "ok": not failed, + "target_callable": target_callable, + "device_kernel": device_kernel, + "capture_target": meta_target, + "total_calls_observed": observed_calls, + "target_marker": marker, + "target_marker_calls": len(spans), + "matched_kernel_calls": len(matched), + "matched_kernel_names": sorted(set(matched)), + "correlated_external_ids": len(related_external_ids), + "candidate_targets_tested": installed_candidates, + "live_candidate_targets": sorted( + candidate for candidate, candidate_evidence in evidence.items() + if candidate_evidence["matched"]), + "deeper_live_candidates": sorted(deeper), + "missing_candidate_markers": missing_candidate_markers, + "deepest_verified": deepest_verified, + "evidence": "installed+live_nested_candidate_markers+torch_profiler_external_id", + "failed": failed, + } + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--target", required=True, help="exact module:attr selected seam") + parser.add_argument("--device-kernel", required=True, help="GPU kernel selected from profile") + parser.add_argument("--capture-meta", required=True, nargs="+", + help="one or more process-local capture_shapes meta.json files") + parser.add_argument("--torch-trace", required=True, nargs="+", + help="one or more process-local capture traces (.json or .json.gz)") + parser.add_argument("--candidate-target", action="append", default=[], + help="exact module:attr candidate marked in the same trace; repeat for all candidates") + parser.add_argument("--out", default="", help="optional verdict JSON path") + args = parser.parse_args(argv) + + trace_paths = list(args.torch_trace) + attempts = [] + for meta_path in args.capture_meta: + with open(meta_path) as fh: + meta = json.load(fh) + capture_pid = str(meta.get("process_id") or "") + matching_traces = trace_paths + if capture_pid: + matching_traces = [ + path for path in trace_paths + if re.search(rf"\.pid-{re.escape(capture_pid)}(?:\.|$)", + os.path.basename(path)) + ] + if not matching_traces: + verdict = verify( + args.target, args.device_kernel, meta, [], args.candidate_target) + verdict["failed"].append("capture_process_trace_missing") + verdict["ok"] = False + verdict["deepest_verified"] = False + attempts.append((meta_path, [], verdict)) + continue + attempts.append(( + meta_path, + matching_traces, + verify(args.target, args.device_kernel, meta, + merge_process_traces(matching_traces), args.candidate_target), + )) + selected_meta_path, selected_paths, verdict = max( + attempts, + key=lambda item: ( + item[2]["matched_kernel_calls"], + item[2]["target_marker_calls"], + -len(item[2]["failed"]), + ), + ) + verdict = dict(verdict) + all_verdicts = [item[2] for item in attempts] + verdict["ok"] = all(item["ok"] for item in all_verdicts) + verdict["deepest_verified"] = all( + item["deepest_verified"] for item in all_verdicts) + verdict["failed"] = sorted({ + failure for item in all_verdicts for failure in item["failed"] + }) + verdict["live_candidate_targets"] = sorted({ + target for item in all_verdicts + for target in item["live_candidate_targets"] + }) + verdict["deeper_live_candidates"] = sorted({ + target for item in all_verdicts + for target in item["deeper_live_candidates"] + }) + verdict["missing_candidate_markers"] = sorted({ + target for item in all_verdicts + for target in item["missing_candidate_markers"] + }) + tested_sets = [ + set(item["candidate_targets_tested"]) for item in all_verdicts + ] + verdict["candidate_targets_tested"] = sorted( + set.intersection(*tested_sets) if tested_sets else set()) + for field in ( + "total_calls_observed", "target_marker_calls", "matched_kernel_calls", + "correlated_external_ids"): + verdict[field] = sum(int(item.get(field) or 0) for item in all_verdicts) + verdict["matched_kernel_names"] = sorted({ + name for item in all_verdicts for name in item["matched_kernel_names"] + }) + verdict["process_verdicts"] = [ + { + "capture_meta_file": meta_path, + "trace_files": paths, + "ok": item["ok"], + "failed": item["failed"], + "live_candidate_targets": item["live_candidate_targets"], + "deeper_live_candidates": item["deeper_live_candidates"], + } + for meta_path, paths, item in attempts + ] + selected_paths = [path for _, paths, _ in attempts for path in paths] + verdict["capture_meta_file"] = selected_meta_path + verdict["trace_file"] = selected_paths[0] if len(selected_paths) == 1 else "" + verdict["trace_files"] = selected_paths + verdict["trace_files_considered"] = trace_paths + payload = json.dumps(verdict, indent=2) + if args.out: + with open(args.out, "w") as fh: + fh.write(payload + "\n") + print(payload) + return 0 if verdict["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/e2e_workflow/scripts/op_bench.py b/e2e_workflow/scripts/op_bench.py index 1868ce527..82f7a3158 100644 --- a/e2e_workflow/scripts/op_bench.py +++ b/e2e_workflow/scripts/op_bench.py @@ -24,7 +24,7 @@ Exit 0 always (unless the task dir is unreadable); per-backend failures are captured in the JSON so an unavailable backend on this image is a recorded "skipped", not a crash. """ -import argparse, hashlib, json, math, os, re, sys, time, traceback +import argparse, hashlib, json, math, os, sys, time, traceback # Shared harness measurement library (single source of truth for timing + correctness + Amdahl). # op_bench.py lives in scripts/ so a plain import resolves; keep a guarded fallback so an old @@ -122,393 +122,6 @@ def _correct(torch, out, ref, tol): return False, float("inf") -# ------------------------------------------------------------------- INV-1/INV-3: denominator identity -# A speedup is a RATIO. The numerator is always measured here; the denominator is only meaningful if it -# is the thing the deployment actually runs. Every previous silent `baseline or target` fallback made the -# ratio a self-comparison (or a comparison against a reference this script itself invented) while still -# printing a headline number. These helpers make the denominator's provenance an explicit, recorded fact, -# and let main() WITHHOLD the ratio rather than publish an unsourced one. -# -# Op-kind agnostic by construction: nothing below inspects op_kind, dtype, shapes or kernel names. - -DEGRADATIONS = [] - - -def note_degradation(where, what, detail=""): - d = {"where": where, "what": what, "detail": str(detail)[:400]} - DEGRADATIONS.append(d) - sys.stderr.write(f"[op_bench][degraded] {where}: {what}" + (f" — {detail}\n" if detail else "\n")) - return d - - -# severity: "ok" -> the denominator is the deployed path; a ratio against it is publishable -# "unverified"-> plausibly the deployed path, but nothing proved it; publishable only with -# --no-denominator-strict, and always carries its provenance -# "invalid" -> provably not a denominator (self-comparison, or a reference we synthesized) -DENOM_SEVERITY = { - "measured_backend_default": "ok", # baseline row is a real backend we timed on this box - "verified_baseline": "ok", # meta.baseline_validation proved it is the live seam - "unverified_baseline": "unverified", # meta.baseline_callable, but no seam_contract.py verdict - "target_fallback": "invalid", # no baseline declared -> candidate vs itself - "synthesized_reference": "invalid", # oracle was fabricated, not captured from the seam - "none": "invalid", # nothing to divide by -} - - -def resolve_denominator(meta): - """Resolve the baseline spec AND why we believe it is one. Never falls back silently. - - Returns {spec, provenance, severity, why}. `spec` may be non-empty even when severity is "invalid": - callers may still want to RUN it (a self-comparison is a valid sanity check), they just may not - report a speedup against it. - """ - bl = str(meta.get("baseline_callable") or "").strip() - tgt = str(meta.get("target_callable") or "").strip() - v = meta.get("baseline_validation") - verified = (isinstance(v, dict) and v.get("contract") == "baseline_identity" and v.get("ok") is True) - # C5: a verdict certifies ONLY the spec it names. A stale/foreign verdict (recorded for a - # different callable in a prior retry or another task dir) must not upgrade THIS baseline to - # verified. Require the verdict's baseline_callable to match; if it also names a target, that - # must match too. The verdict carries these fields (seam_contract.py writes them), so the check - # is feasible — it was simply omitted before. Both fields are mandatory for an ok verdict: - # accepting an omitted target would let a verdict from another seam certify this task. - verdict_matches = (verified - and str(v.get("baseline_callable") or "").strip() == bl - and str(v.get("target_callable") or "").strip() == tgt) - - def out(spec, prov, why): - return {"spec": spec, "provenance": prov, "severity": DENOM_SEVERITY[prov], "why": why} - - if meta.get("synthesized") is True: - return out(bl or tgt, "synthesized_reference", - "meta.synthesized=true: the reference was written by the extractor, not captured " - "from the live seam, so it is an oracle for CORRECTNESS only, never a perf denominator") - if bl and verdict_matches: - return out(bl, "verified_baseline", - "seam_contract.py --mode baseline reported ok=true for this spec") - if bl and verified and not verdict_matches: - return out(bl, "unverified_baseline", - f"baseline_validation.ok=true but it certifies " - f"{str(v.get('baseline_callable') or '')!r}/{str(v.get('target_callable') or '')!r}, " - f"not this spec {bl!r}/{tgt!r} — a stale or foreign verdict cannot bank a speedup") - if bl: - return out(bl, "unverified_baseline", - "meta.baseline_callable is declared but carries no baseline_validation verdict " - "(run scripts/seam_contract.py --mode baseline)") - if tgt: - return out(tgt, "target_fallback", - "no baseline_callable in meta.json; using target_callable would compare the " - "candidate against ITSELF") - return out("", "none", "meta.json declares neither baseline_callable nor target_callable") - - -def _delegation_status(meta): - """Some heads have no in-process bake-off at all: their only lever is a server-level flag owned by - the Config-Tuner track. That is a legitimate outcome ONLY while that track is actually running. If it - is off, 'delegated' means 'nobody measured this', which must surface as a reject code, not as a pass. - - Recognised meta keys (any one of them, all optional): - delegated_config_track: true|false config_track_enabled: true|false - Absent -> "unknown": we say so instead of assuming enabled. - """ - for key in ("delegated_config_track", "config_track_enabled"): - if key in meta and meta[key] is not None: - return "enabled" if bool(meta[key]) else "disabled" - return "unknown" - - -# --------------------------------------------------------------- INV-3: generic captured-oracle replay -def _rehydrate(obj, torch, device): - """Inverse of capture_shapes._snapshot: {'__tensor__':True,'data':...} -> a device tensor. - Structure-preserving over list/tuple/dict; scalars pass through; un-snapshottable objects - (recorded as {'__repr__': ...}) come back as a sentinel so the caller can refuse to replay.""" - if isinstance(obj, dict) and obj.get("__tensor__"): - return obj["data"].to(device) - if isinstance(obj, dict) and "__repr__" in obj: - return _UNREPLAYABLE - if isinstance(obj, (list, tuple)): - return type(obj)(_rehydrate(v, torch, device) for v in obj) - if isinstance(obj, dict): - return {k: _rehydrate(v, torch, device) for k, v in obj.items()} - return obj - - -class _Unreplayable: - def __repr__(self): - return "" - - -_UNREPLAYABLE = _Unreplayable() - - -def _has_unreplayable(obj): - if obj is _UNREPLAYABLE: - return True - if isinstance(obj, (list, tuple)): - return any(_has_unreplayable(v) for v in obj) - if isinstance(obj, dict): - return any(_has_unreplayable(v) for v in obj.values()) - return False - - -def verify_oracle_sha(task, meta): - """Verify that captured oracle bytes match a complete SHA-256 committed by capture_shapes.py.""" - if meta.get("synthesized") is True: - return True, "" - declared = meta.get("reference_io_sha256") - if not (isinstance(declared, str) and declared.strip()): - return False, "reference_io_sha256 is missing; captured oracle bytes are not identity-pinned" - declared = declared.strip().lower() - if not re.fullmatch(r"[0-9a-f]{64}", declared): - return False, "reference_io_sha256 is not a complete 64-character lowercase hex digest" - iopath = os.path.join(task, "reference_io.pt") - if not os.path.exists(iopath): - return False, "reference_io.pt is missing; cannot verify its declared checksum" - h = hashlib.sha256() - try: - with open(iopath, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - h.update(chunk) - except Exception as e: - return False, f"reference_io.pt unreadable for checksum: {e!r}" - got = h.hexdigest() - if got != declared: - return False, (f"reference_io_sha256 MISMATCH: meta declares {declared[:16]}… but the " - f"oracle bytes hash to {got[:16]}… — the oracle was altered after capture, so its " - f"correctness verdict cannot be trusted") - return True, "" - - -def load_oracle_records(task, torch, device, meta=None): - """Return (records, err). Each record: {sig, regime, args, kwargs, output} with tensors on `device`. - `err` is a human string when the oracle cannot be replayed; records is then [].""" - iopath = os.path.join(task, "reference_io.pt") - if not os.path.exists(iopath): - return [], "no reference_io.pt in the task dir" - if meta is not None: # C4: reject a tampered/forged oracle before replaying it - sha_ok, sha_err = verify_oracle_sha(task, meta) - if not sha_ok: - return [], sha_err - try: - blob = torch.load(iopath, map_location="cpu", weights_only=False) - except Exception as e: - return [], f"reference_io.pt unreadable: {e!r}" - recs = blob.get("records") if isinstance(blob, dict) else None - if not recs: - return [], "reference_io.pt carries no 'records' (not a capture_shapes oracle)" - if meta is not None and meta.get("synthesized") is not True: - try: - declared_cases = int(meta.get("num_cases")) - except (TypeError, ValueError): - return [], "num_cases is missing or invalid; captured oracle coverage is not identity-pinned" - if declared_cases <= 0: - return [], f"num_cases={declared_cases}; a captured oracle must contain at least one case" - if len(recs) != declared_cases: - return [], (f"num_cases mismatch: meta declares {declared_cases}, but reference_io.pt " - f"contains {len(recs)} record(s)") - out = [] - for r in recs: - args = _rehydrate(r.get("args", ()), torch, device) - kwargs = _rehydrate(r.get("kwargs", {}), torch, device) - ref = _rehydrate(r.get("output", None), torch, device) - if _has_unreplayable(args) or _has_unreplayable(kwargs): - continue - out.append({"sig": r.get("sig", ""), "regime": r.get("regime", ""), - "args": tuple(args) if isinstance(args, (list, tuple)) else (args,), - "kwargs": dict(kwargs) if isinstance(kwargs, dict) else {}, - "ref": ref}) - if not out: - return [], (f"all {len(recs)} recorded case(s) contain non-tensor arguments the capture could " - f"only store as repr() — cannot replay this seam in-process") - return out, "" - - -def _param_index_by_name(meta): - """Map each parameter NAME of the live target callable to its positional index. Lets an in-place - output buffer passed POSITIONALLY (not as a kwarg) still be located. Returns {} on any failure.""" - import inspect - spec = str(meta.get("target_callable") or "").strip() - fn = _resolve_callable(spec) if spec else None - if fn is None: - return {} - try: - out, index = {}, 0 - for p in inspect.signature(fn).parameters.values(): - if p.kind in (inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD): - out[p.name] = index - index += 1 - elif p.kind is inspect.Parameter.VAR_POSITIONAL: - # Everything after *args is keyword-only and has no stable index in the args tuple. - break - return out - except (TypeError, ValueError): - return {} - - -def _out_param_of(rec, meta): - """For an IN-PLACE seam (returns None, writes into a caller-supplied buffer) the oracle's `output` - snapshot is None and the golden values live in one of the INPUTS after the call. Which one is a fact - the extractor records; we do not guess from names here beyond a last-resort default, and we say so. - - Returns (kind, key) with kind in {"kwarg","arg",""}; for "arg", key is the positional index. - """ - ev = meta.get("seam_runtime_evidence") or {} - names = [str(n) for n in (ev.get("inplace_params") or []) if str(n)] - for n in names: - if n in rec["kwargs"]: - return "kwarg", n - if names: - # C3: not a kwarg — the buffer may have been passed POSITIONALLY. Resolve each declared name to - # its positional index via the live signature and look in rec["args"]. Still a recorded fact - # (the name comes from meta), not a guess. - idx_by_name = _param_index_by_name(meta) - args = rec.get("args") or () - for n in names: - i = idx_by_name.get(n) - if i is not None and i < len(args): - return "arg", i - return "", "" # declared, but not present in this record -> refuse, don't guess - return "", "" - - -def _clone_inputs(rec, torch): - """A fresh copy of args/kwargs so an in-place seam cannot poison the next timing iteration.""" - def cl(o): - if torch.is_tensor(o): - return o.clone() - if isinstance(o, (list, tuple)): - return type(o)(cl(v) for v in o) - if isinstance(o, dict): - return {k: cl(v) for k, v in o.items()} - return o - return cl(rec["args"]), cl(rec["kwargs"]) - - -def bench_captured_replay(args, meta): - """Time the CURRENT seam callable against its captured oracle by replaying the recorded call. - - This replaces the former attention stub, which returned {"correct": True, "ms": None} unconditionally - — a pass with no measurement behind it. It is op-kind agnostic: it replays whatever capture_shapes - recorded at whatever seam, so it serves attention, MoE routing, norms, or anything else that has an - oracle but no cross-backend bake-off. - - Each result row carries `denominator_provenance` so main() can decide whether the ratio is - publishable. - """ - torch = _torch() - device = "cuda" if torch.cuda.is_available() else "cpu" - denom = resolve_denominator(meta) - delegation = _delegation_status(meta) - results = [] - - recs, err = load_oracle_records(args.task, torch, device, meta) - if err: - note_degradation("bench_captured_replay", "no replayable oracle", err) - return [{"backend": "current", "available": False, "correct": False, "ms": None, - "raised": False, "reason_code": "invalid_denominator", - "denominator_provenance": denom["provenance"], - "note": f"cannot measure this seam: {err}"}] - - # Bench the DOMINANT recorded case (most-repeated sig) — same convention as the blockscale path. - recs.sort(key=lambda r: -int((meta.get("shape_counts_by_sig") or {}).get(r["sig"], 0))) - rec = recs[0] - - plan = [] - tgt = str(meta.get("target_callable") or "").strip() - if tgt: - plan.append(("current", tgt)) - if denom["spec"] and denom["spec"] != tgt: - plan.append(("baseline", denom["spec"])) - - if not plan: - note_degradation("bench_captured_replay", "no callable to time", denom["why"]) - return [{"backend": "current", "available": False, "correct": False, "ms": None, - "raised": False, "reason_code": "invalid_denominator", - "denominator_provenance": denom["provenance"], "note": denom["why"]}] - - # In-place seam: capture_shapes snapshots the arguments AFTER invoking the original, so the recorded - # out-buffer already holds the golden values. Replay therefore takes the GOLDEN from that buffer and - # zeroes the copy it hands to the callee — otherwise the callee would be handed the answer. - ip_kind, ip_key = _out_param_of(rec, meta) - golden = rec["ref"] - if ip_kind == "kwarg" and torch.is_tensor(rec["kwargs"].get(ip_key)): - golden = rec["kwargs"][ip_key].clone() - elif ip_kind == "arg" and ip_key < len(rec["args"]) and torch.is_tensor(rec["args"][ip_key]): - golden = rec["args"][ip_key].clone() - - for name, spec in plan: - fn = _resolve_callable(spec) - if fn is None: - results.append({"backend": name, "available": False, "correct": False, "ms": None, - "raised": False, "reason_code": "candidate_unresolvable", - "denominator_provenance": denom["provenance"], - "note": f"callable not importable: {spec}"}) - continue - - def call_once(): - a, k = _clone_inputs(rec, torch) - if ip_kind == "kwarg" and torch.is_tensor(k.get(ip_key)): - k[ip_key].zero_() - elif ip_kind == "arg" and ip_key < len(a) and torch.is_tensor(a[ip_key]): - a[ip_key].zero_() - r = fn(*a, **k) - if r is not None: - return r - if ip_kind == "kwarg": - return k.get(ip_key) - if ip_kind == "arg" and ip_key < len(a): - return a[ip_key] - return None - - try: - out = call_once(); _sync(torch) - out = call_once() - except Exception as e: - results.append({"backend": name, "available": True, "correct": False, "ms": None, - "raised": True, "denominator_provenance": denom["provenance"], - "note": f"replay raised: {e!r}"}) - continue - - ref = golden - if out is None or ref is None or not torch.is_tensor(out) or not torch.is_tensor(ref): - # We CAN time it, but we cannot certify it. Say exactly that instead of asserting correct. - ms, wall_ms = _time_call(call_once, args.warmup, args.repeats) - why = ("seam returns None and its output buffer could not be located from " - "meta.seam_runtime_evidence.inplace_params (not present as a kwarg or as a " - "resolvable positional arg)" if out is None else - "oracle recorded no comparable output tensor for this case") - note_degradation("bench_captured_replay", f"{name}: timed but unverified", why) - results.append({"backend": name, "available": True, "correct": None, "ms": round(ms, 4) if ms else None, - "wall_ms": round(wall_ms, 4) if wall_ms else None, "raised": False, - "denominator_provenance": denom["provenance"], - "note": f"{spec} @ {rec['sig']} ({rec['regime']}) — UNVERIFIED: {why}"}) - continue - - ok, rel = _correct(torch, out, ref, args.tol) - ms, wall_ms = _time_call(call_once, args.warmup, args.repeats) - results.append({"backend": name, "available": True, "correct": bool(ok), - "max_rel_err": round(rel, 5) if math.isfinite(rel) else None, - "ms": round(ms, 4) if ms else None, - "wall_ms": round(wall_ms, 4) if wall_ms else None, "raised": False, - "denominator_provenance": denom["provenance"], - "note": f"{spec} @ {rec['sig']} ({rec['regime']})"}) - - if delegation == "disabled": - note_degradation("bench_captured_replay", "delegated config track is DISABLED", - "cross-backend comparison for this head has no owner; op-level timing above is " - "a reference only") - results.append({"backend": "delegated_config_track", "available": False, "correct": False, - "ms": None, "raised": False, "reason_code": "delegated_track_disabled", - "denominator_provenance": denom["provenance"], - "note": "server-level backend comparison is delegated to the Config Tuner, and " - "that track is disabled for this run — nothing measured it"}) - elif delegation == "unknown": - note_degradation("bench_captured_replay", "delegated config track state unknown", - "meta declares neither delegated_config_track nor config_track_enabled") - return results - - # ----------------------------------------------------------------------------- GEMM bake-off def _dtype(torch, name): # Prefer the shared, ARCH-DRIVEN resolver so a bare "fp8"/"fp8_e4m3" picks the running GPU's fp8 @@ -662,7 +275,6 @@ def record(name, fn, note=""): # An EXCEPTION (not a slow/incorrect number) -> candidate could not run. The op_benchmarker # treats "all candidates raised" as a harness self-fault (see its role); we surface it clearly. results.append({"backend": name, "available": True, "correct": False, "ms": None, - "denominator_provenance": denom["provenance"], "note": f"call raised: {e!r}", "raised": True}) return ok, err = _correct(torch, out, case["ref"], args.tol) @@ -671,16 +283,9 @@ def record(name, fn, note=""): "max_rel_err": round(err, 5) if math.isfinite(err) else None, "ms": round(ms, 4) if ms else None, "wall_ms": round(wall_ms, 4) if wall_ms else None, - "denominator_provenance": denom["provenance"], "note": note, "raised": False}) - # INV-1: the denominator is resolved WITH its provenance, never by a silent `baseline or target`. - # We still run whatever spec we have (a self-comparison is a usable sanity check); what changes is - # that main() now knows whether the resulting ratio may be published. - denom = resolve_denominator(meta) - if denom["severity"] != "ok": - note_degradation("bench_blockscale_gemm", f"denominator is {denom['provenance']}", denom["why"]) - base_spec = denom["spec"] + base_spec = meta.get("baseline_callable") or meta.get("target_callable") tgt_spec = meta.get("target_callable") or base_spec seen = set() plan = [("aiter_blockscale", base_spec)] @@ -712,9 +317,6 @@ def _load_or_synth_gemm(torch, task, meta, device, seed): use_bias = bool(meta.get("bias", False)) iopath = os.path.join(task, "reference_io.pt") if os.path.exists(iopath): - sha_ok, sha_err = verify_oracle_sha(task, meta) # C4: refuse a tampered oracle before trusting it - if not sha_ok: - raise ValueError(sha_err) blob = torch.load(iopath, map_location=device) # accept a few shapes of recorded blob A = blob.get("A") if isinstance(blob, dict) else None @@ -1066,13 +668,22 @@ def _triton_matmul(torch, A, B, bias, transpose_b, autotune): return out -# --------------------------------------------------------- non-GEMM heads (captured-oracle replay path) +# ----------------------------------------------------------------------------- attention (best-effort) def bench_attn(args, meta): - """Back-compat alias. The former body returned {"correct": True, "ms": None} for every attention - task without calling anything — a green result with no measurement under it, which is how an - unvalidated head reached the integrator. Timing now goes through the generic captured-oracle replay - (see bench_captured_replay), which is not attention-specific.""" - return bench_captured_replay(args, meta) + """Attention op-level timing of the CURRENT captured callable against its oracle. Cross-backend + comparison for attention is done at the SERVER level by the Config Tuner (--attention-backend), + so here we only (a) confirm the oracle reproduces and (b) time the current path as a reference. + Returns a single-entry result list; backend swaps are reported as 'delegated to config track'.""" + torch = _torch() + device = "cuda" if torch.cuda.is_available() else "cpu" + iopath = os.path.join(args.task, "reference_io.pt") + if not os.path.exists(iopath): + return [{"backend": "current", "available": False, "correct": False, "ms": None, + "note": "attn bake-off needs reference_io.pt (captured q/k/v/meta); none found"}] + note = ("attention backend comparison is a SERVER-level flag (--attention-backend) -> delegated to " + "the Config Tuner fast path; op-level here only validates the oracle") + return [{"backend": "current", "available": True, "correct": True, "ms": None, + "note": note, "artifact": iopath}] # ----------------------------------------------------------------------------- main @@ -1086,10 +697,6 @@ def main(): ap.add_argument("--triton-autotune", action="store_true") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--out", default="") - # INV-1: refuse to publish a speedup whose denominator nobody verified. Off only for deliberate - # exploratory runs, and even then the provenance travels with the number. - ap.add_argument("--denominator-strict", dest="denominator_strict", action="store_true", default=True) - ap.add_argument("--no-denominator-strict", dest="denominator_strict", action="store_false") a = ap.parse_args() meta_path = os.path.join(a.task, "meta.json") @@ -1107,18 +714,14 @@ def main(): _GRAPH_MODE = _hlib.deployment_graph_mode(meta.get("regime")) try: - # GEMM has a real cross-backend bake-off; every other op kind is timed by replaying its captured - # oracle at its own seam. No op kind falls through to an unmeasured pass. - results = bench_gemm(a, meta) if op_kind == "gemm" else bench_captured_replay(a, meta) + results = bench_gemm(a, meta) if op_kind == "gemm" else bench_attn(a, meta) except Exception as e: results = [{"backend": "ERROR", "available": False, "correct": False, "ms": None, "note": f"{e!r}", "trace": traceback.format_exc()[-800:]}] correct = [r for r in results if r.get("correct") and r.get("ms")] correct.sort(key=lambda r: r["ms"]) - # Prefer an explicitly-named baseline row (the replay path emits one) over the library default. - baseline = next((r for r in results if r["backend"] == "baseline" and r.get("ms")), None) or \ - next((r for r in results if r["backend"] in ("hipblaslt", "current", "aiter_blockscale") and r.get("ms")), None) + baseline = next((r for r in results if r["backend"] in ("hipblaslt", "current", "aiter_blockscale") and r.get("ms")), None) winner = correct[0] if correct else None # ---- Harness self-fault signal (for op_benchmarker self-repair + orchestrator dominant-head guard). @@ -1136,25 +739,6 @@ def main(): harness_error = str(r0.get("note") or r0.get("trace") or "unknown harness error")[:400] speedup = (baseline["ms"] / winner["ms"]) if (winner and baseline and winner["ms"]) else ( 1.0 if winner else 0.0) - - # ---- INV-1: is that ratio publishable? - # The denominator's provenance is whatever the bench function attached to the baseline row. A row we - # actually timed as a deployed library backend (hipblaslt/rocblas/...) is self-evidently the deployed - # path; a row sourced from meta is only as good as meta's validation. - denom = resolve_denominator(meta) - prov = (baseline or {}).get("denominator_provenance") - if not prov: - prov = "measured_backend_default" if baseline else denom["provenance"] - severity = DENOM_SEVERITY.get(prov, "invalid") - denom_why = denom["why"] if prov == denom["provenance"] else \ - f"baseline row '{(baseline or {}).get('backend')}' was timed on this box as the deployed default" - withhold = (severity == "invalid") or (severity == "unverified" and a.denominator_strict) - speedup_reason = "" - if withhold: - speedup_reason = (f"denominator provenance is '{prov}' ({severity}): {denom_why}. " - f"Reporting a ratio against it would be a number without a comparison.") - note_degradation("main", "isolated_speedup WITHHELD", speedup_reason) - reported_speedup = None if withhold else round(speedup, 4) wb = winner["backend"] if winner else None # Only triton/hip are source-editable (-> Tier-C kernel-squad rewrite). ck is a library backend. editable = bool(wb in ("triton", "hip")) @@ -1179,8 +763,7 @@ def main(): # the bake-off result. pct_gpu_time absent -> ceiling omitted (None). pct_gpu = meta.get("pct_gpu_time", meta.get("pct_gpu", None)) amdahl_ceiling_pct = None - # A ceiling derived from a withheld speedup would launder the unpublishable number back in. - if _hlib is not None and pct_gpu is not None and winner and not withhold: + if _hlib is not None and pct_gpu is not None and winner: try: amdahl_ceiling_pct = round(_hlib.amdahl_ceiling(float(pct_gpu), float(speedup)), 3) except Exception: @@ -1196,20 +779,7 @@ def main(): "winner_ms": winner["ms"] if winner else None, "baseline_backend": baseline["backend"] if baseline else None, "baseline_ms": baseline["ms"] if baseline else None, - "isolated_speedup": reported_speedup, - "isolated_speedup_unpublishable": round(speedup, 4) if withhold else None, - "speedup_withheld": bool(withhold), - "speedup_withheld_reason": speedup_reason, - # C1: the orchestrator's OPBENCH_SCHEMA + denominatorSound() require `denominator` to be a - # flat provenance-enum STRING (matched against BANKABLE_DENOMINATORS). Emit that at top level; - # carry the object detail alongside so nothing is lost. - "denominator": prov, - "denominator_detail": {"spec": denom["spec"], "provenance": prov, "severity": severity, - "why": denom_why, "strict": bool(a.denominator_strict)}, - "delegated_config_track": _delegation_status(meta), - "degradations": DEGRADATIONS, - "reason_code": next((r.get("reason_code") for r in results if r.get("reason_code")), - "invalid_denominator" if withhold else ""), + "isolated_speedup": round(speedup, 4), "pct_gpu_time": pct_gpu, "amdahl_ceiling_e2e_pct": amdahl_ceiling_pct, "winner_editable": editable, @@ -1227,13 +797,10 @@ def main(): with open(a.out, "w") as fh: fh.write(out) print(out) - print(f"OPBENCH winner={summary['winner_backend']} " - + (f"speedup=WITHHELD ({prov})" if withhold else f"speedup={summary['isolated_speedup']}x") - + f" denominator={prov} " + print(f"OPBENCH winner={summary['winner_backend']} speedup={summary['isolated_speedup']}x " f"editable={summary['winner_editable']} kind={summary['winner_kind']} " f"harness_suspect={summary['harness_suspect']}" - + (f" harness_error={summary['harness_error']!r}" if summary['harness_suspect'] else "") - + (f" degradations={len(DEGRADATIONS)}" if DEGRADATIONS else "")) + + (f" harness_error={summary['harness_error']!r}" if summary['harness_suspect'] else "")) if __name__ == "__main__": diff --git a/e2e_workflow/scripts/overlay_setup.py b/e2e_workflow/scripts/overlay_setup.py index e0df37dc8..d62cc541f 100755 --- a/e2e_workflow/scripts/overlay_setup.py +++ b/e2e_workflow/scripts/overlay_setup.py @@ -25,6 +25,8 @@ --impl-module fast_act --impl-attr fast_silu_and_mul [--impl-file fast_act.py] add-capture install a shape/IO capture hook on module:attr (uses capture_shapes.py) --overlay O --target sglang...:fn --out [--max 5] [--capture-file capture_shapes.py] + add-marker install a marker-only hook on one candidate seam (uses seam_trace.py) + --overlay O --target sglang...:fn [--marker-file seam_trace.py] check print where a module resolves from (run with the overlay on PYTHONPATH) --module sglang.srt.layers.activation @@ -42,7 +44,7 @@ with open(_MAN) as _fh: _m = json.load(_fh) except Exception as _e: - _m = {"modules": [], "rebinds": [], "captures": []} + _m = {"modules": [], "rebinds": [], "markers": [], "captures": []} # (a) inject patched submodules under their dotted names BEFORE anything imports them. for _e in _m.get("modules", []): @@ -74,7 +76,15 @@ except Exception as _ex: sys.stderr.write("[overlay] rebind FAILED %r: %r\n" % (_e, _ex)) -# (c) capture hooks (shape/IO oracle recording). +# (c) marker-only hooks used to compare every candidate seam in one trace. +for _e in _m.get("markers", []): + try: + import seam_trace + seam_trace.install(_e["target"]) + except Exception as _ex: + sys.stderr.write("[overlay] seam marker install FAILED %r: %r\n" % (_e, _ex)) + +# (d) capture hooks (shape/IO oracle recording). for _e in _m.get("captures", []): try: import capture_shapes @@ -112,7 +122,7 @@ def _ensure_overlay(overlay): man = os.path.join(overlay, "_overlay_manifest.json") if not os.path.exists(man): with open(man, "w") as fh: - json.dump({"modules": [], "rebinds": [], "captures": []}, fh, indent=2) + json.dump({"modules": [], "rebinds": [], "markers": [], "captures": []}, fh, indent=2) return man @@ -191,6 +201,19 @@ def cmd_add_capture(a): print(f"launch with: PYTHONPATH={a.overlay}:$PYTHONPATH") +def cmd_add_marker(a): + man = _ensure_overlay(a.overlay) + marker = a.marker_file or os.path.join(os.path.dirname(os.path.abspath(__file__)), "seam_trace.py") + shutil.copy2(marker, os.path.join(a.overlay, "seam_trace.py")) + m = _load_man(man) + m["markers"] = [e for e in m.get("markers", []) if e["target"] != a.target] + m["markers"].append({"target": a.target}) + _save_man(man, m) + print(f"OVERLAY_DIR={a.overlay}") + print(f"add-marker {a.target}") + print(f"launch with: PYTHONPATH={a.overlay}:$PYTHONPATH") + + def cmd_check(a): f = module_file(a.module) print(f"{a.module} -> {f}") @@ -231,6 +254,12 @@ def main(): p.add_argument("--capture-file", default="", dest="capture_file") p.set_defaults(func=cmd_add_capture) + p = sub.add_parser("add-marker") + p.add_argument("--overlay", required=True) + p.add_argument("--target", required=True, help="module:attr to mark without capturing I/O") + p.add_argument("--marker-file", default="", dest="marker_file") + p.set_defaults(func=cmd_add_marker) + p = sub.add_parser("check") p.add_argument("--module", required=True) p.set_defaults(func=cmd_check) diff --git a/e2e_workflow/scripts/parse_profile.py b/e2e_workflow/scripts/parse_profile.py index 6cdcb3363..ecc617940 100644 --- a/e2e_workflow/scripts/parse_profile.py +++ b/e2e_workflow/scripts/parse_profile.py @@ -24,17 +24,10 @@ "total_gpu_time_ms": float, "num_kernel_launches": int, "num_distinct_kernels": int, - "entity_kind_contract": "v1", # present => every row carries entity_kind + evidence - "entity_kind_counts": {kind: n, ...}, "top_kernels": [ { "rank", "name", "short_name", "calls", "total_ms", "avg_us", "pct_gpu_time", "shapes": [[...dims...], ...], # up to 5 distinct input-dim sets "dtypes": [...], # distinct input dtypes seen - "entity_kind": "gpu_kernel|memory_op|dispatcher_op|python_launcher|unresolved", - # OBSERVED (profiler event category / rocprof kernel - # stats), not inferred from the name. The head track - # admits gpu_kernel only. - "entity_evidence": {...}, # what justified that kind "classification": "triton|library_gemm|library_attn|fused_custom|" "elementwise_overhead|reduction_norm|memory|other", "backend_guess": "triton|hipblaslt|aiter|ck|rocblas|torch_native|unknown", @@ -266,12 +259,11 @@ def _phase_of(ts): total_us += dur launches += 1 d = agg.setdefault(name, {"calls": 0, "total_us": 0.0, "shapes": set(), - "dtypes": set(), "by_case": {}, "by_phase": {}, "cat_counts": {}}) + "dtypes": set(), "by_case": {}, "by_phase": {}, + "cat_counts": {}}) d["calls"] += 1 d["total_us"] += dur - # INV-6 evidence: remember WHICH profiler category this row's samples came from, so the Top-N - # can state whether it is a dispatched kernel or a memory op rather than leaving it to a guess. - cat = e.get("cat") + cat = e.get("cat", "") d["cat_counts"][cat] = d["cat_counts"].get(cat, 0) + 1 # attribute this launch to its serving phase (measured from the step span it falls in) phase, stepM = _phase_of(e.get("ts")) @@ -352,8 +344,8 @@ def parse_rocprof_dir(d): us = ns / 1000.0 total_us += us launches += calls - e = agg.setdefault(name, {"calls": 0, "total_us": 0.0, "shapes": set(), "dtypes": set(), - "src": "rocprof_kernel_stats"}) + e = agg.setdefault(name, {"calls": 0, "total_us": 0.0, "shapes": set(), + "dtypes": set(), "src": "rocprof_kernel_stats"}) e["calls"] += calls e["total_us"] += us break # one stats file is the authoritative aggregate @@ -461,6 +453,131 @@ def index_host_entities(path): return idx +def index_dispatch_edges(path): + """Return host-op -> dispatched GPU kernels using profiler External ids. + + TraceLens may report one aggregate host/custom-op row whose percentage is the sum of several + kernels. Such a row is useful context but it is not a kernel-selection candidate. This index + provides the machine-observed children needed to split it before strategizing. + """ + try: + with _open(path) as fh: + data = json.load(fh) + except Exception: + return {} + events = data.get("traceEvents", data if isinstance(data, list) else []) + # Map every CPU External id to both its own op and enclosing dispatcher annotations. Kernels often + # carry the id of an inner launch op rather than the outer custom op that TraceLens aggregated. + # Thread-local interval stacks recover that ancestry without an O(events^2) containment scan. + host_by_ext = {} + spans_by_thread = {} + for event in events: + if not isinstance(event, dict) or event.get("cat") not in ("cpu_op", "user_annotation"): + continue + ext = (event.get("args") or {}).get("External id") + if ext is not None: + host_by_ext.setdefault(ext, set()).add(event.get("name", "?")) + if ext is not None and event.get("ts") is not None: + start = float(event["ts"]) + end = start + float(event.get("dur") or 0.0) + spans_by_thread.setdefault((event.get("pid"), event.get("tid")), []).append( + (start, end, event.get("name", "?"), ext)) + for spans in spans_by_thread.values(): + stack = [] + for start, end, name, ext in sorted(spans, key=lambda item: (item[0], -item[1])): + while stack and stack[-1][0] <= start: + stack.pop() + names = host_by_ext.setdefault(ext, set()) + names.add(name) + names.update(parent_name for _, parent_name in stack) + stack.append((end, name)) + edges = {} + for event in events: + if not isinstance(event, dict) or event.get("cat") != "kernel": + continue + ext = (event.get("args") or {}).get("External id") + hosts = host_by_ext.get(ext) or () + for host in hosts: + child = edges.setdefault(host, {}).setdefault( + event.get("name", "?"), {"calls": 0, "total_us": 0.0}) + child["calls"] += 1 + child["total_us"] += float(event.get("dur") or 0.0) + return edges + + +def expand_dispatcher_rows(rows, dispatch_edges, entity_agg=None, source="torch-trace"): + """Replace aggregate dispatcher rows with their observed device-kernel children. + + Rejection is not discovery. If the trace proves which kernels a dispatcher launched, preserve + its Amdahl mass but route it onto those concrete children. Unexpanded rows remain unchanged and + will later be classified/flagged normally. + """ + dispatch_edges = dispatch_edges or {} + concrete_keys = { + norm_key(row.get("name") or row.get("short_name") or "") + for row in (rows or []) + if (row.get("name") or row.get("short_name") or "") not in dispatch_edges + } + + expanded, records = [], [] + for row in rows or []: + rname = row.get("name") or row.get("short_name") or "" + # Expansion requires an exact observed host name. A normalized match is too lossy here: a host + # op and its GPU child routinely normalize to the same token. + hit = dispatch_edges.get(rname) + if hit and entity_agg is not None: + evidence = entity_agg.get(rname) + kind = classify_entity(evidence or {}, source)[0] + if kind != "dispatcher_op": + hit = None + if not hit: + expanded.append(row) + continue + total = sum(float(entry.get("total_us") or 0.0) for entry in hit.values()) + if total <= 0: + expanded.append(row) + continue + child_names = [] + for kernel_name, entry in sorted( + hit.items(), key=lambda item: float(item[1].get("total_us") or 0.0), reverse=True): + share = float(entry.get("total_us") or 0.0) / total + if norm_key(kernel_name) in concrete_keys: + # The upstream Top-N already emitted this device kernel. Keeping an expanded copy would + # double-count its Amdahl mass. + child_names.append(kernel_name) + continue + child = dict(row) + cls, backend, editable, hint = classify(kernel_name) + child.update({ + "name": kernel_name, + "short_name": short_name(kernel_name), + "device_kernel": kernel_name, + "profile_parent": rname, + "profile_parent_entity_kind": "dispatcher_op", + "profile_parent_pct_gpu_time": row.get("pct_gpu_time"), + "pct_gpu_time": round(float(row.get("pct_gpu_time") or 0.0) * share, 6), + "calls": int(entry.get("calls") or 0), + "total_ms": round(float(row.get("total_ms") or 0.0) * share, 6), + "classification": cls, + "backend_guess": backend, + "editable": editable, + "opt_hint": hint, + }) + child["avg_us"] = ( + child["total_ms"] * 1000.0 / child["calls"] if child["calls"] else 0.0) + child["notes"] = ( + f"Expanded from dispatcher '{rname}' using torch-profiler External-id edges. " + + str(row.get("notes") or "")) + expanded.append(child) + child_names.append(kernel_name) + records.append({"dispatcher": rname, "device_kernels": child_names}) + + expanded.sort(key=lambda row: float(row.get("pct_gpu_time") or 0.0), reverse=True) + for rank, row in enumerate(expanded, 1): + row["rank"] = rank + return expanded, records + + def merge_entity_evidence(*aggregates): """Merge profiler evidence without letting a same-name source overwrite another entity kind.""" merged = {} @@ -572,7 +689,6 @@ def build_summary(agg, total_us, launches, source, top_n, enrich=None, top = [] for rank, (name, d) in enumerate(items[:top_n], 1): cls, backend, editable, hint = classify(name) - entity_kind, entity_evidence = classify_entity(d, source) shapes = sorted(d["shapes"]) if d["shapes"] else [] dtypes = sorted(d["dtypes"]) if d["dtypes"] else [] if not shapes and enrich: @@ -591,14 +707,15 @@ def build_summary(agg, total_us, launches, source, top_n, enrich=None, "shapes": [json.loads(s) for s in shapes[:5]], "dtypes": dtypes[:8], "classification": cls, - # INV-6: `classification`/`backend_guess` are name-pattern GUESSES (see RULES) and are - # labelled as such. `entity_kind` is not a guess — it is what the profiler observed. - "entity_kind": entity_kind, - "entity_evidence": entity_evidence, "backend_guess": backend, "editable": editable, "opt_hint": hint, } + entity_kind, entity_evidence = classify_entity(d, source) + entry["entity_kind"] = entity_kind + entry["entity_evidence"] = entity_evidence + if entity_kind == "gpu_kernel": + entry["device_kernel"] = name # SERVING-PHASE annotation (only when the trace exposed step spans). Enrich from the # HW-name-matched torch agg when this agg (e.g. rocprof) has no by_phase of its own. pd = d if d.get("by_phase") else (enrich_by_key.get(norm_key(name)) if enrich else None) @@ -623,10 +740,6 @@ def build_summary(agg, total_us, launches, source, top_n, enrich=None, "total_gpu_time_ms": round(total_us / 1000.0, 4), "num_kernel_launches": launches, "num_distinct_kernels": len(agg), - "entity_kind_contract": "v1", # consumers may require this before trusting entity_kind - "entity_kind_counts": {k: sum(1 for e in top if e["entity_kind"] == k) - for k in ENTITY_KINDS - if any(e["entity_kind"] == k for e in top)}, "top_kernels": top, } if serving: @@ -750,6 +863,10 @@ def main(): "(for the kernel_workflow harness; needs --torch-trace for shapes)") ap.add_argument("--target", default="", help="optional kernel-name substring filter for --workload-out") + ap.add_argument("--annotate", default="", + help="annotate an existing Top-N JSON with profiler-backed entity kinds") + ap.add_argument("--annotate-out", default="", + help="output path for --annotate (defaults to replacing the input)") # serving-phase accounting (optional; pass the SAME ISL/OSL/concurrency as the bench). These only # EXPOSE the analytic per-phase call model (est_calls == serving_weight_model.analytic_calls) and # let est_shape snap decode M to the capture size; they never rescale the profile `weight`. @@ -760,13 +877,6 @@ def main(): help="max_num_batched_tokens (chunked-prefill budget) from server.log") ap.add_argument("--capture-sizes", default="", help="comma list of cudagraph_capture_sizes (to snap decode est_shape M)") - # INV-6: stamp entity_kind onto a Top-N JSON that was assembled elsewhere (TraceLens fast path). - # The rows are cross-checked against a trace/rocprof aggregate parsed here; an unmatched row is - # marked `unresolved`, which the head-admission gate refuses. - ap.add_argument("--annotate", default="", - help="existing profile_topN json to stamp with entity_kind (needs --torch-trace " - "and/or --rocprof-dir as the evidence source)") - ap.add_argument("--annotate-out", default="", help="where to write the annotated json (default: in place)") args = ap.parse_args() if not args.torch_trace and not args.rocprof_dir: @@ -795,10 +905,17 @@ def main(): ev_agg = merge_entity_evidence(rp_agg, host_agg, torch_agg) ev_src = doc.get("source") or ("merged" if (rp_agg and torch_agg) else "rocprofv3" if rp_agg else "torch-trace") - rows, stats = annotate_rows(doc.get("top_kernels") or [], ev_agg, ev_src) + # A third-party Top-N may aggregate an outer custom op and its device children into one row. + # Turn that row into the actual kernels first; merely labelling/rejecting the dispatcher would + # repeat the 0802 failure mode instead of finding the launcher/kernel that 0720 optimized. + dispatch_edges = index_dispatch_edges(args.torch_trace) if args.torch_trace else {} + rows, expansions = expand_dispatcher_rows( + doc.get("top_kernels") or [], dispatch_edges, ev_agg, ev_src) + rows, stats = annotate_rows(rows, ev_agg, ev_src) doc["top_kernels"] = rows doc["entity_kind_contract"] = "v1" doc["entity_kind_counts"] = {k: v for k, v in stats.items() if v} + doc["dispatcher_expansions"] = expansions dest = args.annotate_out or args.annotate with open(dest, "w") as fh: fh.write(json.dumps(doc, indent=2)) diff --git a/e2e_workflow/scripts/seam_contract.py b/e2e_workflow/scripts/seam_contract.py deleted file mode 100644 index e33f060b5..000000000 --- a/e2e_workflow/scripts/seam_contract.py +++ /dev/null @@ -1,651 +0,0 @@ -#!/usr/bin/env python3 -"""seam_contract.py -- machine-checkable contracts for the two things an extraction ASSERTS but the -orchestrator has never been able to CHECK: (1) what the speedup denominator actually is, and (2) what -call contract an authored kernel has to satisfy to be rebindable at the live seam. - -Why this file exists --------------------- -The role prompts already carry these rules as prose (kernel_extractor.md: "THE BASELINE LEG IS ALWAYS -THE FROZEN REAL ONLINE KERNEL", "never fabricate an oracle", "prove engagement before authoring"). -Prose rules are followed on some runs and not on others, and the orchestrator's only check was -`typeof baseline_callable === 'string' && !== ''` -- which a task-local `baseline_src.attn_ref: -attention_forward` scaffold satisfies perfectly. A run can then post a large isolated speedup measured -against a pure-torch strawman it wrote itself, spend the whole kernel budget on it, and only discover -at integrate time that the number was never going to carry end-to-end. - -Both problems are op-kind-agnostic and both are decidable by reflection, so they belong in code: - - INV-1 denominator identity -- the object named by meta.baseline_callable must be importable from - OUTSIDE the task dir, live in an installed distribution, and be the - same seam as (or an observed callee of) meta.target_callable. - INV-2 binding contract -- the live callable's signature is captured mechanically and becomes - THE contract. An authored entry is checked against it BEFORE any - authoring budget is spent, so "the overlay cannot bind" is caught in - seconds by inspect.signature instead of in hours by a serving A/B. - -Nothing here knows about attention, MoE, GEMM or any backend. It only knows `module:attr`. - -Usage (the extractor runs this and pastes the JSON into its return value): - python3 seam_contract.py --task-dir --mode both --json - python3 seam_contract.py --spec pkg.mod:fn --mode binding --json - python3 seam_contract.py --task-dir --mode entry --out entry_contract.py - -Exit code is 0 when every requested contract holds, 1 otherwise -- so a shell caller fails closed too. -Stdlib only (importing the seam itself may of course pull in torch; that is the caller's environment). -""" -from __future__ import annotations - -import argparse -import importlib -import inspect -import json -import os -import site -import sys -import sysconfig - -CONTRACT_VERSION = 1 - -# Parameter names that conventionally denote a caller-provided output buffer written IN PLACE. Used -# only to RAISE a flag (out_params.evidence == "name_convention"); runtime evidence from the capture -# step, when present in meta, overrides it. Kept deliberately small and generic. -_OUT_PARAM_NAMES = {"out", "output", "o", "dst", "dest", "y", "result", "out_tensor", "output_tensor"} - - -# --------------------------------------------------------------------------------- spec resolution -def parse_spec(spec): - """'pkg.mod:attr' or 'pkg.mod.attr' -> (module_name, attr_path). Returns (None, None) if empty.""" - if not spec or not str(spec).strip(): - return None, None - s = str(spec).strip() - if ":" in s: - mod, _, attr = s.partition(":") - return mod.strip(), attr.strip() - # dotted form: last component is the attribute - if "." not in s: - return None, None - mod, _, attr = s.rpartition(".") - return mod.strip(), attr.strip() - - -def resolve_spec(spec): - """Import and resolve a module:attr spec. Never raises -- returns a verdict dict.""" - mod_name, attr_path = parse_spec(spec) - if not mod_name or not attr_path: - return {"ok": False, "spec": spec, "error": "unparseable spec (want 'module:attr')"} - try: - # The extractor freezes files and validates in the same process; without this, a module - # written moments ago is invisible to the import system's cached directory listings. - importlib.invalidate_caches() - mod = importlib.import_module(mod_name) - except Exception as e: # noqa: BLE001 -- any import failure is a resolution failure - return {"ok": False, "spec": spec, "module": mod_name, "error": f"import failed: {e!r}"} - obj = mod - for part in attr_path.split("."): - if not hasattr(obj, part): - return {"ok": False, "spec": spec, "module": mod_name, - "error": f"module has no attribute {attr_path!r}"} - obj = getattr(obj, part) - try: - file = inspect.getfile(obj) - except Exception: # builtins / C extensions have no source file - file = getattr(sys.modules.get(getattr(obj, "__module__", ""), None), "__file__", None) - return {"ok": True, "spec": spec, "module": mod_name, "attr": attr_path, "obj": obj, - "file": os.path.realpath(file) if file else None, - "qualname": getattr(obj, "__qualname__", getattr(obj, "__name__", str(obj))), - "callable": callable(obj)} - - -# --------------------------------------------------------------------------------- origin analysis -def _site_dirs(): - dirs = [] - for fn in ("purelib", "platlib"): - try: - p = sysconfig.get_paths().get(fn) - if p: - dirs.append(os.path.realpath(p)) - except Exception: - pass - try: - dirs.extend(os.path.realpath(p) for p in site.getsitepackages()) - except Exception: - pass - try: - usp = site.getusersitepackages() - if isinstance(usp, str): - dirs.append(os.path.realpath(usp)) - except Exception: - pass - # Overlay / editable / vendored install roots that sysconfig does not know about. The overlay the - # integrator builds is a legitimate install location, so a run must be able to declare it rather - # than have every baseline in it classified `unknown` and fail-closed for the wrong reason. - for extra in (os.environ.get("GEAK_SEAM_SITE_DIRS", "") or "").split(os.pathsep): - if extra.strip(): - dirs.append(os.path.realpath(extra.strip())) - return sorted(set(d for d in dirs if d)) - - -def _stdlib_dirs(): - out = [] - for fn in ("stdlib", "platstdlib"): - try: - p = sysconfig.get_paths().get(fn) - if p: - out.append(os.path.realpath(p)) - except Exception: - pass - return sorted(set(out)) - - -def _under(path, root): - if not path or not root: - return False - path = os.path.realpath(path) - root = os.path.realpath(root) - return path == root or path.startswith(root.rstrip(os.sep) + os.sep) - - -def origin_of(path, task_dir=None, eval_dir=None): - """Classify where a resolved callable's source file lives. - - Order matters: a file under the task dir is task_local EVEN IF the task dir happens to sit inside - site-packages, because the point of the check is "did the extraction time itself against something - it wrote". `unknown` is a FAILING classification, not a benign one -- an anonymous path is exactly - what a scaffold dropped in cwd looks like. - """ - if not path: - return {"kind": "unresolved", "distribution": None, "path": None} - rp = os.path.realpath(path) - if task_dir and _under(rp, task_dir): - return {"kind": "task_local", "distribution": None, "path": rp} - if eval_dir and _under(rp, eval_dir): - return {"kind": "eval_local", "distribution": None, "path": rp} - for d in _stdlib_dirs(): - if _under(rp, d) and not any(_under(rp, s) for s in _site_dirs()): - return {"kind": "stdlib", "distribution": None, "path": rp} - for d in _site_dirs(): - if _under(rp, d): - return {"kind": "installed", "distribution": _distribution_for(rp, d), "path": rp} - return {"kind": "unknown", "distribution": None, "path": rp} - - -def _distribution_for(realpath, site_dir): - """Best-effort distribution name: the first path component under site-packages.""" - rel = os.path.relpath(realpath, site_dir) - top = rel.split(os.sep)[0] - top = top[:-3] if top.endswith(".py") else top - try: - import importlib.metadata as md - for dist, tops in (md.packages_distributions() or {}).items(): - if dist == top and tops: - return tops[0] - except Exception: - pass - return top or None - - -# ------------------------------------------------------------------------- INV-1 baseline identity -def validate_baseline(task_dir=None, meta=None, eval_dir=None): - """INV-1: is meta.baseline_callable a legitimate speedup DENOMINATOR? - - Six checks, each independently reportable so a failure says which one broke. A `False` on any of - them means the isolated speedup this task will produce is not comparable to anything the live - server runs, and the task must be dropped (`editable:false`) rather than authored against. - """ - meta = dict(meta or {}) - checks = [] - - def add(cid, ok, detail): - checks.append({"id": cid, "ok": bool(ok), "detail": detail}) - - base_spec = (meta.get("baseline_callable") or "").strip() - tgt_spec = (meta.get("target_callable") or "").strip() - - add("B1_baseline_declared", bool(base_spec), - f"meta.baseline_callable={base_spec!r}" if base_spec else "meta.baseline_callable is missing/empty") - - add("B6_not_synthesized", meta.get("synthesized") is not True, - "meta.synthesized is true -- the oracle was fabricated, it cannot be the denominator" - if meta.get("synthesized") is True else "meta.synthesized is not true") - - base_res = resolve_spec(base_spec) if base_spec else {"ok": False, "error": "no spec"} - add("B2_baseline_resolvable", base_res.get("ok"), - base_res.get("error") or f"resolved to {base_res.get('qualname')} @ {base_res.get('file')}") - - base_origin = origin_of(base_res.get("file"), task_dir, eval_dir) if base_res.get("ok") else \ - {"kind": "unresolved", "distribution": None, "path": None} - add("B3_baseline_origin_installed", base_origin["kind"] == "installed", - f"origin={base_origin['kind']}" - + (f" dist={base_origin['distribution']}" if base_origin["distribution"] else "") - + (" -- a baseline living in the task/eval dir is the candidate's own scaffold, not the online kernel" - if base_origin["kind"] in ("task_local", "eval_local") else "") - + (" -- source file is not inside any installed distribution" if base_origin["kind"] == "unknown" else "")) - - tgt_res = resolve_spec(tgt_spec) if tgt_spec else {"ok": False, "error": "meta.target_callable missing/empty"} - add("B4_target_resolvable", tgt_res.get("ok"), - tgt_res.get("error") or f"resolved to {tgt_res.get('qualname')} @ {tgt_res.get('file')}") - - # B5: the denominator must BE the seam, or be provably reached FROM it. Identity is checked on the - # resolved code object (so two spellings of the same function pass). Otherwise we require positive - # evidence recorded at capture time -- an assertion in prose is not evidence. - same_obj = bool(base_res.get("ok") and tgt_res.get("ok") - and (base_res.get("obj") is tgt_res.get("obj") - or getattr(base_res.get("obj"), "__code__", None) - is getattr(tgt_res.get("obj"), "__code__", object()))) - ev = meta.get("baseline_capture_evidence") or {} - observed = isinstance(ev, dict) and int(ev.get("observed_calls") or 0) > 0 \ - and (ev.get("from_seam") or "").strip() == tgt_spec - add("B5_denominator_is_the_seam", same_obj or observed, - "baseline_callable IS target_callable" if same_obj else - (f"observed {ev.get('observed_calls')} live call(s) from {ev.get('from_seam')!r}" if observed else - "baseline_callable is neither the seam nor an OBSERVED callee of it " - "(set meta.baseline_capture_evidence={from_seam, observed_calls} at capture time)")) - - ok = all(c["ok"] for c in checks) - return { - "contract": "baseline_identity", "contract_version": CONTRACT_VERSION, "ok": ok, - "baseline_callable": base_spec, "target_callable": tgt_spec, - "baseline_origin": {k: v for k, v in base_origin.items() if k != "obj"}, - "checks": checks, - "failed": [c["id"] for c in checks if not c["ok"]], - "verdict": ("denominator is the live online kernel" if ok else - "INVALID DENOMINATOR -- any speedup measured against it is not comparable to e2e"), - } - - -# -------------------------------------------------------------------------- INV-2 binding contract -def _describe_signature(sig, spec, qualname, resolved_file=None, meta=None): - """Build a binding descriptor from an already-resolved inspect.Signature.""" - meta = dict(meta or {}) - params, required_positional, out_params = [], [], [] - accepts_varargs = accepts_varkw = False - for name, p in sig.parameters.items(): - if p.kind is inspect.Parameter.VAR_POSITIONAL: - accepts_varargs = True - if p.kind is inspect.Parameter.VAR_KEYWORD: - accepts_varkw = True - ann = "" if p.annotation is inspect.Parameter.empty else _ann_str(p.annotation) - has_def = p.default is not inspect.Parameter.empty - params.append({"name": name, "kind": p.kind.name, "has_default": has_def, "annotation": ann}) - if (not has_def and p.kind in (inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY)): - required_positional.append(name) - if name.lower() in _OUT_PARAM_NAMES: - out_params.append(name) - - ret_ann = "" if sig.return_annotation is inspect.Signature.empty else _ann_str(sig.return_annotation) - returns_none = ret_ann in ("None", "NoneType") - - # Runtime evidence from the capture step, when the extractor recorded it, beats the name heuristic. - ev = meta.get("seam_runtime_evidence") or {} - if isinstance(ev, dict) and ev.get("inplace_params") is not None: - out_params = list(ev.get("inplace_params") or []) - evidence = "observed_inplace" - elif isinstance(ev, dict) and ev.get("returns_none") is not None: - returns_none = bool(ev.get("returns_none")) - evidence = "observed_return" - else: - evidence = "name_convention" if out_params else "none" - - # Inputs the callable reads that do NOT arrive through its parameters (forward-context, layer - # registries, module globals). A non-empty list means the seam is NOT a pure function of its - # arguments, so an out-of-tree rewrite cannot be given the same inputs -- see check_binding. - hidden_known = isinstance(ev, dict) and isinstance(ev.get("hidden_context"), list) - hidden_ctx = list(ev["hidden_context"]) if hidden_known else [] - - return { - "contract": "binding", "contract_version": CONTRACT_VERSION, "ok": True, - "seam": spec, "qualname": qualname, "resolved_file": resolved_file, - "params": params, "required_positional": required_positional, - "accepts_varargs": accepts_varargs, "accepts_varkw": accepts_varkw, - "arity_required": len(required_positional), "arity_total": len(params), - "returns_annotation": ret_ann, "returns_none": returns_none, - "out_params": out_params, "out_params_evidence": evidence, - "hidden_context": hidden_ctx, - "hidden_context_evidence": "declared" if hidden_known else "unknown", - "signature": f"{qualname}{sig}", - } - - -def describe_binding(spec, meta=None): - """Capture the LIVE callable's call contract by reflection. This descriptor -- not an agent's - recollection of it -- is what an authored entry has to satisfy.""" - meta = dict(meta or {}) - res = resolve_spec(spec) - if not res.get("ok"): - return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": spec, "error": res.get("error")} - obj = res["obj"] - if not callable(obj): - return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": spec, "error": "seam resolves to a non-callable"} - try: - sig = inspect.signature(obj) - except Exception as e: # noqa: BLE001 - return {"contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": spec, "error": f"signature unavailable: {e!r}"} - return _describe_signature(sig, spec, res.get("qualname"), res.get("file"), meta) - - -def _ann_str(ann): - if ann is None: - return "None" - if isinstance(ann, str): - return ann - return getattr(ann, "__name__", None) or str(ann) - - -def _signature_from_descriptor(desc): - """Reconstruct an inspect.Signature for call-compatibility checks.""" - params = [] - for p in desc.get("params") or []: - kind = getattr(inspect.Parameter, p["kind"]) - default = None if p.get("has_default") else inspect.Parameter.empty - params.append(inspect.Parameter(p["name"], kind, default=default)) - return inspect.Signature(params) - - -def _representative_calls(desc): - """Calls spanning the live signature's positional/keyword and optional surfaces.""" - calls = [] - for include_optional in (False, True): - for keyword_pok in (False, True): - args, kwargs = [], {} - for p in desc.get("params") or []: - kind, name = p["kind"], p["name"] - required = not p.get("has_default") and kind not in ("VAR_POSITIONAL", "VAR_KEYWORD") - if kind == "VAR_POSITIONAL": - if include_optional: - args.append(object()) - continue - if kind == "VAR_KEYWORD": - if include_optional: - kwargs["__geak_extra_kwarg__"] = object() - continue - if not required and not include_optional: - continue - value = object() - if kind == "POSITIONAL_ONLY": - args.append(value) - elif kind == "POSITIONAL_OR_KEYWORD": - if keyword_pok: - kwargs[name] = value - else: - args.append(value) - elif kind == "KEYWORD_ONLY": - kwargs[name] = value - calls.append((args, kwargs)) - return calls - - -def check_binding(descriptor, candidate): - """Can `candidate` be bound AT the seam described by `descriptor`? - - `candidate` may be a module:attr spec, an already-built descriptor, or a dict - {params, returns_none, ...}. Mismatch codes are a closed set so the orchestrator can route on them - instead of pattern-matching prose. - """ - if not descriptor or not descriptor.get("ok"): - return {"contract": "binding_check", "contract_version": CONTRACT_VERSION, "bindable": False, - "seam": (descriptor or {}).get("seam", ""), "candidate": str(candidate), - "mismatches": [{"code": "no_seam_descriptor", - "detail": (descriptor or {}).get("error", "seam was never described")}], - "codes": ["no_seam_descriptor"]} - - if isinstance(candidate, str): - cand = describe_binding(candidate) - if not cand.get("ok"): - return {"contract": "binding_check", "contract_version": CONTRACT_VERSION, "bindable": False, - "seam": descriptor.get("seam"), "candidate": candidate, - "mismatches": [{"code": "candidate_unresolvable", "detail": cand.get("error")}], - "codes": ["candidate_unresolvable"]} - else: - cand = dict(candidate or {}) - - m = [] - live_req = list(descriptor.get("required_positional") or []) - cand_req = list(cand.get("required_positional") or []) - cand_varargs = bool(cand.get("accepts_varargs")) - cand_varkw = bool(cand.get("accepts_varkw")) - - # Arity. A single opaque `args`/`kwargs`-style parameter standing in for N live tensors is the - # classic out-of-tree rewrite that can never be rebound; it shows up here as an arity mismatch. - if not cand_varargs and len(cand_req) != len(live_req): - m.append({"code": "arity_mismatch", - "detail": f"seam requires {len(live_req)} positional arg(s) {live_req}; " - f"candidate entry requires {len(cand_req)} {cand_req}"}) - - # Names. The overlay rebinds by NAME at keyword call sites, so a renamed required parameter is a - # hard break even when the arity lines up. - if not (cand_varargs and cand_varkw): - missing = [n for n in live_req if n not in [p["name"] for p in (cand.get("params") or [])]] - if missing and not cand_varkw: - m.append({"code": "param_name_mismatch", - "detail": f"seam parameter(s) {missing} absent from the candidate entry"}) - - # Return contract. A seam that writes into a caller-owned buffer and returns None cannot be - # replaced by something that allocates and returns a fresh tensor -- the caller never reads it. - live_inplace = bool(descriptor.get("out_params")) or bool(descriptor.get("returns_none")) - cand_inplace = bool(cand.get("out_params")) or bool(cand.get("returns_none")) - if live_inplace and not cand_inplace: - m.append({"code": "return_contract_mismatch", - "detail": f"seam writes in place (out_params={descriptor.get('out_params')}, " - f"returns_none={descriptor.get('returns_none')}) but the candidate returns a " - f"fresh value; the live caller would discard the result"}) - if cand_inplace and not live_inplace: - m.append({"code": "return_contract_mismatch", - "detail": "candidate writes in place but the seam's callers consume a returned value"}) - - # C6: an OPTIONAL live parameter that the candidate DROPS is still passed by existing callers, so the - # bound call raises TypeError even though the REQUIRED names+arity line up. (Comparing only the - # required-name sets missed this.) A candidate that swallows extras via **kwargs is exempt. - if not cand_varkw: - cand_names = {p["name"] for p in (cand.get("params") or [])} - live_optional = [p["name"] for p in (descriptor.get("params") or []) - if p.get("has_default") and p["kind"] in ("POSITIONAL_OR_KEYWORD", "KEYWORD_ONLY")] - dropped = [n for n in live_optional if n not in cand_names] - if dropped: - m.append({"code": "optional_param_dropped", - "detail": f"seam accepts optional param(s) {dropped} that live callers may pass; the " - f"candidate entry omits them and would raise TypeError when they are"}) - - # Check actual Python binding behavior over the live signature's minimal/maximal and - # positional/keyword call surfaces. This catches positional->keyword-only changes, reordered - # positional parameters when keyword calls are also legal, and variadic incompatibilities without - # incorrectly rejecting a live POSITIONAL_ONLY parameter against an identical candidate. - try: - cand_sig = _signature_from_descriptor(cand) - for call_args, call_kwargs in _representative_calls(descriptor): - try: - cand_sig.bind(*call_args, **call_kwargs) - except TypeError as e: - if not any(x["code"] in ("arity_mismatch", "param_name_mismatch", - "optional_param_dropped", "param_kind_mismatch") for x in m): - m.append({"code": "param_kind_mismatch", - "detail": f"candidate rejects a call accepted by the live seam: {e}"}) - break - except (TypeError, ValueError, KeyError) as e: - m.append({"code": "param_kind_mismatch", - "detail": f"candidate signature descriptor is invalid: {e}"}) - - # Hidden context. If the seam reads state that never crosses the parameter boundary, an authored - # replacement cannot be handed the same inputs -- authoring it is wasted budget regardless of how - # fast it is. This is the check that costs seconds and saves a kernel budget. - if descriptor.get("hidden_context_evidence") != "declared": - m.append({"code": "hidden_context_inputs", - "detail": "seam_runtime_evidence.hidden_context is missing; purity is unknown, so the " - "seam cannot be admitted under the fail-closed binding contract"}) - elif descriptor.get("hidden_context"): - m.append({"code": "hidden_context_inputs", - "detail": f"seam reads non-parameter inputs {descriptor['hidden_context']}; it is not a " - f"pure function of its arguments -- rebind at an inner seam that is"}) - - return {"contract": "binding_check", "contract_version": CONTRACT_VERSION, - "bindable": not m, "seam": descriptor.get("seam"), - "candidate": cand.get("seam") or cand.get("qualname") or "", - "mismatches": m, "codes": [x["code"] for x in m]} - - -def render_entry(descriptor, entry_name="entry"): - """Emit the unittest entry contract FROM the live signature, so an authored kernel is written - against the real call shape instead of one the extractor invented. Generated, never hand-written: - that is what makes 'signature mismatch' structurally impossible rather than merely detected.""" - if not descriptor or not descriptor.get("ok"): - raise ValueError("cannot render an entry from a failed binding descriptor") - parts = [] - for p in descriptor.get("params") or []: - k, n = p["kind"], p["name"] - if k == "VAR_POSITIONAL": - parts.append(f"*{n}") - elif k == "VAR_KEYWORD": - parts.append(f"**{n}") - elif p["has_default"]: - parts.append(f"{n}=None") - else: - parts.append(n) - posonly = [p["name"] for p in (descriptor.get("params") or []) if p["kind"] == "POSITIONAL_ONLY"] - if posonly: - last = posonly[-1] - i = parts.index(last) if last in parts else parts.index(f"{last}=None") - parts.insert(i + 1, "/") - has_star = any(p["kind"] == "VAR_POSITIONAL" for p in descriptor.get("params") or []) - kwonly = [p["name"] for p in (descriptor.get("params") or []) if p["kind"] == "KEYWORD_ONLY"] - if kwonly and not has_star: - i = min(parts.index(n) if n in parts else parts.index(f"{n}=None") for n in kwonly) - parts.insert(i, "*") - sig = ", ".join(parts) - out = descriptor.get("out_params") or [] - ret = (" # The live seam writes into %s and returns None -- write there, return None.\n" - " raise NotImplementedError\n" % (out,)) if (out or descriptor.get("returns_none")) else \ - " # The live seam returns its result -- return it.\n raise NotImplementedError\n" - return ( - '"""AUTO-GENERATED from the live seam by seam_contract.render_entry -- DO NOT EDIT BY HAND.\n' - f'Seam: {descriptor.get("seam")}\n' - f'Live signature: {descriptor.get("signature")}\n' - 'Any authored kernel MUST implement exactly this contract; the overlay rebinds this name.\n' - '"""\n\n' - f"CONTRACT_SEAM = {descriptor.get('seam')!r}\n" - f"CONTRACT_VERSION = {CONTRACT_VERSION}\n\n\n" - f"def {entry_name}({sig}):\n{ret}") - - -# ------------------------------------------------------------------------------------------- CLI -def _load_meta(task_dir): - p = os.path.join(task_dir, "meta.json") - if not os.path.exists(p): - return {}, f"no meta.json in {task_dir}" - try: - with open(p) as fh: - return json.load(fh), None - except Exception as e: # noqa: BLE001 - return {}, f"meta.json unreadable: {e!r}" - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("--task-dir", default="", help="op task dir containing meta.json") - ap.add_argument("--eval-dir", default="", help="run EVAL_DIR (also disqualified as a baseline origin)") - # C10: the binding DESCRIPTOR must be built from the deployment TARGET, never the baseline. Two - # explicit flags make the intent unambiguous; --spec stays as a DEPRECATED alias for --target-spec so - # existing callers keep working (it used to feed the descriptor and, when a role piped the - # baseline_callable through it, silently described the denominator instead of the deployment seam). - ap.add_argument("--target-spec", default="", help="module:attr of the DEPLOYMENT seam to describe (the binding target)") - ap.add_argument("--baseline-spec", default="", help="module:attr of the baseline/denominator (overrides meta.baseline_callable for validation)") - ap.add_argument("--spec", default="", help="DEPRECATED alias for --target-spec") - ap.add_argument("--mode", default="both", choices=["baseline", "binding", "both", "entry"]) - ap.add_argument("--candidate", default="", help="module:attr of the authored entry, for --mode binding") - ap.add_argument("--entry-name", default="entry") - ap.add_argument("--out", default="", help="write the rendered entry contract here (--mode entry)") - ap.add_argument("--json", action="store_true", help="print the verdict as JSON (default)") - ap.add_argument("--site-root", action="append", default=[], - help="extra install root to count as 'installed' (overlay/editable/vendored); repeatable") - args = ap.parse_args(argv) - - if args.site_root: - prev = os.environ.get("GEAK_SEAM_SITE_DIRS", "") - os.environ["GEAK_SEAM_SITE_DIRS"] = os.pathsep.join([p for p in ([prev] + args.site_root) if p]) - - meta, meta_err = ({}, None) - if args.task_dir: - meta, meta_err = _load_meta(args.task_dir) - - # Explicit CLI specs override the corresponding meta fields for this validation run. - if args.baseline_spec: - meta = dict(meta) - meta["baseline_callable"] = args.baseline_spec - if args.target_spec or args.spec: - meta = dict(meta) - meta["target_callable"] = args.target_spec or args.spec - - result = {"contract_version": CONTRACT_VERSION, "task_dir": args.task_dir or None} - if meta_err: - result["meta_error"] = meta_err - if args.spec and not args.target_spec: - sys.stderr.write("seam_contract: --spec is deprecated; use --target-spec for the deployment seam " - "(--spec builds the binding DESCRIPTOR, never the baseline)\n") - - if args.mode in ("baseline", "both"): - result["baseline_validation"] = validate_baseline(args.task_dir or None, meta, args.eval_dir or None) - - if args.mode in ("binding", "both", "entry"): - # C10: the descriptor is the DEPLOYMENT target's contract. Precedence: --target-spec, then the - # deprecated --spec alias, then meta.target_callable. The baseline is NEVER what gets described. - spec = args.target_spec or args.spec or (meta.get("target_callable") or "") - desc = describe_binding(spec, meta) if spec else { - "contract": "binding", "contract_version": CONTRACT_VERSION, "ok": False, - "seam": "", "error": "no target_callable / --target-spec given"} - result["binding_descriptor"] = desc - if args.candidate: - result["binding_check"] = check_binding(desc, args.candidate) - elif desc.get("ok") and args.mode in ("binding", "both"): - # Validate the ACTUAL signature emitted by render_entry(), not the descriptor against itself. - # This proves the generated immutable entry contract accepts every live call shape. - try: - ns = {} - exec(render_entry(desc, args.entry_name), ns) # generated source; no untrusted input runs - generated = ns[args.entry_name] - generated_meta = {"seam_runtime_evidence": { - "inplace_params": list(desc.get("out_params") or []), - "returns_none": bool(desc.get("returns_none")), - "hidden_context": [], - }} - cand_desc = _describe_signature(inspect.signature(generated), - "", - args.entry_name, None, generated_meta) - result["binding_check"] = check_binding(desc, cand_desc) - result["binding_check"]["candidate"] = "" - except Exception as e: # noqa: BLE001 - result["binding_check"] = { - "contract": "binding_check", "contract_version": CONTRACT_VERSION, - "bindable": False, "seam": desc.get("seam"), "candidate": "", - "mismatches": [{"code": "candidate_unresolvable", - "detail": f"generated entry could not be inspected: {e!r}"}], - "codes": ["candidate_unresolvable"], - } - if args.mode == "entry": - if not desc.get("ok"): - result["entry_error"] = desc.get("error") - else: - src = render_entry(desc, args.entry_name) - result["entry_contract"] = src - if args.out: - with open(args.out, "w") as fh: - fh.write(src) - result["entry_path"] = os.path.realpath(args.out) - - ok = True - if "baseline_validation" in result: - ok = ok and bool(result["baseline_validation"].get("ok")) - if args.mode in ("binding", "entry") or ("binding_check" in result): - ok = ok and bool(result.get("binding_descriptor", {}).get("ok")) - if "binding_check" in result: - ok = ok and bool(result["binding_check"].get("bindable")) - result["ok"] = ok - - print(json.dumps(result, indent=2, default=str)) - return 0 if ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/e2e_workflow/scripts/seam_trace.py b/e2e_workflow/scripts/seam_trace.py new file mode 100644 index 000000000..ca17b3fc3 --- /dev/null +++ b/e2e_workflow/scripts/seam_trace.py @@ -0,0 +1,186 @@ +"""Install safe profiler markers on candidate live call seams.""" + +import atexit +import functools +import importlib +import inspect +import os +import sys +import threading +import types + + +MARKER_PREFIX = "GEAK_TARGET::" +INSTALL_PREFIX = "GEAK_INSTALLED::" +_INSTALLED = {} +_TLS = threading.local() +_PROFILE = { + "lock": threading.Lock(), + "active": False, + "done": False, + "owner": None, + "profiler": None, + "active_calls": 0, + "root_calls": 0, + "out": "", + "trace_index": 0, + "atexit_registered": False, +} + + +def _rank(): + for key in ("RANK", "LOCAL_RANK", "TP_RANK", "SLURM_PROCID"): + value = os.environ.get(key) + if value is not None: + return str(value) + return "unknown" + + +def _trace_path(trace_index): + template = os.environ.get("GEAK_SELECTION_TRACE", "").strip() + if not template: + return "" + pid = os.getpid() + if "{pid}" in template or "{rank}" in template: + path = template.format(pid=pid, rank=_rank()) + else: + root, ext = os.path.splitext(template) + path = f"{root}.pid-{pid}.rank-{_rank()}{ext or '.json'}" + if os.environ.get("GEAK_SELECTION_TRACE_UNIQUE", "1") == "0": + return template + root, ext = os.path.splitext(path) + return f"{root}.call-{trace_index}{ext or '.json'}" + + +def _record_install_markers(): + """Put proof of successful installation in every process-local trace.""" + import torch + for target in sorted(_INSTALLED): + with torch.profiler.record_function(INSTALL_PREFIX + target): + pass + + +def _start_profile(): + """Start the next bounded process-local root-call profile.""" + with _PROFILE["lock"]: + if _PROFILE["active"] or _PROFILE["done"]: + return False + budget = max(1, int(os.environ.get("GEAK_SELECTION_PROFILE_CALLS", "32"))) + if _PROFILE["trace_index"] >= budget: + _PROFILE["done"] = True + return False + out = _trace_path(_PROFILE["trace_index"] + 1) + if not out: + return False + ident = threading.get_ident() + try: + import torch + activities = [torch.profiler.ProfilerActivity.CPU] + if hasattr(torch.profiler.ProfilerActivity, "CUDA"): + activities.append(torch.profiler.ProfilerActivity.CUDA) + profiler = torch.profiler.profile(activities=activities) + profiler.__enter__() + _PROFILE.update(active=True, owner=ident, profiler=profiler, out=out, + active_calls=0, root_calls=0) + _record_install_markers() + if not _PROFILE["atexit_registered"]: + atexit.register(_finish_profile) + _PROFILE["atexit_registered"] = True + return True + except Exception as exc: + _PROFILE["done"] = True + sys.stderr.write(f"[seam_trace] profiler start failed: {exc!r}\n") + return False + + +def _finish_profile(): + """Stop and atomically export this process's trace once.""" + with _PROFILE["lock"]: + if not _PROFILE["active"] or _PROFILE["done"]: + return + profiler = _PROFILE.get("profiler") + out = _PROFILE.get("out") + _PROFILE["trace_index"] += 1 + budget = max(1, int(os.environ.get("GEAK_SELECTION_PROFILE_CALLS", "32"))) + _PROFILE.update(active=False, done=_PROFILE["trace_index"] >= budget, + owner=None, profiler=None) + try: + profiler.__exit__(None, None, None) + os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True) + root, ext = os.path.splitext(out) + tmp = f"{root}.tmp-{os.getpid()}{ext}" + profiler.export_chrome_trace(tmp) + os.replace(tmp, out) + sys.stderr.write(f"[seam_trace] selection trace -> {out}\n") + except Exception as exc: + sys.stderr.write(f"[seam_trace] profiler export failed: {exc!r}\n") + + +def _wrappable(value): + """Only replace callables whose identity/protocol a Python wrapper preserves.""" + if isinstance(value, (types.FunctionType, types.MethodType, functools.partial)): + return True + if isinstance(value, (types.BuiltinFunctionType, types.BuiltinMethodType)): + return False + if any(hasattr(value, attr) + for attr in ("fn", "cache", "warmup", "run", "__torch_dispatch__")): + return False + return callable(value) and type(value).__module__ != "builtins" + + +def _enter_call(): + _start_profile() + depth = getattr(_TLS, "depth", 0) + _TLS.depth = depth + 1 + with _PROFILE["lock"]: + if _PROFILE["active"]: + _PROFILE["active_calls"] += 1 + return depth == 0 + + +def _leave_call(root_call): + _TLS.depth = max(0, getattr(_TLS, "depth", 1) - 1) + finish = False + with _PROFILE["lock"]: + if _PROFILE["active"]: + _PROFILE["active_calls"] = max(0, _PROFILE["active_calls"] - 1) + if root_call: + _PROFILE["root_calls"] += 1 + finish = root_call and _PROFILE["active_calls"] == 0 + if finish: + _finish_profile() + + +def install(target): + """Wrap one module:attr with a record_function marker; idempotent per target.""" + if target in _INSTALLED: + return + module_name, attr = target.split(":", 1) + module = importlib.import_module(module_name) + original = getattr(module, attr) + if not _wrappable(original): + raise RuntimeError(f"cannot safely mark non-Python callable {target}") + + @functools.wraps(original) + def marked(*args, **kwargs): + root_call = _enter_call() + try: + import torch + record_function = torch.profiler.record_function + except Exception: + record_function = None + try: + if record_function is None: + return original(*args, **kwargs) + with record_function(MARKER_PREFIX + target): + return original(*args, **kwargs) + finally: + _leave_call(root_call) + + try: + marked.__signature__ = inspect.signature(original) + except (TypeError, ValueError): + pass + _INSTALLED[target] = original + setattr(module, attr, marked) + sys.stderr.write(f"[seam_trace] marked {target}\n") diff --git a/e2e_workflow/scripts/tests/test_capture_shapes.py b/e2e_workflow/scripts/tests/test_capture_shapes.py index f46f7fc9b..c0ef7e4ea 100644 --- a/e2e_workflow/scripts/tests/test_capture_shapes.py +++ b/e2e_workflow/scripts/tests/test_capture_shapes.py @@ -576,7 +576,7 @@ def test_oracle_file_is_written_and_hashed(self): io_path = os.path.join(self.out_dir, "reference_io.pt") self.assertTrue(os.path.exists(io_path)) payload, path = self.torch.saved[-1] - self.assertEqual(path, io_path) + self.assertTrue(path.startswith(io_path + ".tmp-")) self.assertEqual(payload["target"], "fake_serving_layer:op") self.assertEqual([r["regime"] for r in payload["records"]], ["decode", "prefill"]) sha = self._meta()["reference_io_sha256"] @@ -857,6 +857,15 @@ def test_install_from_env_reads_target_out_and_max(self): self.assertEqual(cs._STATE["out_dir"], self.out_dir) self.assertIsNot(mod.op, cs._STATE["orig"]) + def test_selection_capture_uses_a_process_local_artifact_directory(self): + self._target_module() + with _env(GEAK_SELECTION_TRACE=os.path.join(self.out_dir, "selection.json")): + with _stderr(): + cs.install("fake_serving_layer:op", self.out_dir) + expected_prefix = os.path.join( + self.out_dir, f"capture.pid-{os.getpid()}.rank-") + self.assertTrue(cs._STATE["out_dir"].startswith(expected_prefix)) + # --------------------------------------------------------------------------- # # import-time self-install (the overlay PYTHONPATH / sitecustomize entry point) diff --git a/e2e_workflow/scripts/tests/test_kernel_selection.py b/e2e_workflow/scripts/tests/test_kernel_selection.py new file mode 100644 index 000000000..f47c77982 --- /dev/null +++ b/e2e_workflow/scripts/tests/test_kernel_selection.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Regression tests for machine-verified live kernel selection.""" + +import importlib.util +import json +import os +import tempfile +import unittest + + +SCRIPTS = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SPEC = importlib.util.spec_from_file_location( + "kernel_selection", os.path.join(SCRIPTS, "kernel_selection.py")) +ks = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ks) + +TARGET = ( + "vllm.v1.attention.ops.chunked_prefill_paged_decode:" + "chunked_prefill_paged_decode" +) +KERNEL = "kernel_paged_attention_2d" + + +def trace(kernel=KERNEL, under_target=True): + marker_start = 100 + kernel_start = 120 if under_target else 250 + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": marker_start, "dur": 100}, + {"cat": "kernel", "name": f"void vllm::{kernel}(int)", + "ph": "X", "ts": kernel_start, "dur": 25, "args": {"External id": 7}}, + ] + if under_target: + events.insert(1, {"cat": "cpu_op", "name": "launch", "ph": "X", + "pid": 1, "tid": 2, "ts": 110, "dur": 5, + "args": {"External id": 7}}) + return events + + +class TestCallableSpec(unittest.TestCase): + def test_only_exact_machine_specs_are_accepted(self): + self.assertTrue(ks.valid_callable_spec(TARGET)) + self.assertFalse(ks.valid_callable_spec(TARGET + " -> inner kernel")) + self.assertFalse(ks.valid_callable_spec("vllm/path.py:launcher")) + + +class TestKernelMatching(unittest.TestCase): + def test_demangled_kernel_matches_profile_identity(self): + self.assertTrue(ks.kernel_matches( + KERNEL, "void vllm::kernel_paged_attention_2d(int)")) + self.assertFalse(ks.kernel_matches(KERNEL, "unrelated_attention_kernel")) + + +class TestSelectionVerdict(unittest.TestCase): + def meta(self, calls=7, target=TARGET): + module, attr = target.split(":", 1) + return {"module": module, "attr": attr, "total_calls_observed": calls} + + def test_0720_live_inner_launcher_is_selection_success(self): + verdict = ks.verify(TARGET, KERNEL, self.meta(), trace()) + self.assertTrue(verdict["ok"]) + self.assertEqual(verdict["matched_kernel_calls"], 1) + self.assertEqual(verdict["failed"], []) + + def test_kernel_seen_elsewhere_is_not_enough(self): + verdict = ks.verify(TARGET, KERNEL, self.meta(), trace(under_target=False)) + self.assertFalse(verdict["ok"]) + self.assertIn("device_kernel_not_under_target", verdict["failed"]) + + def test_async_kernel_after_marker_is_linked_by_external_id(self): + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 40}, + {"cat": "cpu_op", "name": "launch", "ph": "X", "pid": 1, "tid": 2, + "ts": 110, "dur": 5, + "args": {"External id": 42}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "ts": 300, "dur": 20, + "args": {"External id": 42}}, + ] + verdict = ks.verify(TARGET, KERNEL, self.meta(), events) + self.assertTrue(verdict["ok"], verdict) + self.assertEqual(verdict["correlated_external_ids"], 1) + + def test_0802_outer_wrapper_fails_when_0720_launcher_is_marked(self): + outer = "vllm.v1.attention.layer:unified_attention_with_output" + inner = TARGET + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + outer, + "ph": "X", "pid": 1, "tid": 2, "ts": 80, "dur": 1}, + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + inner, + "ph": "X", "pid": 1, "tid": 2, "ts": 82, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + outer, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 100}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + inner, + "ph": "X", "pid": 1, "tid": 2, "ts": 120, "dur": 50}, + {"cat": "cpu_op", "name": "launch", "ph": "X", "pid": 1, "tid": 2, + "ts": 130, "dur": 5, "args": {"External id": 9}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "ts": 300, "dur": 20, + "args": {"External id": 9}}, + ] + outer_meta = {"module": "vllm.attention", "attr": "outer", "total_calls_observed": 2} + outer_verdict = ks.verify(outer, KERNEL, outer_meta, events, [outer, inner]) + self.assertFalse(outer_verdict["ok"]) + self.assertIn("deeper_live_candidate_exists", outer_verdict["failed"]) + self.assertEqual(outer_verdict["deeper_live_candidates"], [inner]) + + inner_verdict = ks.verify(inner, KERNEL, self.meta(), events, [outer, inner]) + self.assertTrue(inner_verdict["ok"], inner_verdict) + self.assertTrue(inner_verdict["deepest_verified"]) + + def test_every_declared_probe_candidate_must_have_an_installed_marker(self): + missing = "vllm.attention:missing_candidate" + verdict = ks.verify(TARGET, KERNEL, self.meta(), trace(), [TARGET, missing]) + self.assertFalse(verdict["ok"]) + self.assertIn("candidate_marker_not_installed", verdict["failed"]) + self.assertEqual(verdict["missing_candidate_markers"], [missing]) + + def test_installed_but_inactive_alternative_branch_does_not_fail(self): + alternative = "vllm.attention:prefill_only" + events = trace() + events.insert(1, { + "cat": "cpu_op", "name": ks.INSTALL_PREFIX + alternative, + "ph": "X", "pid": 1, "tid": 2, "ts": 92, "dur": 1, + }) + verdict = ks.verify( + TARGET, KERNEL, self.meta(), events, [TARGET, alternative]) + self.assertTrue(verdict["ok"], verdict) + self.assertEqual( + sorted(verdict["candidate_targets_tested"]), + sorted([TARGET, alternative]), + ) + + def test_capture_of_a_different_callable_is_rejected(self): + wrong = "vllm.model_executor.layers.attention.attention:outer_wrapper" + verdict = ks.verify(TARGET, KERNEL, self.meta(target=wrong), trace()) + self.assertFalse(verdict["ok"]) + self.assertIn("capture_target_mismatch", verdict["failed"]) + + def test_zero_live_calls_is_rejected(self): + verdict = ks.verify(TARGET, KERNEL, self.meta(calls=0), trace()) + self.assertFalse(verdict["ok"]) + self.assertIn("target_not_observed", verdict["failed"]) + + def test_cli_writes_the_same_machine_verdict(self): + with tempfile.TemporaryDirectory() as root: + meta_path = os.path.join(root, "meta.json") + trace_path = os.path.join(root, "trace.json") + out_path = os.path.join(root, "selection.json") + with open(meta_path, "w") as fh: + json.dump(self.meta(), fh) + with open(trace_path, "w") as fh: + json.dump({"traceEvents": trace()}, fh) + rc = ks.main([ + "--target", TARGET, + "--device-kernel", KERNEL, + "--capture-meta", meta_path, + "--torch-trace", trace_path, + "--out", out_path, + ]) + self.assertEqual(rc, 0) + with open(out_path) as fh: + self.assertTrue(json.load(fh)["ok"]) + + def test_cli_fails_closed_when_capture_pid_has_no_trace(self): + with tempfile.TemporaryDirectory() as root: + meta_path = os.path.join(root, "capture.pid-111.rank-0", "meta.json") + os.makedirs(os.path.dirname(meta_path)) + trace_path = os.path.join(root, "selection.pid-999.rank-0.json") + out_path = os.path.join(root, "selection.json") + meta = self.meta() + meta["process_id"] = 111 + with open(meta_path, "w") as fh: + json.dump(meta, fh) + with open(trace_path, "w") as fh: + json.dump({"traceEvents": trace()}, fh) + rc = ks.main([ + "--target", TARGET, + "--device-kernel", KERNEL, + "--capture-meta", meta_path, + "--torch-trace", trace_path, + "--out", out_path, + ]) + self.assertEqual(rc, 1) + with open(out_path) as fh: + verdict = json.load(fh) + self.assertFalse(verdict["ok"]) + self.assertIn("capture_process_trace_missing", verdict["failed"]) + self.assertEqual(verdict["trace_file"], "") + + def test_cli_merges_calls_before_deciding_deepest_candidate(self): + mid = "vllm.attention:mid" + inner = TARGET + with tempfile.TemporaryDirectory() as root: + meta_path = os.path.join(root, "capture.pid-111.rank-0", "meta.json") + os.makedirs(os.path.dirname(meta_path)) + meta = self.meta(target=mid) + meta["process_id"] = 111 + with open(meta_path, "w") as fh: + json.dump(meta, fh) + + common_installs = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + mid, + "ph": "X", "pid": 1, "tid": 2, "ts": 80, "dur": 1}, + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + inner, + "ph": "X", "pid": 1, "tid": 2, "ts": 82, "dur": 1}, + ] + call_one = common_installs + [ + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + mid, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 50}, + {"cat": "cpu_op", "name": "launch", "ph": "X", + "pid": 1, "tid": 2, "ts": 110, "dur": 5, + "args": {"External id": 7}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "ts": 180, "dur": 10, + "args": {"External id": 7}}, + ] + call_two = common_installs + [ + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + mid, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 80}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + inner, + "ph": "X", "pid": 1, "tid": 2, "ts": 120, "dur": 40}, + {"cat": "cpu_op", "name": "launch", "ph": "X", + "pid": 1, "tid": 2, "ts": 130, "dur": 5, + "args": {"External id": 9}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "ts": 200, "dur": 10, + "args": {"External id": 9}}, + ] + trace_paths = [] + for index, events in enumerate((call_one, call_two), 1): + path = os.path.join( + root, f"selection.pid-111.rank-0.call-{index}.json") + with open(path, "w") as fh: + json.dump({"traceEvents": events}, fh) + trace_paths.append(path) + out_path = os.path.join(root, "selection.json") + rc = ks.main([ + "--target", mid, + "--device-kernel", KERNEL, + "--capture-meta", meta_path, + "--torch-trace", *trace_paths, + "--candidate-target", mid, + "--candidate-target", inner, + "--out", out_path, + ]) + self.assertEqual(rc, 1) + with open(out_path) as fh: + verdict = json.load(fh) + self.assertIn("deeper_live_candidate_exists", verdict["failed"]) + self.assertEqual(verdict["deeper_live_candidates"], [inner]) + + def test_cli_requires_deepest_selection_on_every_capture_process(self): + mid = "vllm.attention:mid" + inner = TARGET + + def process_events(with_inner): + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + mid, + "ph": "X", "pid": 1, "tid": 2, "ts": 80, "dur": 1}, + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + inner, + "ph": "X", "pid": 1, "tid": 2, "ts": 82, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + mid, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 80}, + ] + if with_inner: + events.append({ + "cat": "cpu_op", "name": ks.MARKER_PREFIX + inner, + "ph": "X", "pid": 1, "tid": 2, "ts": 120, "dur": 40, + }) + events.extend([ + {"cat": "cpu_op", "name": "launch", "ph": "X", + "pid": 1, "tid": 2, "ts": 130, "dur": 5, + "args": {"External id": 9}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "ts": 200, "dur": 10, + "args": {"External id": 9}}, + ]) + return events + + with tempfile.TemporaryDirectory() as root: + meta_paths, trace_paths = [], [] + for pid, with_inner in ((111, False), (222, True)): + meta_path = os.path.join( + root, f"capture.pid-{pid}.rank-0", "meta.json") + os.makedirs(os.path.dirname(meta_path)) + meta = self.meta(target=mid) + meta["process_id"] = pid + with open(meta_path, "w") as fh: + json.dump(meta, fh) + meta_paths.append(meta_path) + trace_path = os.path.join( + root, f"selection.pid-{pid}.rank-0.call-1.json") + with open(trace_path, "w") as fh: + json.dump({"traceEvents": process_events(with_inner)}, fh) + trace_paths.append(trace_path) + out_path = os.path.join(root, "selection.json") + rc = ks.main([ + "--target", mid, + "--device-kernel", KERNEL, + "--capture-meta", *meta_paths, + "--torch-trace", *trace_paths, + "--candidate-target", mid, + "--candidate-target", inner, + "--out", out_path, + ]) + self.assertEqual(rc, 1) + with open(out_path) as fh: + verdict = json.load(fh) + self.assertFalse(verdict["ok"]) + self.assertEqual(verdict["deeper_live_candidates"], [inner]) + self.assertEqual( + sorted(verdict["live_candidate_targets"]), sorted([mid, inner])) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e_workflow/scripts/tests/test_op_bench.py b/e2e_workflow/scripts/tests/test_op_bench.py index b03ab5a22..0ae442bcb 100644 --- a/e2e_workflow/scripts/tests/test_op_bench.py +++ b/e2e_workflow/scripts/tests/test_op_bench.py @@ -33,7 +33,6 @@ tearDown so the other test modules in this directory keep seeing a torch-free image. """ import contextlib -import hashlib import importlib.util import io import json @@ -174,9 +173,6 @@ def float(self): def contiguous(self): return self._like() - def clone(self): - return self._like() - def reshape(self, *shape): if len(shape) == 1 and isinstance(shape[0], (tuple, list)): shape = tuple(shape[0]) @@ -225,10 +221,6 @@ def __getitem__(self, idx): def __setitem__(self, idx, value): self.val = value.val if isinstance(value, _T) else float(value) - def zero_(self): # in-place clear, mirroring torch's out-buffer zeroing - self.val = 0.0 - return self - # ---- reductions / elementwise def abs(self): return self._like(val=abs(self.val)) @@ -368,7 +360,6 @@ def _build(self): torch = types.ModuleType("torch") for dt in (BF16, FP16, FP32, INT8, UINT8, FP8_E4M3FNUZ, FP8_E5M2FNUZ, FP8_E4M3FN, FP8_E5M2): setattr(torch, dt.name, dt) - torch.is_tensor = lambda value: isinstance(value, _T) class _Finfo: def __init__(self, dt): @@ -397,7 +388,7 @@ def make(*shape, **kw): return _T(shape, kw.get("dtype") or FP32, val, kw.get("device") or "cpu") return make - def _load_blob(path, map_location=None, **_kwargs): + def _load_blob(path, map_location=None): stack.calls.append(("torch.load", os.path.basename(path), map_location)) if stack.loaded_blob is None: raise RuntimeError("fake torch.load has no blob registered for %s" % path) @@ -1084,18 +1075,11 @@ def _task_with_io(self, blob): self.stack.loaded_blob = blob return d - @staticmethod - def _pinned_meta(task, **extra): - with open(os.path.join(task, "reference_io.pt"), "rb") as fh: - sha = hashlib.sha256(fh.read()).hexdigest() - return {"reference_io_sha256": sha, **extra} - def test_recorded_oracle_is_preferred_over_synthesis(self): blob = {"A": _T((8, 16), FP32, 0.25), "B": _T((32, 16), FP32, 0.5), "bias": None, "output": _T((8, 32), FP32, 4.0)} d = self._task_with_io(blob) - A, B, bias, tb, ref = ob._load_or_synth_gemm( - self.stack.torch, d, self._pinned_meta(d, dtype="bf16"), "cpu", 0) + A, B, bias, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, {"dtype": "bf16"}, "cpu", 0) self.assertIs(A.dtype, BF16) self.assertIs(B.dtype, BF16) self.assertIsNone(bias) @@ -1108,8 +1092,7 @@ def test_oracle_without_a_recorded_output_is_recomputed_with_bias(self): blob = {"A": _T((8, 16), FP32, 0.5), "B": _T((32, 16), FP32, 2.0), "bias": _T((32,), FP32, 1.0), "output": None} d = self._task_with_io(blob) - A, B, bias, tb, ref = ob._load_or_synth_gemm( - self.stack.torch, d, self._pinned_meta(d), "cpu", 0) + A, B, bias, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, {}, "cpu", 0) self.assertEqual(ref.shape, (8, 32)) self.assertEqual(ref.val, 0.5 * 2.0 * 16 + 1.0) self.assertIs(bias.dtype, BF16) @@ -1117,14 +1100,14 @@ def test_oracle_without_a_recorded_output_is_recomputed_with_bias(self): def test_recomputed_oracle_honours_transpose_b_false(self): blob = {"A": _T((8, 16), FP32, 1.0), "B": _T((16, 32), FP32, 1.0)} d = self._task_with_io(blob) - _, _, _, tb, ref = ob._load_or_synth_gemm( - self.stack.torch, d, self._pinned_meta(d, transpose_b=False), "cpu", 0) + _, _, _, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, {"transpose_b": False}, + "cpu", 0) self.assertFalse(tb) self.assertEqual(ref.shape, (8, 32)) def test_unrecognised_blob_falls_back_to_synthesis(self): d = self._task_with_io(["not", "a", "dict"]) - meta = self._pinned_meta(d, a_shape=[4, 16], b_shape=[32, 16]) + meta = {"a_shape": [4, 16], "b_shape": [32, 16]} A, B, bias, tb, ref = ob._load_or_synth_gemm(self.stack.torch, d, meta, "cpu", 3) self.assertEqual((A.shape, B.shape, ref.shape), ((4, 16), (32, 16), (4, 32))) @@ -1516,166 +1499,19 @@ def test_missing_capture_is_reported_as_unavailable(self): self.assertEqual(res[0]["backend"], "current") self.assertFalse(res[0]["available"]) self.assertIsNone(res[0]["ms"]) - self.assertIn("no reference_io.pt", res[0]["note"]) + self.assertIn("needs reference_io.pt", res[0]["note"]) def test_captured_oracle_is_validated_and_backend_swaps_are_delegated(self): d = self._task_dir() io_path = os.path.join(d, "reference_io.pt") open(io_path, "w").close() - mod = types.ModuleType("fake_attn_seam") - mod.current = lambda x: x - sys.modules[mod.__name__] = mod - self.addCleanup(sys.modules.pop, mod.__name__, None) - self.stack.loaded_blob = {"records": [{ - "sig": "decode", "regime": "decode", "args": (_T((2, 4), FP32, 1.0),), - "kwargs": {}, "output": _T((2, 4), FP32, 1.0), - }]} - with open(io_path, "rb") as fh: - sha = hashlib.sha256(fh.read()).hexdigest() - meta = {"op_kind": "attn", "target_callable": "fake_attn_seam:current", - "reference_io_sha256": sha, "num_cases": 1} - res = ob.bench_attn(self._args(task=d), meta) + res = ob.bench_attn(self._args(task=d), {"op_kind": "attn"}) self.assertTrue(res[0]["available"]) self.assertTrue(res[0]["correct"]) - self.assertIsNotNone(res[0]["ms"]) - self.assertIn("fake_attn_seam:current", res[0]["note"]) - - -# --------------------------------------------------------------------------- # -# bench_captured_replay -- POSITIONAL in-place output buffer, END-TO-END (C3) -# --------------------------------------------------------------------------- # -class TestCapturedReplayPositionalBuffer(_FakeStackMixin, unittest.TestCase): - """C3 END-TO-END: TestOutParamOf covers _out_param_of at the unit level; this drives the - positional ('arg') branch through bench_captured_replay itself. An in-place seam whose output - buffer is passed POSITIONALLY (entry(query,key,value,output,kv_cache)->None, output at index 3) - must have its golden taken from rec['args'][ip_key] (op_bench.py ~437-438), the copy handed to the - callee zeroed (~453), the written buffer returned (~460-461) and the result verified. A resolved - index that lands on a NON-tensor arg must degrade to the UNVERIFIED path (correct=None), never a - false-green.""" - - RAW = b"positional-inplace-oracle-bytes" - - def setUp(self): - super().setUp() - self.seen_buffer_vals = [] - seen = self.seen_buffer_vals - self.mod = types.ModuleType("fake_pos_seam") - - def attn_inplace(query, key, value, output, kv_cache=None): - # In-place seam: record what the callee was handed (must be zeroed), then write the answer. - seen.append(output.val) - output[:] = query - return None - - def attn_noop(query, key, value, output, kv_cache=None): - return None # never writes the buffer (output here is a non-tensor) - - self.mod.attn_inplace = attn_inplace - self.mod.attn_noop = attn_noop - sys.modules["fake_pos_seam"] = self.mod - self.addCleanup(sys.modules.pop, "fake_pos_seam", None) - - def _replay_task(self, records, target): - d = self._task_dir() - with open(os.path.join(d, "reference_io.pt"), "wb") as fh: - fh.write(self.RAW) - self.stack.loaded_blob = {"records": records} - return d, {"op_kind": "attn", "target_callable": target, - "reference_io_sha256": hashlib.sha256(self.RAW).hexdigest(), - "num_cases": len(records), - "seam_runtime_evidence": {"inplace_params": ["output"]}} - - def test_positional_output_buffer_is_zeroed_verified_and_returned(self): - # output snapshot is None (in-place seam); the golden lives in the recorded arg at index 3. - buf = _T((2, 4), FP32, 1.0) # captured AFTER the original ran -> holds the golden - rec = {"sig": "decode", "regime": "decode", - "args": (_T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), buf), - "kwargs": {}, "output": None} - d, meta = self._replay_task([rec], "fake_pos_seam:attn_inplace") - res = ob.bench_captured_replay(self._args(task=d), meta) - row = self._by_backend(res)["current"] - self.assertTrue(row["available"]) - # correct:True is only reachable if the golden was pulled from rec['args'][3] (output=None), - # the callee rewrote the returned buffer, and _correct verified it. - self.assertTrue(row["correct"]) - self.assertFalse(row["raised"]) - self.assertEqual(row["ms"], 1.5) - self.assertEqual(row["max_rel_err"], 0.0) - # every replay iteration handed the callee a ZEROED buffer (~453), not the pre-filled golden. - self.assertTrue(self.seen_buffer_vals) - self.assertTrue(all(v == 0.0 for v in self.seen_buffer_vals), self.seen_buffer_vals) - - def test_positional_index_on_a_non_tensor_arg_is_unverified_not_false_green(self): - rec = {"sig": "decode", "regime": "decode", - "args": (_T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), _T((2, 4), FP32, 1.0), 42), - "kwargs": {}, "output": None} # index 3 ('output') resolves onto a NON-tensor - d, meta = self._replay_task([rec], "fake_pos_seam:attn_noop") - res = ob.bench_captured_replay(self._args(task=d), meta) - row = self._by_backend(res)["current"] - self.assertTrue(row["available"]) - self.assertIsNone(row["correct"]) # cannot certify -> NOT correct:True - self.assertFalse(row["raised"]) - self.assertIsNotNone(row["ms"]) # still timed, just unverified - self.assertIn("UNVERIFIED", row["note"]) - - -# --------------------------------------------------------------------------- # -# load_oracle_records -- num_cases record-count verification FAILURE modes (C4) -# --------------------------------------------------------------------------- # -class TestLoadOracleRecordCount(_FakeStackMixin, unittest.TestCase): - """C4: after the SHA identity check passes, load_oracle_records must also verify the declared - num_cases matches the actual record count (op_bench.py ~299-308). A missing / non-numeric / - non-positive / mismatched count is a fail-closed reject (records=[]), never a bankable replay. - A real reference_io.pt with a correct recomputed SHA is written so the count check is actually - reached -- it runs AFTER the sha check.""" - - RAW = b"reference-io-count-oracle-bytes" - - def _task(self, n_records): - d = self._task_dir() - with open(os.path.join(d, "reference_io.pt"), "wb") as fh: - fh.write(self.RAW) - recs = [{"sig": "s%d" % i, "regime": "decode", "args": (_T((2, 2), FP32, 1.0),), - "kwargs": {}, "output": _T((2, 2), FP32, 1.0)} for i in range(n_records)] - self.stack.loaded_blob = {"records": recs} - return d, hashlib.sha256(self.RAW).hexdigest() - - def _meta(self, sha, **extra): - return dict({"op_kind": "attn", "reference_io_sha256": sha}, **extra) - - def test_matching_count_passes_the_gate(self): - # positive control: proves the sha check is passed and the count gate is actually reached. - d, sha = self._task(2) - recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha, num_cases=2)) - self.assertEqual(err, "") - self.assertEqual(len(recs), 2) - - def test_missing_num_cases_is_rejected(self): - d, sha = self._task(1) - recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha)) - self.assertEqual(recs, []) - self.assertIn("num_cases is missing or invalid", err) - - def test_non_numeric_num_cases_is_rejected(self): - d, sha = self._task(1) - recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", - self._meta(sha, num_cases="not-a-number")) - self.assertEqual(recs, []) - self.assertIn("num_cases is missing or invalid", err) - - def test_nonpositive_num_cases_is_rejected(self): - d, sha = self._task(1) - recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha, num_cases=0)) - self.assertEqual(recs, []) - self.assertIn("at least one case", err) - - def test_count_mismatch_is_rejected(self): - d, sha = self._task(2) - recs, err = ob.load_oracle_records(d, self.stack.torch, "cpu", self._meta(sha, num_cases=5)) - self.assertEqual(recs, []) - self.assertIn("num_cases mismatch", err) - self.assertIn("declares 5", err) - self.assertIn("2 record", err) + self.assertIsNone(res[0]["ms"]) # op-level attention is not raced here + self.assertEqual(res[0]["artifact"], io_path) + self.assertIn("--attention-backend", res[0]["note"]) + self.assertIn("Config Tuner", res[0]["note"]) # --------------------------------------------------------------------------- # @@ -1693,7 +1529,6 @@ def fake_bench(args, m): return list(results or []) ob.bench_gemm = fake_bench ob.bench_attn = fake_bench - ob.bench_captured_replay = fake_bench argv = ["op_bench.py", "--task", d] + list(extra_argv) if out_path: argv += ["--out", out_path] @@ -1819,45 +1654,12 @@ def test_hipblaslt_winner_has_nothing_to_deploy(self): self.assertEqual(summary["isolated_speedup"], 1.0) # it is its own baseline self.assertIn("nothing to deploy", summary["deployable_note"]) - def test_measured_backend_default_is_a_bankable_flat_denominator(self): - # C1: a real, timed library baseline (hipblaslt) -> denominator is the BANKABLE flat string - # 'measured_backend_default'; the object travels under denominator_detail. - summary, _ = self._run_main( - {"op_kind": "gemm"}, - results=[self._res("hipblaslt", ms=2.0, correct=True), - self._res("aiter", ms=1.0, correct=True)]) - self.assertEqual(summary["denominator"], "measured_backend_default") - self.assertIsInstance(summary["denominator"], str) - self.assertEqual(ob.DENOM_SEVERITY[summary["denominator"]], "ok") # bankable - self.assertFalse(summary["speedup_withheld"]) - self.assertEqual(summary["isolated_speedup"], 2.0) - detail = summary["denominator_detail"] - self.assertIsInstance(detail, dict) - self.assertEqual(detail["provenance"], "measured_backend_default") - self.assertEqual(detail["severity"], "ok") - self.assertIn("spec", detail) - - def test_verified_baseline_row_banks_a_flat_verified_denominator(self): - # C1: an explicit 'baseline' row carrying a verified provenance -> the flat bankable string - # 'verified_baseline' at top level, still with the object under denominator_detail. - summary, _ = self._run_main( - {"op_kind": "gemm"}, - results=[self._res("baseline", ms=2.0, correct=True, - denominator_provenance="verified_baseline"), - self._res("current", ms=1.0, correct=True)]) - self.assertEqual(summary["denominator"], "verified_baseline") - self.assertEqual(ob.DENOM_SEVERITY[summary["denominator"]], "ok") # bankable - self.assertFalse(summary["speedup_withheld"]) - self.assertEqual(summary["isolated_speedup"], 2.0) - self.assertEqual(summary["denominator_detail"]["provenance"], "verified_baseline") - - def test_library_winner_without_a_baseline_withholds_the_speedup(self): + def test_library_winner_without_a_baseline_reports_a_neutral_speedup(self): summary, _ = self._run_main({"op_kind": "gemm"}, results=[self._res("flydsl", ms=1.0, correct=True)]) self.assertEqual(summary["winner_backend"], "flydsl") self.assertIsNone(summary["baseline_backend"]) - self.assertIsNone(summary["isolated_speedup"]) - self.assertTrue(summary["speedup_withheld"]) + self.assertEqual(summary["isolated_speedup"], 1.0) self.assertEqual(summary["winner_kind"], "none") self.assertIn("verify deployability", summary["deployable_note"]) @@ -1956,182 +1758,5 @@ def test_out_file_carries_the_task_dir_and_full_result_list(self): self.assertEqual(summary["apply_flags"], "") -# --------------------------------------------------------------------------- # -# resolve_denominator -- verdict-to-spec binding (C5) and the flat provenance (C1) -# --------------------------------------------------------------------------- # -class TestResolveDenominator(unittest.TestCase): - """A machine verdict certifies ONE spec. resolve_denominator must only bank a speedup when the - recorded verdict actually names the current baseline (and target, if it names one) -- a stale or - foreign verdict from a prior retry/task must fall through to unverified_baseline.""" - - def test_matching_verdict_upgrades_to_verified_baseline(self): - meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:fast", - "baseline_validation": {"contract": "baseline_identity", "ok": True, - "baseline_callable": "pkg.new:fast", - "target_callable": "pkg.new:fast"}} - d = ob.resolve_denominator(meta) - self.assertEqual(d["provenance"], "verified_baseline") - self.assertEqual(d["severity"], "ok") - - def test_verdict_for_a_different_callable_does_not_verify(self): - # ok:true, but the verdict certifies pkg.OLD:other -- not this baseline. Must NOT bank. - meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:fast", - "baseline_validation": {"contract": "baseline_identity", "ok": True, - "baseline_callable": "pkg.OLD:other", - "target_callable": "pkg.OLD:other"}} - d = ob.resolve_denominator(meta) - self.assertEqual(d["provenance"], "unverified_baseline") - self.assertEqual(d["severity"], "unverified") - self.assertIn("stale or foreign", d["why"]) - - def test_verdict_target_mismatch_alone_blocks_verification(self): - meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:v2", - "baseline_validation": {"contract": "baseline_identity", "ok": True, - "baseline_callable": "pkg.new:fast", - "target_callable": "pkg.new:OTHER"}} - d = ob.resolve_denominator(meta) - self.assertEqual(d["provenance"], "unverified_baseline") - - def test_verdict_without_a_target_is_not_identity_bound(self): - meta = {"baseline_callable": "pkg.new:fast", "target_callable": "pkg.new:fast", - "baseline_validation": {"contract": "baseline_identity", "ok": True, - "baseline_callable": "pkg.new:fast"}} - self.assertEqual(ob.resolve_denominator(meta)["provenance"], "unverified_baseline") - - -# --------------------------------------------------------------------------- # -# _out_param_of -- the in-place output buffer, kwarg OR positional (C3) -# --------------------------------------------------------------------------- # -class TestOutParamOf(unittest.TestCase): - """An in-place seam writes into a caller-supplied buffer. The extractor records its NAME; the - driver must locate it whether callers pass it as a kwarg or POSITIONALLY -- otherwise the real - target path (e.g. entry(query,key,value,output,kv_cache)->None) can never be verified.""" - - def setUp(self): - self.mod = types.ModuleType("fake_inplace_seam") - - def entry(query, key, value, output, kv_cache=None): - return None - self.mod.entry = entry - sys.modules["fake_inplace_seam"] = self.mod - self.addCleanup(sys.modules.pop, "fake_inplace_seam", None) - self.meta = {"target_callable": "fake_inplace_seam:entry", - "seam_runtime_evidence": {"inplace_params": ["output"]}} - - def test_kwarg_output_is_found_as_a_kwarg(self): - rec = {"args": (1, 2, 3), "kwargs": {"output": object()}} - self.assertEqual(ob._out_param_of(rec, self.meta), ("kwarg", "output")) - - def test_positional_output_is_resolved_to_its_index(self): - rec = {"args": (1, 2, 3, object()), "kwargs": {}} # output is the 4th positional - self.assertEqual(ob._out_param_of(rec, self.meta), ("arg", 3)) - - def test_declared_but_absent_buffer_refuses_rather_than_guessing(self): - rec = {"args": (1, 2, 3), "kwargs": {}} # only 3 positionals -> not present - self.assertEqual(ob._out_param_of(rec, self.meta), ("", "")) - - def test_no_declared_inplace_param_returns_empty(self): - rec = {"args": (1, 2, 3, 4), "kwargs": {}} - self.assertEqual(ob._out_param_of(rec, {"target_callable": "fake_inplace_seam:entry"}), ("", "")) - - def test_param_index_map_is_empty_for_an_unresolvable_target(self): - self.assertEqual(ob._param_index_by_name({"target_callable": "no_mod:none"}), {}) - - def test_keyword_only_output_is_never_mapped_into_the_args_tuple(self): - def variadic(query, *rest, output=None): - return None - self.mod.variadic = variadic - meta = {"target_callable": "fake_inplace_seam:variadic", - "seam_runtime_evidence": {"inplace_params": ["output"]}} - rec = {"args": (1, 2, 3), "kwargs": {}} - self.assertNotIn("output", ob._param_index_by_name(meta)) - self.assertEqual(ob._out_param_of(rec, meta), ("", "")) - - -class TestDenominatorIsFlatString(_ObStateMixin, unittest.TestCase): - """C1: the orchestrator's OPBENCH_SCHEMA + denominatorSound() match `denominator` as a flat - provenance-enum STRING. main() must emit that at top level, with the object under - denominator_detail.""" - - def _run(self, meta): - d = self._task_dir(meta) - out_path = os.path.join(d, "result.json") - ob.bench_gemm = lambda a, m: [{"backend": "hipblaslt", "available": True, - "correct": True, "ms": 1.0}] - ob.bench_attn = ob.bench_gemm - old = sys.argv - sys.argv = ["op_bench.py", "--task", d, "--out", out_path] - try: - with contextlib.redirect_stdout(io.StringIO()): - ob.main() - finally: - sys.argv = old - with open(out_path) as fh: - return json.load(fh) - - def test_denominator_is_a_string_and_detail_is_the_object(self): - s = self._run({"op_kind": "gemm"}) - self.assertIsInstance(s["denominator"], str) - self.assertIn(s["denominator"], set(ob.DENOM_SEVERITY)) - self.assertIsInstance(s["denominator_detail"], dict) - self.assertEqual(s["denominator_detail"]["provenance"], s["denominator"]) - self.assertIn("spec", s["denominator_detail"]) - - -class TestVerifyOracleSha(unittest.TestCase): - """C4: a declared reference_io_sha256 used to be trusted verbatim -- even a fabricated 'abc123' - passed, so an oracle altered after capture could still certify a 'correct' verdict. The verifier - now requires a complete digest and the referenced bytes, then recomputes and compares the identity.""" - - def _task(self, data=b"golden-oracle-bytes"): - d = tempfile.mkdtemp(prefix="op_sha_") - self.addCleanup(shutil.rmtree, d, True) - p = os.path.join(d, "reference_io.pt") - with open(p, "wb") as fh: - fh.write(data) - return d, hashlib.sha256(data).hexdigest() - - def test_matching_digest_passes(self): - d, sha = self._task() - ok, err = ob.verify_oracle_sha(d, {"reference_io_sha256": sha}) - self.assertTrue(ok, err) - self.assertEqual(err, "") - - def test_mismatched_digest_is_rejected_as_tamper(self): - d, _ = self._task() - ok, err = ob.verify_oracle_sha(d, {"reference_io_sha256": "a" * 64}) - self.assertFalse(ok) - self.assertIn("MISMATCH", err) - - def test_fabricated_short_hash_no_longer_passes(self): - """The exact fail-open the reviewer named: 'abc123' certified anything.""" - d, _ = self._task() - ok, _ = ob.verify_oracle_sha(d, {"reference_io_sha256": "abc123"}) - self.assertFalse(ok) - - def test_synthesized_oracle_is_exempt(self): - d, _ = self._task() - ok, err = ob.verify_oracle_sha(d, {"synthesized": True, "reference_io_sha256": "abc123"}) - self.assertTrue(ok, err) - - def test_absent_declared_sha_fails_closed(self): - d, _ = self._task() - self.assertFalse(ob.verify_oracle_sha(d, {})[0]) - self.assertFalse(ob.verify_oracle_sha(d, {"reference_io_sha256": ""})[0]) - - def test_absent_file_cannot_verify_the_digest(self): - d = tempfile.mkdtemp(prefix="op_sha_") - self.addCleanup(shutil.rmtree, d, True) - ok, err = ob.verify_oracle_sha(d, {"reference_io_sha256": "a" * 64}) - self.assertFalse(ok) - self.assertIn("missing", err) - - def test_loader_refuses_a_tampered_oracle(self): - d, _ = self._task(b"records-blob") - recs, err = ob.load_oracle_records(d, None, "cpu", {"reference_io_sha256": "a" * 64}) - self.assertEqual(recs, []) - self.assertIn("MISMATCH", err) - - if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/e2e_workflow/scripts/tests/test_overlay_setup.py b/e2e_workflow/scripts/tests/test_overlay_setup.py index 6824b27bd..fad27954b 100644 --- a/e2e_workflow/scripts/tests/test_overlay_setup.py +++ b/e2e_workflow/scripts/tests/test_overlay_setup.py @@ -59,7 +59,7 @@ def _load(mod_name, filename): ov = _load("overlay_setup", "overlay_setup.py") -EMPTY_MANIFEST = {"modules": [], "rebinds": [], "captures": []} +EMPTY_MANIFEST = {"modules": [], "rebinds": [], "markers": [], "captures": []} class _RecordingRun: @@ -596,6 +596,27 @@ def test_distinct_targets_compound(self): self.assertEqual([e["target"] for e in self._manifest()["captures"]], ["m:a", "m:b"]) +class TestAddMarker(_OverlayCase): + def test_marker_targets_compound_and_copy_the_probe(self): + marker = self._write("custom_marker.py", "def install(target):\n pass\n") + self._run(ov.cmd_add_marker, self._ns(target="m:outer", marker_file=marker)) + self._run(ov.cmd_add_marker, self._ns(target="m:inner", marker_file=marker)) + self.assertEqual( + [entry["target"] for entry in self._manifest()["markers"]], + ["m:outer", "m:inner"], + ) + self.assertEqual( + self._read(os.path.join(self.overlay, "seam_trace.py")), + "def install(target):\n pass\n", + ) + + def test_readding_a_marker_is_idempotent(self): + marker = self._write("custom_marker.py", "def install(target):\n pass\n") + self._run(ov.cmd_add_marker, self._ns(target="m:inner", marker_file=marker)) + self._run(ov.cmd_add_marker, self._ns(target="m:inner", marker_file=marker)) + self.assertEqual(self._manifest()["markers"], [{"target": "m:inner"}]) + + # --------------------------------------------------------------------------- # # check -- "is the overlay actually shadowing this module?" # --------------------------------------------------------------------------- # diff --git a/e2e_workflow/scripts/tests/test_parse_profile.py b/e2e_workflow/scripts/tests/test_parse_profile.py index 627542c2f..8b4931a78 100644 --- a/e2e_workflow/scripts/tests/test_parse_profile.py +++ b/e2e_workflow/scripts/tests/test_parse_profile.py @@ -326,6 +326,78 @@ def kinds(*aggs): "aten::Mul": "unresolved"}) +class TestDispatcherExpansion(_TmpMixin, unittest.TestCase): + def test_external_id_edges_expand_wrapper_into_device_kernels(self): + events = [ + {"cat": "cpu_op", "name": "vllm::attention", + "args": {"External id": 17}}, + {"cat": "kernel", "name": "kernel_paged_attention_2d", + "ts": 10, "dur": 80, "args": {"External id": 17}}, + {"cat": "kernel", "name": "_fwd_kernel", + "ts": 20, "dur": 20, "args": {"External id": 17}}, + ] + edges = pp.index_dispatch_edges(self._trace_file(events)) + rows, records = pp.expand_dispatcher_rows([{ + "rank": 1, + "name": "vllm::attention", + "short_name": "attention (paged + prefill)", + "pct_gpu_time": 25.0, + "total_ms": 10.0, + "calls": 3, + "notes": "aggregate", + }], edges) + + self.assertEqual([r["short_name"] for r in rows], + ["kernel_paged_attention_2d", "_fwd_kernel"]) + self.assertEqual([r["pct_gpu_time"] for r in rows], [20.0, 5.0]) + self.assertTrue(all(r["profile_parent"] == "vllm::attention" for r in rows)) + self.assertEqual(records[0]["dispatcher"], "vllm::attention") + + def test_nested_launch_external_id_is_attributed_to_outer_dispatcher(self): + events = [ + {"cat": "cpu_op", "name": "vllm::attention", "pid": 1, "tid": 2, + "ts": 100, "dur": 100, "args": {"External id": 17}}, + {"cat": "cpu_op", "name": "triton::launch", "pid": 1, "tid": 2, + "ts": 120, "dur": 10, "args": {"External id": 18}}, + {"cat": "kernel", "name": "kernel_paged_attention_2d", + "ts": 250, "dur": 80, "args": {"External id": 18}}, + ] + edges = pp.index_dispatch_edges(self._trace_file(events)) + self.assertIn("kernel_paged_attention_2d", edges["vllm::attention"]) + self.assertIn("kernel_paged_attention_2d", edges["triton::launch"]) + + def test_unrelated_dispatcher_is_preserved_for_fail_closed_classification(self): + original = {"name": "vllm::unknown", "pct_gpu_time": 9.0} + rows, records = pp.expand_dispatcher_rows([original], {}) + self.assertEqual(rows, [original]) + self.assertEqual(records, []) + + def test_existing_device_row_is_not_duplicated_by_dispatcher_expansion(self): + rows, _ = pp.expand_dispatcher_rows([ + {"name": "vllm::attention", "pct_gpu_time": 20.0, "total_ms": 10.0}, + {"name": "kernel_paged_attention_2d", "pct_gpu_time": 20.0, "total_ms": 10.0}, + ], { + "vllm::attention": { + "kernel_paged_attention_2d": {"calls": 4, "total_us": 100.0}, + }, + }) + self.assertEqual( + [row["name"] for row in rows].count("kernel_paged_attention_2d"), 1) + self.assertEqual(sum(row["pct_gpu_time"] for row in rows), 20.0) + + def test_exact_host_device_collision_is_not_expanded(self): + row = {"name": "shared", "pct_gpu_time": 20.0} + evidence = pp.merge_entity_evidence( + {"shared": {"calls": 1, "total_us": 5.0, "cat_counts": {"cpu_op": 1}}}, + {"shared": {"calls": 1, "total_us": 7.0, "cat_counts": {"kernel": 1}}}, + ) + rows, records = pp.expand_dispatcher_rows( + [row], {"shared": {"child_kernel": {"calls": 1, "total_us": 7.0}}}, + evidence, "torch-trace") + self.assertEqual(rows, [row]) + self.assertEqual(records, []) + + # --------------------------------------------------------------------------- # # serving-phase step windows # --------------------------------------------------------------------------- # diff --git a/e2e_workflow/scripts/tests/test_seam_contract.py b/e2e_workflow/scripts/tests/test_seam_contract.py deleted file mode 100644 index c3c8c3ff0..000000000 --- a/e2e_workflow/scripts/tests/test_seam_contract.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for the live-seam binding contract.""" -import contextlib -import importlib.util -import inspect -import io -import json -import os -import shutil -import sys -import tempfile -import types -import unittest - - -SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SPEC = importlib.util.spec_from_file_location( - "seam_contract", os.path.join(SCRIPTS_DIR, "seam_contract.py")) -sc = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(sc) - - -def _descriptor(fn, seam="fixture:entry", evidence=True): - runtime = {"inplace_params": [], "returns_none": False} - if evidence: - runtime["hidden_context"] = [] - return sc._describe_signature( - inspect.signature(fn), seam, fn.__name__, None, - {"seam_runtime_evidence": runtime}) - - -class TestBindingCompatibility(unittest.TestCase): - def test_identical_positional_only_signature_is_bindable(self): - def entry(a, b, /, c=None): - return a - - desc = _descriptor(entry) - verdict = sc.check_binding(desc, desc) - self.assertTrue(verdict["bindable"], verdict["mismatches"]) - - def test_rendered_entry_preserves_positional_only_marker(self): - def entry(a, b, /, c=None): - return a - - namespace = {} - exec(sc.render_entry(_descriptor(entry)), namespace) - self.assertEqual(str(inspect.signature(namespace["entry"])), "(a, b, /, c=None)") - - def test_missing_hidden_context_evidence_fails_closed(self): - def entry(x): - return x - - desc = _descriptor(entry, evidence=False) - verdict = sc.check_binding(desc, _descriptor(entry)) - self.assertFalse(verdict["bindable"]) - self.assertIn("hidden_context_inputs", verdict["codes"]) - - def test_null_hidden_context_is_not_explicit_purity_evidence(self): - def entry(x): - return x - - desc = sc._describe_signature( - inspect.signature(entry), "fixture:entry", "entry", None, - {"seam_runtime_evidence": {"hidden_context": None}}) - self.assertEqual(desc["hidden_context_evidence"], "unknown") - self.assertFalse(sc.check_binding(desc, _descriptor(entry))["bindable"]) - - def test_dropped_optional_parameter_is_rejected(self): - def live(a, optional=None): - return a - - def candidate(a): - return a - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertFalse(verdict["bindable"]) - self.assertIn("optional_param_dropped", verdict["codes"]) - - # ---- C6 case #2: POSITIONAL_OR_KEYWORD live param declared POSITIONAL_ONLY on candidate. - # The reviewer's own second false-positive. live f(a, b) accepts a keyword call f(a=.., b=..); - # a candidate f(a, b, /) rejects that call, so it CANNOT be rebound at a keyword call site even - # though required names + arity line up. The representative-call bind simulation must catch it. - # Descriptors carry hidden_context=[] (via _descriptor) so the strict hidden_context gate does - # not mask the param-kind logic under test. - def test_positional_or_keyword_bound_as_positional_only_is_rejected(self): - def live(a, b): - return a - - def candidate(a, b, /): - return a - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertFalse(verdict["bindable"], verdict["mismatches"]) - self.assertIn("param_kind_mismatch", verdict["codes"]) - # It is specifically the keyword-call surface (not arity/name) that fails. - self.assertNotIn("arity_mismatch", verdict["codes"]) - self.assertNotIn("param_name_mismatch", verdict["codes"]) - - def test_reordered_positional_parameters_are_accepted(self): - # live f(a, b) vs candidate f(b, a): every representative live call (positional AND keyword) - # still binds against the candidate, so this is harmless and PASSES. - def live(a, b): - return a - - def candidate(b, a): - return b - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertTrue(verdict["bindable"], verdict["mismatches"]) - - def test_candidate_varargs_varkw_swallows_live_signature(self): - # A candidate that accepts (*args, **kwargs) can absorb every representative live call, so the - # varargs/varkw candidate binds. Exercises the varargs/varkw short-circuits in check_binding. - def live(a, b): - return a - - def candidate(*args, **kwargs): - return args - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertTrue(verdict["bindable"], verdict["mismatches"]) - - def test_live_varargs_not_absorbed_by_fixed_candidate_is_rejected(self): - # live f(a, *args) accepts an extra positional; the representative-call simulation appends one - # (include_optional leg), and a fixed-arity candidate f(a) rejects it -> param_kind_mismatch. - def live(a, *args): - return a - - def candidate(a): - return a - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertFalse(verdict["bindable"], verdict["mismatches"]) - self.assertIn("param_kind_mismatch", verdict["codes"]) - - def test_live_varkw_not_absorbed_by_fixed_candidate_is_rejected(self): - # live f(a, **kwargs) accepts an extra keyword; the representative-call simulation adds one, and - # a fixed candidate f(a) rejects it -> param_kind_mismatch via the bind simulation. - def live(a, **kwargs): - return a - - def candidate(a): - return a - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertFalse(verdict["bindable"], verdict["mismatches"]) - self.assertIn("param_kind_mismatch", verdict["codes"]) - - def test_positional_to_keyword_only_is_rejected_deliberate_safe_direction(self): - # DOCUMENTED CURRENT BEHAVIOR: live f(a, b) vs candidate f(a, *, b). The reviewer flagged this - # as harmless (a keyword-only candidate param can still receive the live positional value by - # name), but the revised code intentionally FAILS CLOSED: a representative live call passes b - # positionally, which the keyword-only candidate rejects -> param_kind_mismatch. This is a - # deliberate safe-direction over-rejection (better to reject a bindable candidate than to admit - # an unbindable one); this test pins that intended behavior, not an accidental one. - def live(a, b): - return a - - def candidate(a, *, b): - return a - - verdict = sc.check_binding(_descriptor(live), _descriptor(candidate)) - self.assertFalse(verdict["bindable"], verdict["mismatches"]) - self.assertIn("param_kind_mismatch", verdict["codes"]) - - # ---- C7 edge: hidden_context that is neither a list nor None. Only None was covered; a dict or a - # str must ALSO be treated as unknown (not "declared") and therefore fail closed as not bindable. - def test_hidden_context_dict_is_unknown_and_fails_closed(self): - def entry(x): - return x - - desc = sc._describe_signature( - inspect.signature(entry), "fixture:entry", "entry", None, - {"seam_runtime_evidence": {"inplace_params": [], "returns_none": False, - "hidden_context": {"forward_ctx": "layer"}}}) - self.assertEqual(desc["hidden_context_evidence"], "unknown") - verdict = sc.check_binding(desc, _descriptor(entry)) - self.assertFalse(verdict["bindable"]) - self.assertIn("hidden_context_inputs", verdict["codes"]) - - def test_hidden_context_str_is_unknown_and_fails_closed(self): - def entry(x): - return x - - desc = sc._describe_signature( - inspect.signature(entry), "fixture:entry", "entry", None, - {"seam_runtime_evidence": {"inplace_params": [], "returns_none": False, - "hidden_context": "forward_ctx"}}) - self.assertEqual(desc["hidden_context_evidence"], "unknown") - verdict = sc.check_binding(desc, _descriptor(entry)) - self.assertFalse(verdict["bindable"]) - self.assertIn("hidden_context_inputs", verdict["codes"]) - - -class TestCliBinding(unittest.TestCase): - def test_target_override_updates_descriptor_identity_and_checks_rendered_entry(self): - module = types.ModuleType("seam_contract_fixture") - - def entry(a, /, b=None): - return a - - module.entry = entry - sys.modules[module.__name__] = module - self.addCleanup(sys.modules.pop, module.__name__, None) - task = tempfile.mkdtemp(prefix="seam_contract_") - self.addCleanup(shutil.rmtree, task, True) - with open(os.path.join(task, "meta.json"), "w") as fh: - json.dump({ - "target_callable": "wrong.module:entry", - "seam_runtime_evidence": { - "inplace_params": [], - "returns_none": False, - "hidden_context": [], - }, - }, fh) - - output = io.StringIO() - with contextlib.redirect_stdout(output): - rc = sc.main([ - "--task-dir", task, - "--target-spec", "seam_contract_fixture:entry", - "--mode", "binding", - "--json", - ]) - result = json.loads(output.getvalue()) - self.assertEqual(rc, 0) - self.assertEqual(result["binding_descriptor"]["seam"], "seam_contract_fixture:entry") - self.assertEqual(result["binding_check"]["candidate"], "") - self.assertTrue(result["binding_check"]["bindable"]) - - def _install_fixture_module(self, name, **members): - module = types.ModuleType(name) - for attr, fn in members.items(): - setattr(module, attr, fn) - sys.modules[name] = module - self.addCleanup(sys.modules.pop, name, None) - return module - - def _write_meta(self, **overrides): - task = tempfile.mkdtemp(prefix="seam_contract_") - self.addCleanup(shutil.rmtree, task, True) - meta = { - "seam_runtime_evidence": { - "inplace_params": [], "returns_none": False, "hidden_context": [], - }, - } - meta.update(overrides) - with open(os.path.join(task, "meta.json"), "w") as fh: - json.dump(meta, fh) - return task - - # ---- C10 NEGATIVE: --baseline-spec must NEVER leak into the binding descriptor. The descriptor is - # built from the deployment TARGET; the baseline only steers baseline_validation. (The positive - # direction -- --target-spec drives the descriptor -- is covered above.) - def test_baseline_spec_does_not_leak_into_binding_descriptor(self): - def entry(a, /, b=None): - return a - - def baseline(x, y, z): # deliberately a DIFFERENT callable / different arity - return x - - self._install_fixture_module("seam_c10_target", entry=entry) - self._install_fixture_module("seam_c10_baseline", baseline=baseline) - task = self._write_meta( - target_callable="wrong.module:entry", baseline_callable="wrong.module:baseline") - - output = io.StringIO() - with contextlib.redirect_stdout(output): - sc.main([ - "--task-dir", task, - "--target-spec", "seam_c10_target:entry", - "--baseline-spec", "seam_c10_baseline:baseline", - "--mode", "both", - "--json", - ]) - result = json.loads(output.getvalue()) - # The descriptor is the TARGET seam, never the baseline. - self.assertEqual(result["binding_descriptor"]["seam"], "seam_c10_target:entry") - self.assertNotEqual(result["binding_descriptor"]["seam"], "seam_c10_baseline:baseline") - # Descriptor is built from the target's real signature, not the baseline's (x, y, z). - self.assertTrue(result["binding_descriptor"]["signature"].endswith("(a, /, b=None)")) - # --baseline-spec only steers baseline_validation. - self.assertEqual( - result["baseline_validation"]["baseline_callable"], "seam_c10_baseline:baseline") - self.assertEqual(result["baseline_validation"]["target_callable"], "seam_c10_target:entry") - - # ---- C2: '--mode both' with NO --candidate must still emit a binding_check against the rendered - # entry (candidate == ""). The existing coverage used '--mode binding'. - def test_mode_both_without_candidate_checks_rendered_entry(self): - def entry(a, /, b=None): - return a - - self._install_fixture_module("seam_c2_target", entry=entry) - task = self._write_meta(target_callable="seam_c2_target:entry") - - output = io.StringIO() - with contextlib.redirect_stdout(output): - sc.main([ - "--task-dir", task, - "--target-spec", "seam_c2_target:entry", - "--mode", "both", - "--json", - ]) - result = json.loads(output.getvalue()) - self.assertIn("binding_check", result) - self.assertEqual(result["binding_check"]["candidate"], "") - self.assertTrue(result["binding_check"]["bindable"], result["binding_check"]["mismatches"]) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/e2e_workflow/scripts/tests/test_seam_trace.py b/e2e_workflow/scripts/tests/test_seam_trace.py new file mode 100644 index 000000000..80cbe4988 --- /dev/null +++ b/e2e_workflow/scripts/tests/test_seam_trace.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Tests for marker-only candidate tracing.""" + +import importlib.util +import os +import sys +import tempfile +import types +import unittest + + +SCRIPTS = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SPEC = importlib.util.spec_from_file_location("seam_trace", os.path.join(SCRIPTS, "seam_trace.py")) +st = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(st) + + +class _Context: + def __init__(self, events, name): + self.events = events + self.name = name + + def __enter__(self): + self.events.append(("enter", self.name)) + + def __exit__(self, *unused): + self.events.append(("exit", self.name)) + + +class _Profiler: + def __init__(self, events): + self.events = events + + def __enter__(self): + self.events.append(("profile", "start")) + return self + + def __exit__(self, *unused): + self.events.append(("profile", "stop")) + + def export_chrome_trace(self, path): + with open(path, "w") as fh: + fh.write("{}") + + +class TestSeamTrace(unittest.TestCase): + def setUp(self): + self.events = [] + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.trace = os.path.join(self.tmp.name, "selection.json") + os.environ["GEAK_SELECTION_TRACE"] = self.trace + os.environ["GEAK_SELECTION_TRACE_UNIQUE"] = "0" + os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "1" + self.addCleanup(os.environ.pop, "GEAK_SELECTION_TRACE", None) + self.addCleanup(os.environ.pop, "GEAK_SELECTION_TRACE_UNIQUE", None) + self.addCleanup(os.environ.pop, "GEAK_SELECTION_PROFILE_CALLS", None) + + torch = types.ModuleType("torch") + torch.profiler = types.SimpleNamespace( + ProfilerActivity=types.SimpleNamespace(CPU="cpu", CUDA="cuda"), + profile=lambda activities: _Profiler(self.events), + record_function=lambda name: _Context(self.events, name), + ) + self.saved_torch = sys.modules.get("torch") + sys.modules["torch"] = torch + self.addCleanup(self._restore_torch) + + module = types.ModuleType("_seam_trace_fixture") + module.inner = lambda value: value + 1 + module.outer = lambda value: module.inner(value) * 2 + sys.modules[module.__name__] = module + self.module = module + self.addCleanup(sys.modules.pop, module.__name__, None) + + st._INSTALLED.clear() + st._PROFILE.update(lock=st._PROFILE["lock"], active=False, done=False, + owner=None, profiler=None, active_calls=0, root_calls=0, out="", + trace_index=0, atexit_registered=False) + + def _restore_torch(self): + if self.saved_torch is None: + sys.modules.pop("torch", None) + else: + sys.modules["torch"] = self.saved_torch + + def test_first_outer_call_profiles_nested_candidate_markers_once(self): + st.install("_seam_trace_fixture:outer") + st.install("_seam_trace_fixture:inner") + self.assertEqual(self.module.outer(3), 8) + self.assertTrue(os.path.isfile(self.trace)) + names = [value for action, value in self.events if action == "enter"] + self.assertEqual(names, [ + st.INSTALL_PREFIX + "_seam_trace_fixture:inner", + st.INSTALL_PREFIX + "_seam_trace_fixture:outer", + st.MARKER_PREFIX + "_seam_trace_fixture:outer", + st.MARKER_PREFIX + "_seam_trace_fixture:inner", + ]) + self.assertEqual(self.events.count(("profile", "start")), 1) + self.assertEqual(self.events.count(("profile", "stop")), 1) + + def test_jit_protocol_callable_is_rejected_before_replacement(self): + class JitLike: + def __call__(self, value): + return value + + def run(self, value): + return value + + original = JitLike() + self.module.jit_entry = original + with self.assertRaisesRegex(RuntimeError, "cannot safely mark"): + st.install("_seam_trace_fixture:jit_entry") + self.assertIs(self.module.jit_entry, original) + + def test_each_root_call_exports_before_process_exit(self): + os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "32" + st.install("_seam_trace_fixture:outer") + self.assertEqual(self.module.outer(3), 8) + self.assertTrue(os.path.isfile(self.trace)) + self.assertEqual(self.events.count(("profile", "stop")), 1) + + def test_process_local_call_traces_do_not_overwrite(self): + os.environ.pop("GEAK_SELECTION_TRACE_UNIQUE", None) + os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "2" + st.install("_seam_trace_fixture:outer") + self.module.outer(1) + self.module.outer(2) + traces = sorted( + name for name in os.listdir(self.tmp.name) + if name.startswith("selection.pid-") and name.endswith(".json")) + self.assertEqual(len(traces), 2) + self.assertIn(".call-1.json", traces[0]) + self.assertIn(".call-2.json", traces[1]) + + +if __name__ == "__main__": + unittest.main() From 546d38f466548e1f6cde6c444d08a1a99e000c6d Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Wed, 19 Aug 2026 23:28:54 +0000 Subject: [PATCH 4/8] Settle "deepest live seam" from observed nesting, not a self-reported depth Verifying the discovery contract against the 88-run archive turned up six ways it could be bypassed or misled. The selection contract trusted candidate.depth, an integer supplied by the same agent whose choice is under review. An extractor could append a candidate of its own carrying depth:999, nominate it, and the gate returned ok even though the architect had declared a genuinely deeper launcher. kernel_selection.py already derives deeper_live_candidates from the profiler's host-span nesting; the gate now reads that. The declared chain stays as a second, independent way to fail -- it can add a rejection but can never buy a pass. A depth that is absent or not a finite number now removes the candidate instead of ranking it as zero, where every NaN comparison was silently false. seam_trace could not resolve a dotted attr, so Class.method seams dropped out of the probe -- and an omitted candidate is exactly what the coverage check exists to catch. Marker installs also had to move after the capture hooks: importing a target module during a marker install aliases the un-captured function, so the oracle and the probed object were not the same callable. The profiler re-emits each marker on the device timeline, where an outer seam's short span routinely lands inside an inner one's; reading those projections inverted the call nesting and refused the correct launcher. A gpu_kernel head naming no device symbol at all reached extraction with the contract switched off: requiredDeviceKernel returned '', which the gate reports as "not a profiled GPU-head extraction". The vacuous pass is right for a non-kernel head, so admission is what stops it now. Finally, a bare `mismatch` in the correctness regex was tested first and swallowed signature_mismatch, sending a seam defect to the corrective that hunts data_ptr over-fits rather than the one that matches the live dispatch signature. Co-authored-by: Cursor --- e2e_workflow/e2e_workflow.js | 57 ++++++++++++++++--- e2e_workflow/scripts/kernel_selection.py | 13 +++++ e2e_workflow/scripts/overlay_setup.py | 20 ++++--- e2e_workflow/scripts/seam_trace.py | 17 +++++- .../scripts/tests/test_kernel_selection.py | 22 +++++++ .../scripts/tests/test_overlay_setup.py | 7 +++ e2e_workflow/scripts/tests/test_seam_trace.py | 12 ++++ 7 files changed, 129 insertions(+), 19 deletions(-) diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index d5e598f78..b3504ce8f 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -220,10 +220,18 @@ function isImplausibleSpeedup(pct_gpu_time, isolated, integ) { return ((integ && integ.e2e_delta_pct) || 0) > ceilPct * (1 + IMPLAUSIBLE_SPEEDUP_MARGIN) + 1e-9; } // Classify a reject reason into a fix-and-retry class ('' = terminal, not auto-correctable). +// POSTURE IS TESTED FIRST. CORRECTNESS_REJECT_RX contains a bare `mismatch`, which used to swallow +// the `signature_mismatch` that FIXABLE_REJECT_RX names explicitly: a seam whose signature does not +// match the live call site was routed to the correctness corrective ("your kernel computes the wrong +// thing on the live path, look for a data_ptr over-fit") when what it needs is the integration one +// ("find the method the live server actually dispatches and match its call signature"). The posture +// tokens are specific (signature/seam/engagement/capture), so testing them first cannot capture a +// genuine output-correctness reject; and until the candidate is installed at the right seam, any +// statement about its output is meaningless anyway. function rejectClass(reason) { const r = reason || ''; - if (CORRECTNESS_REJECT_RX.test(r)) return 'correctness'; if (FIXABLE_REJECT_RX.test(r)) return 'integration'; + if (CORRECTNESS_REJECT_RX.test(r)) return 'correctness'; return ''; } // A gate 'accept'/'stack' only counts as a REAL win if the measured e2e delta is not an implausible @@ -731,12 +739,22 @@ function kernelIdentitiesMatch(a, b) { } function requiredDeviceKernel(h) { if (!h || h.entity_kind !== 'gpu_kernel') return ''; - return String(h.device_kernel || h.profile_kernel || + // entity_evidence.matched_profiled_kernel is what parse_profile.py --annotate stamped on the row, + // so it is the profiler's own name for this entity and outranks the display short_name. + return String(h.device_kernel || (h.entity_evidence && h.entity_evidence.matched_profiled_kernel) || h.short_name || h.name || '').trim(); } const candidateSpec = (c) => String((c && (c.target_callable || c.callable || c.spec)) || '').trim(); +// A declared depth is a STRUCTURAL claim about the call chain. It orders candidates before any +// runtime evidence exists, so a value that is absent or not a finite number is unusable: `Number()` +// turns it into NaN, every comparison against it is silently false, and a genuinely deeper candidate +// stops being able to outrank anything. Refuse the candidate instead of ranking it as depth 0. +const candidateDepth = (c) => { + const depth = Number(c && c.depth); + return Number.isFinite(depth) ? depth : null; +}; function selectionCandidatesForHead(h, ext) { const required = requiredDeviceKernel(h); const fused = !!(h && (h.is_fused_kernel === true || h.op_kind === 'moe')); @@ -754,6 +772,7 @@ function selectionCandidatesForHead(h, ext) { return false; const role = String(candidate.role || '').toLowerCase(); if (role === 'kernel_entry') return false; // native/JIT object identity is unsafe to monkeypatch + if (candidateDepth(candidate) === null) return false; const kernels = Array.isArray(candidate.device_kernels) ? candidate.device_kernels : []; if (required && !kernels.some((kernel) => kernelIdentitiesMatch(required, kernel))) return false; return (fused @@ -773,7 +792,7 @@ function prepareHeadSelection(h) { const role = String(candidate.role || '').toLowerCase(); return fused ? role === 'op_seam' : ['inner_launcher', 'op_seam'].includes(role); }); - deployable.sort((a, b) => Number(b.depth || 0) - Number(a.depth || 0)); + deployable.sort((a, b) => candidateDepth(b) - candidateDepth(a)); out.target_callable = deployable.length ? candidateSpec(deployable[0]) : ''; out.selection_status = out.target_callable ? 'candidate_selected_needs_runtime_verification' @@ -819,14 +838,24 @@ function kernelSelectionVerified(h, ext) { why: fused ? `fused head must select the whole-operation op_seam, got '${selectedRole || 'unknown'}'` : `selected target role '${selectedRole || 'unknown'}' is not a deployable inner/op seam` }; + // "Deepest" is settled by the profiler's OBSERVED host-span nesting (kernel_selection.py computes + // deeper_live_candidates from it), not by the declared `depth`. The declared integer is a claim by + // the same agent whose choice is under review, and an appended candidate carrying a large depth + // would otherwise outrank a genuinely deeper one. The structural claim is still checked, but only + // as an ADDITIONAL way to fail: it can never buy a pass that the observed nesting did not grant. + const observedDeeper = (verdict.deeper_live_candidates || []) + .map((value) => String(value || '').trim()).filter(Boolean); + if (observedDeeper.length) + return { ok: false, + why: `deeper live candidate(s) observed across calls/ranks: ${observedDeeper.join(', ')}` }; const live = new Set((verdict.live_candidate_targets || []).map((value) => String(value || '').trim())); - const selectedDepth = Number(selected.depth || 0); + const selectedDepth = candidateDepth(selected); const deeper = candidates.filter((candidate) => candidateSpec(candidate) !== target && live.has(candidateSpec(candidate)) && - Number(candidate.depth || 0) > selectedDepth); + candidateDepth(candidate) > selectedDepth); if (deeper.length) return { ok: false, - why: `deeper live candidate(s) observed across calls/ranks: ${deeper.map(candidateSpec).join(', ')}` }; + why: `declared-deeper live candidate(s): ${deeper.map(candidateSpec).join(', ')}` }; if (verdict.deepest_verified !== true) return { ok: false, why: 'selected callable is not machine-verified as deepest' }; return { ok: true, why: `${target} launches profiled kernel ${required}` }; @@ -836,14 +865,26 @@ const PRE_FLAGGED_HEADS = []; function admitHeads(queue, stage) { const admitted = []; for (const head of (queue || []).filter(Boolean)) { + const label = head.short_name || head.name || '(unnamed)'; if (head.entity_kind !== 'gpu_kernel') { - log(` ⚠️ FLAG ${head.short_name || head.name}: entity_kind=${head.entity_kind || 'missing'}; ` + + log(` ⚠️ FLAG ${label}: entity_kind=${head.entity_kind || 'missing'}; ` + `the head track requires a profiler-confirmed gpu_kernel (${stage}).`); - PRE_FLAGGED_HEADS.push({ short_name: head.short_name || head.name, + PRE_FLAGGED_HEADS.push({ short_name: label, pct_gpu_time: head.pct_gpu_time, stage, gate: 'wrong_head_granularity', reason: `entity_kind=${head.entity_kind || 'missing'}; gpu_kernel required` }); continue; } + // A gpu_kernel head with NO device identity at all makes requiredDeviceKernel return '', and + // kernelSelectionVerified then passes it as "not a profiled GPU-head extraction". That vacuous + // pass is the whole contract switched off for this head, so refuse it at admission instead. + if (!requiredDeviceKernel(head)) { + log(` ⚠️ FLAG ${label}: entity_kind=gpu_kernel but no device_kernel/short_name/name; ` + + `there is no profiled symbol to verify a seam against (${stage}).`); + PRE_FLAGGED_HEADS.push({ short_name: label, + pct_gpu_time: head.pct_gpu_time, stage, gate: 'missing_kernel_identity', + reason: 'gpu_kernel head carries no device_kernel/short_name/name' }); + continue; + } admitted.push(prepareHeadSelection(head)); } return admitted; diff --git a/e2e_workflow/scripts/kernel_selection.py b/e2e_workflow/scripts/kernel_selection.py index fe084d87c..21068b140 100644 --- a/e2e_workflow/scripts/kernel_selection.py +++ b/e2e_workflow/scripts/kernel_selection.py @@ -86,12 +86,25 @@ def kernel_matches(expected, observed): ) +def _device_projection(event): + """True for the GPU-timeline copy of a host annotation (``gpu_user_annotation``). + + The profiler re-emits every ``record_function`` marker on the device timeline, where the span + covers the kernels it launched rather than the Python call. Those projections do not preserve + host call nesting -- a short device span for an OUTER seam routinely lands inside the device span + of an INNER one -- so only host-side spans may establish nesting or launch causality. + """ + return str(event.get("cat") or "").startswith("gpu_") + + def _complete_spans(events, name): spans = [] stacks = {} for event in events: if not isinstance(event, dict) or event.get("name") != name: continue + if _device_projection(event): + continue phase = event.get("ph", "X") if phase == "X" and event.get("ts") is not None: start = float(event["ts"]) diff --git a/e2e_workflow/scripts/overlay_setup.py b/e2e_workflow/scripts/overlay_setup.py index d62cc541f..01e5b92d1 100755 --- a/e2e_workflow/scripts/overlay_setup.py +++ b/e2e_workflow/scripts/overlay_setup.py @@ -76,21 +76,23 @@ except Exception as _ex: sys.stderr.write("[overlay] rebind FAILED %r: %r\n" % (_e, _ex)) -# (c) marker-only hooks used to compare every candidate seam in one trace. -for _e in _m.get("markers", []): - try: - import seam_trace - seam_trace.install(_e["target"]) - except Exception as _ex: - sys.stderr.write("[overlay] seam marker install FAILED %r: %r\n" % (_e, _ex)) - -# (d) capture hooks (shape/IO oracle recording). +# (c) capture hooks (shape/IO oracle recording) go on FIRST, so the capture wrapper is the innermost +# stand-in and is already bound before any marker install imports a module that does +# `from import ` (which would otherwise alias the un-captured function). for _e in _m.get("captures", []): try: import capture_shapes capture_shapes.install(_e["target"], _e["out"], int(_e.get("max", 5))) except Exception as _ex: sys.stderr.write("[overlay] capture install FAILED %r: %r\n" % (_e, _ex)) + +# (d) marker-only hooks used to compare every candidate seam in one trace. +for _e in _m.get("markers", []): + try: + import seam_trace + seam_trace.install(_e["target"]) + except Exception as _ex: + sys.stderr.write("[overlay] seam marker install FAILED %r: %r\n" % (_e, _ex)) ''' diff --git a/e2e_workflow/scripts/seam_trace.py b/e2e_workflow/scripts/seam_trace.py index ca17b3fc3..28859a6a3 100644 --- a/e2e_workflow/scripts/seam_trace.py +++ b/e2e_workflow/scripts/seam_trace.py @@ -151,12 +151,25 @@ def _leave_call(root_call): _finish_profile() +def _resolve_owner(target): + """Resolve ``module:attr`` (attr may be dotted, e.g. ``Class.method``) to (owner, leaf). + + Class-method seams such as ``pkg.mod:RunnerCore.run`` are declared candidates just as often as + module-level functions; resolving only the flat case silently dropped them from the probe. + """ + module_name, attr = target.split(":", 1) + owner = importlib.import_module(module_name) + parts = attr.split(".") + for part in parts[:-1]: + owner = getattr(owner, part) + return owner, parts[-1] + + def install(target): """Wrap one module:attr with a record_function marker; idempotent per target.""" if target in _INSTALLED: return - module_name, attr = target.split(":", 1) - module = importlib.import_module(module_name) + module, attr = _resolve_owner(target) original = getattr(module, attr) if not _wrappable(original): raise RuntimeError(f"cannot safely mark non-Python callable {target}") diff --git a/e2e_workflow/scripts/tests/test_kernel_selection.py b/e2e_workflow/scripts/tests/test_kernel_selection.py index f47c77982..13dd2793f 100644 --- a/e2e_workflow/scripts/tests/test_kernel_selection.py +++ b/e2e_workflow/scripts/tests/test_kernel_selection.py @@ -134,6 +134,28 @@ def test_installed_but_inactive_alternative_branch_does_not_fail(self): sorted([TARGET, alternative]), ) + def test_device_projected_annotation_does_not_invert_call_nesting(self): + # The profiler re-emits each marker on the GPU timeline as `gpu_user_annotation`, where an + # OUTER seam's short device span can land inside the INNER launcher's long one. Only the + # host spans describe the real call nesting; the selected inner launcher must still pass. + outer = "sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe:_fused_moe_kernel_sequence" + events = trace() + events.insert(1, {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + outer, + "ph": "X", "pid": 1, "tid": 2, "ts": 92, "dur": 1}) + events.insert(2, {"cat": "cpu_op", "name": ks.MARKER_PREFIX + outer, + "ph": "X", "pid": 1, "tid": 2, "ts": 95, "dur": 200}) + # device projections: outer's is a tiny span inside the target's long one, same pid/tid + events += [ + {"cat": "gpu_user_annotation", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 9, "tid": 9, "ts": 500, "dur": 400}, + {"cat": "gpu_user_annotation", "name": ks.MARKER_PREFIX + outer, + "ph": "X", "pid": 9, "tid": 9, "ts": 600, "dur": 5}, + ] + verdict = ks.verify(TARGET, KERNEL, self.meta(), events, [TARGET, outer]) + self.assertEqual(verdict["deeper_live_candidates"], []) + self.assertTrue(verdict["ok"], verdict) + self.assertTrue(verdict["deepest_verified"]) + def test_capture_of_a_different_callable_is_rejected(self): wrong = "vllm.model_executor.layers.attention.attention:outer_wrapper" verdict = ks.verify(TARGET, KERNEL, self.meta(target=wrong), trace()) diff --git a/e2e_workflow/scripts/tests/test_overlay_setup.py b/e2e_workflow/scripts/tests/test_overlay_setup.py index fad27954b..1e48c193b 100644 --- a/e2e_workflow/scripts/tests/test_overlay_setup.py +++ b/e2e_workflow/scripts/tests/test_overlay_setup.py @@ -210,6 +210,13 @@ def test_shim_calls_capture_shapes_install_as_capture_shapes_defines_it(self): self.assertIn("def install(target, out_dir, max_cases=5):", self._read(os.path.join(SCRIPTS_DIR, "capture_shapes.py"))) + def test_shim_installs_captures_before_markers(self): + # A marker on any module that does `from import ` imports that module + # and freezes the alias. If markers ran first, the later capture hook would rebind only the + # defining module and the live call (through the alias) would never be recorded. + shim = ov.SITECUSTOMIZE + self.assertLess(shim.index('_m.get("captures", [])'), shim.index('_m.get("markers", [])')) + def test_rerun_preserves_an_edited_shim_and_an_existing_manifest(self): # Re-running any add-* must not reset an overlay that already carries accepted kernels. ov._ensure_overlay(self.overlay) diff --git a/e2e_workflow/scripts/tests/test_seam_trace.py b/e2e_workflow/scripts/tests/test_seam_trace.py index 80cbe4988..2f227ca3e 100644 --- a/e2e_workflow/scripts/tests/test_seam_trace.py +++ b/e2e_workflow/scripts/tests/test_seam_trace.py @@ -120,6 +120,18 @@ def test_each_root_call_exports_before_process_exit(self): self.assertTrue(os.path.isfile(self.trace)) self.assertEqual(self.events.count(("profile", "stop")), 1) + def test_class_method_candidate_is_marked_and_proven_installed(self): + class Runner: + def run(self, value): + return value + 1 + + self.module.Runner = Runner + st.install("_seam_trace_fixture:Runner.run") + self.assertEqual(Runner().run(3), 4) + names = [value for action, value in self.events if action == "enter"] + self.assertIn(st.INSTALL_PREFIX + "_seam_trace_fixture:Runner.run", names) + self.assertIn(st.MARKER_PREFIX + "_seam_trace_fixture:Runner.run", names) + def test_process_local_call_traces_do_not_overwrite(self): os.environ.pop("GEAK_SELECTION_TRACE_UNIQUE", None) os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "2" From 9f584adbc87d260db895fb86a3ae219ab9b25b5b Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Thu, 20 Aug 2026 05:18:45 +0000 Subject: [PATCH 5/8] Link Triton kernels by launch correlation, and report probes that were installed but never observed --- e2e_workflow/scripts/kernel_selection.py | 48 +++++++++-- .../scripts/tests/test_kernel_selection.py | 82 +++++++++++++++++++ 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/e2e_workflow/scripts/kernel_selection.py b/e2e_workflow/scripts/kernel_selection.py index 21068b140..5fb59de00 100644 --- a/e2e_workflow/scripts/kernel_selection.py +++ b/e2e_workflow/scripts/kernel_selection.py @@ -60,6 +60,8 @@ def merge_process_traces(paths): args = dict(original.get("args") or {}) if args.get("External id") is not None: args["External id"] = prefix + str(args["External id"]) + if args.get("correlation") is not None: + args["correlation"] = prefix + str(args["correlation"]) event["args"] = args merged.append(event) return merged @@ -138,27 +140,40 @@ def _marker_kernel_evidence(trace_events, target_callable, device_kernel): # Only CPU events on the marker's own thread can establish launch causality. Global timestamp # overlap is unsafe under concurrent serving. The resulting External ids bridge to async GPU events. related_external_ids = set() + # Triton (and any raw hipModuleLaunchKernel) device rows carry only `correlation`, never an + # `External id`, so the External-id bridge alone silently misses them. The host-side launch + # runtime event inside the marker span carries BOTH ids, so its `correlation` is an equally + # strict launch-causality bridge: same thread, inside the marker span, same launch. + related_correlations = set() for event in trace_events or []: if not isinstance(event, dict) or not _within_any_span(event, spans, same_thread=True): continue if event.get("cat") == "kernel": continue - ext = (event.get("args") or {}).get("External id") + args = event.get("args") or {} + ext = args.get("External id") if ext is not None: related_external_ids.add(ext) + corr = args.get("correlation") + if corr is not None: + related_correlations.add(corr) matched = [] for event in trace_events or []: if not isinstance(event, dict) or event.get("cat") != "kernel": continue - ext = (event.get("args") or {}).get("External id") - if ext is not None and ext in related_external_ids and kernel_matches( - device_kernel, event.get("name")): + args = event.get("args") or {} + ext = args.get("External id") + corr = args.get("correlation") + linked = (ext is not None and ext in related_external_ids) or ( + corr is not None and corr in related_correlations) + if linked and kernel_matches(device_kernel, event.get("name")): matched.append(str(event.get("name") or "")) return { "target": target_callable, "marker": marker, "spans": spans, "related_external_ids": related_external_ids, + "related_correlations": related_correlations, "matched": matched, } @@ -219,7 +234,7 @@ def verify(target_callable, device_kernel, capture_meta, trace_events, candidate failed.append("candidate_marker_not_installed") selected_evidence = evidence.get(target_callable) or { "marker": MARKER_PREFIX + target_callable, "spans": [], - "related_external_ids": set(), "matched": [], + "related_external_ids": set(), "related_correlations": set(), "matched": [], } marker = selected_evidence["marker"] spans = selected_evidence["spans"] @@ -237,6 +252,16 @@ def verify(target_callable, device_kernel, capture_meta, trace_events, candidate ] if deeper: failed.append("deeper_live_candidate_exists") + # A candidate whose marker was installed and never fired is NOT evidence that the callable is + # off the live path. Installation rebinds one module attribute; a callable reached through the + # torch dispatcher (`torch.ops.*`) or through an alias a caller imported before installation + # still runs with the wrapper bypassed, and is therefore invisible here. `deeper` can only see + # what the markers saw, so report the unobserved candidates separately instead of letting them + # be read as "not deeper": whoever holds the declared depths decides what an unobserved + # declared-deeper candidate means. + unobserved_candidates = sorted( + candidate for candidate in installed_candidates + if candidate != target_callable and not (evidence.get(candidate) or {}).get("spans")) deepest_verified = bool( spans and matched and not deeper and not invalid_candidates and not missing_candidate_markers) @@ -253,14 +278,16 @@ def verify(target_callable, device_kernel, capture_meta, trace_events, candidate "matched_kernel_calls": len(matched), "matched_kernel_names": sorted(set(matched)), "correlated_external_ids": len(related_external_ids), + "correlated_launch_correlations": len(selected_evidence.get("related_correlations") or ()), "candidate_targets_tested": installed_candidates, "live_candidate_targets": sorted( candidate for candidate, candidate_evidence in evidence.items() if candidate_evidence["matched"]), "deeper_live_candidates": sorted(deeper), "missing_candidate_markers": missing_candidate_markers, + "installed_but_never_live_candidates": unobserved_candidates, "deepest_verified": deepest_verified, - "evidence": "installed+live_nested_candidate_markers+torch_profiler_external_id", + "evidence": "installed+live_nested_candidate_markers+torch_profiler_external_id_or_launch_correlation", "failed": failed, } @@ -338,9 +365,16 @@ def main(argv=None): ] verdict["candidate_targets_tested"] = sorted( set.intersection(*tested_sets) if tested_sets else set()) + # Never-live has to hold on every capture process: one rank observing the candidate is enough + # to make it live, so this intersects rather than unions. + unobserved_sets = [ + set(item["installed_but_never_live_candidates"]) for item in all_verdicts + ] + verdict["installed_but_never_live_candidates"] = sorted( + set.intersection(*unobserved_sets) if unobserved_sets else set()) for field in ( "total_calls_observed", "target_marker_calls", "matched_kernel_calls", - "correlated_external_ids"): + "correlated_external_ids", "correlated_launch_correlations"): verdict[field] = sum(int(item.get(field) or 0) for item in all_verdicts) verdict["matched_kernel_names"] = sorted({ name for item in all_verdicts for name in item["matched_kernel_names"] diff --git a/e2e_workflow/scripts/tests/test_kernel_selection.py b/e2e_workflow/scripts/tests/test_kernel_selection.py index 13dd2793f..5627d77fc 100644 --- a/e2e_workflow/scripts/tests/test_kernel_selection.py +++ b/e2e_workflow/scripts/tests/test_kernel_selection.py @@ -85,6 +85,69 @@ def test_async_kernel_after_marker_is_linked_by_external_id(self): self.assertTrue(verdict["ok"], verdict) self.assertEqual(verdict["correlated_external_ids"], 1) + def test_triton_kernel_without_external_id_is_linked_by_launch_correlation(self): + # ROCm/kineto emits Triton device rows (hipModuleLaunchKernel) with only `correlation`; + # `External id` is absent, so the External-id bridge alone reports a false negative. + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 40}, + {"cat": "cuda_runtime", "name": "hipModuleLaunchKernel", "ph": "X", + "pid": 1, "tid": 2, "ts": 110, "dur": 5, "args": {"correlation": 25}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "pid": 2, "tid": 0, + "ts": 300, "dur": 20, "args": {"correlation": 25, "stream": 0}}, + ] + verdict = ks.verify(TARGET, KERNEL, self.meta(), events) + self.assertTrue(verdict["ok"], verdict) + self.assertEqual(verdict["matched_kernel_calls"], 1) + self.assertEqual(verdict["correlated_launch_correlations"], 1) + + def test_launch_correlation_outside_the_marker_span_is_not_enough(self): + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 40}, + {"cat": "cuda_runtime", "name": "hipModuleLaunchKernel", "ph": "X", + "pid": 1, "tid": 2, "ts": 500, "dur": 5, "args": {"correlation": 25}}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "pid": 2, "tid": 0, + "ts": 600, "dur": 20, "args": {"correlation": 25, "stream": 0}}, + ] + verdict = ks.verify(TARGET, KERNEL, self.meta(), events) + self.assertFalse(verdict["ok"]) + self.assertIn("device_kernel_not_under_target", verdict["failed"]) + + def test_correlation_ids_do_not_collide_across_merged_call_traces(self): + # Every per-call trace restarts `correlation` at 1, so an unprefixed merge would let call-2's + # kernel be attributed to call-1's in-span launch. + def one(marked, corr, kernel_ts): + evs = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "pid": 2, "tid": 0, + "ts": kernel_ts, "dur": 20, "args": {"correlation": corr}}, + ] + if marked: + evs += [ + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 40}, + {"cat": "cuda_runtime", "name": "hipModuleLaunchKernel", "ph": "X", + "pid": 1, "tid": 2, "ts": 110, "dur": 5, "args": {"correlation": corr}}, + ] + return {"traceEvents": evs} + + with tempfile.TemporaryDirectory() as tmp: + paths = [] + # call-1: marked launch, correlation 1. call-2: NO marker at all, correlation 1 again. + for i, doc in enumerate((one(True, 1, 300), one(False, 1, 900))): + p = os.path.join(tmp, "t%d.json" % i) + json.dump(doc, open(p, "w")) + paths.append(p) + merged = ks.merge_process_traces(paths) + verdict = ks.verify(TARGET, KERNEL, self.meta(), merged) + self.assertEqual(verdict["matched_kernel_calls"], 1, verdict) + def test_0802_outer_wrapper_fails_when_0720_launcher_is_marked(self): outer = "vllm.v1.attention.layer:unified_attention_with_output" inner = TARGET @@ -134,6 +197,25 @@ def test_installed_but_inactive_alternative_branch_does_not_fail(self): sorted([TARGET, alternative]), ) + def test_a_probed_candidate_that_never_fired_is_reported_not_silently_dropped(self): + # A marker rebinds one module attribute. A callable dispatched through `torch.ops.*`, or + # reached through an alias a caller imported before installation, runs with the wrapper + # bypassed and produces zero spans while being fully live. Reading that as "not deeper" + # would let a shallower seam collect `deepest_verified`, so the verdict has to say which + # candidates were probed and never observed, apart from the ones never probed at all. + invisible = "vllm.model_executor.layers.attention.attention:unified_attention_with_output" + unprobed = "vllm.attention:never_installed" + events = trace() + events.insert(1, { + "cat": "cpu_op", "name": ks.INSTALL_PREFIX + invisible, + "ph": "X", "pid": 1, "tid": 2, "ts": 92, "dur": 1, + }) + verdict = ks.verify( + TARGET, KERNEL, self.meta(), events, [TARGET, invisible, unprobed]) + self.assertEqual(verdict["installed_but_never_live_candidates"], [invisible]) + self.assertEqual(verdict["missing_candidate_markers"], [unprobed]) + self.assertNotIn(invisible, verdict["deeper_live_candidates"]) + def test_device_projected_annotation_does_not_invert_call_nesting(self): # The profiler re-emits each marker on the GPU timeline as `gpu_user_annotation`, where an # OUTER seam's short device span can land inside the INNER launcher's long one. Only the From c040738c12232dedab3db42f884572e777a61002 Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Thu, 20 Aug 2026 07:08:22 +0000 Subject: [PATCH 6/8] Cover the selection scripts, and canonicalize nested template arguments Both red L0 checks came from this branch's own new code. Coverage sat at 96.78% against a 97% gate: kernel_selection.py, seam_trace.py and the parse_profile.py additions landed with 82 uncovered statements between them, and --annotate -- the entry point that turns a Top-N doc into routable head candidates, exit code included -- had no CLI test at all. All three are now at 98-100% and the total is 98.43%. CodeQL flagged canonicalDeviceKernel's single greedy pass over `<.*>`. It really is lossy: the pass spans from the first '<' to the last '>', so `k(t)` loses the '(' that ends the name, and a nested `k>` leaves a delimiter behind. Balanced groups are now removed innermost-first until the symbol stops changing. canonical_kernel_name in kernel_selection.py is the same function on the Python side and had the same defect, so both move together -- a drift there would let a kernel pass one side of the contract and be refused by the other. Co-authored-by: Cursor --- e2e_workflow/e2e_workflow.js | 19 +- e2e_workflow/scripts/kernel_selection.py | 18 +- .../scripts/tests/test_kernel_selection.py | 123 ++++++++++++ .../scripts/tests/test_parse_profile.py | 183 ++++++++++++++++++ e2e_workflow/scripts/tests/test_seam_trace.py | 154 +++++++++++++++ 5 files changed, 493 insertions(+), 4 deletions(-) diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index 538e092b1..7cc88b60a 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -747,9 +747,22 @@ async function ensureFlydslGate() { // two machine fields separate and require runtime evidence before bake-off or authoring. const CALLABLE_SPEC_RX = /^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$/; const validCallableSpec = (s) => CALLABLE_SPEC_RX.test(String(s || '').trim()); -const canonicalDeviceKernel = (s) => String(s || '').trim().toLowerCase() - .replace(/\[clone[^\]]*\]/g, '').replace(/<.*>/g, '').split('(', 1)[0] - .split('::').pop().replace(/[^a-z0-9_]+/g, ''); +// Template arguments are removed innermost-first until the symbol stops changing. One greedy pass +// over `<.*>` spans from the FIRST '<' to the LAST '>', so `k(t)` loses the '(' that marks the +// end of the name; and a nested `k>` leaves the leftover delimiter behind. Removing only +// balanced groups, repeatedly, is what makes a templated symbol reduce to the same token as its bare +// spelling -- which is the whole job here, since a mismatch refuses a real head. +const stripTemplateArgs = (symbol) => { + let text = symbol; + for (let previous = null; previous !== text;) { + previous = text; + text = text.replace(/<[^<>]*>/g, ''); + } + return text; +}; +const canonicalDeviceKernel = (s) => stripTemplateArgs( + String(s || '').trim().toLowerCase().replace(/\[clone[^\]]*\]/g, '')) + .split('(', 1)[0].split('::').pop().replace(/[^a-z0-9_]+/g, ''); function kernelIdentitiesMatch(a, b) { const x = canonicalDeviceKernel(a), y = canonicalDeviceKernel(b); return !!x && !!y && (x === y || diff --git a/e2e_workflow/scripts/kernel_selection.py b/e2e_workflow/scripts/kernel_selection.py index 5fb59de00..a4d4e2cfa 100644 --- a/e2e_workflow/scripts/kernel_selection.py +++ b/e2e_workflow/scripts/kernel_selection.py @@ -67,11 +67,27 @@ def merge_process_traces(paths): return merged +def _strip_template_args(text): + """Remove balanced ``<...>`` groups innermost-first until the symbol stops changing. + + One greedy pass over ``<.*>`` spans from the FIRST ``<`` to the LAST ``>``, so ``k(t)`` + loses the ``(`` that marks the end of the name, and a nested ``k>`` leaves the leftover + delimiter behind. This must stay identical to ``canonicalDeviceKernel`` in e2e_workflow.js: the JS + gate and this verdict compare the same two symbols, and a canonicalization that drifted between + them would let a kernel pass one side and be refused by the other. + """ + previous = None + while previous != text: + previous = text + text = re.sub(r"<[^<>]*>", "", text) + return text + + def canonical_kernel_name(value): """Return a stable token for matching mangled/demangled kernel symbols.""" text = str(value or "").strip().lower() text = re.sub(r"\[clone[^\]]*\]", "", text) - text = re.sub(r"<.*>", "", text) + text = _strip_template_args(text) text = text.split("(", 1)[0] text = text.rsplit("::", 1)[-1] return re.sub(r"[^a-z0-9_]+", "", text) diff --git a/e2e_workflow/scripts/tests/test_kernel_selection.py b/e2e_workflow/scripts/tests/test_kernel_selection.py index 5627d77fc..0a699c3f0 100644 --- a/e2e_workflow/scripts/tests/test_kernel_selection.py +++ b/e2e_workflow/scripts/tests/test_kernel_selection.py @@ -52,6 +52,25 @@ def test_demangled_kernel_matches_profile_identity(self): KERNEL, "void vllm::kernel_paged_attention_2d(int)")) self.assertFalse(ks.kernel_matches(KERNEL, "unrelated_attention_kernel")) + def test_nested_template_arguments_reduce_to_the_bare_kernel_name(self): + """Demangled C++ symbols nest their template arguments, and the argument list can carry both + '::' and '(' -- the two delimiters the rest of the canonicalization splits on. Leaving any of + it behind produces a token that never matches the profile's own name for the same kernel.""" + for decorated in ( + "at::native::vectorized_elementwise_kernel<4, at::native::AddFunctor, " + "at::detail::Array >(int, float)", + "void ns::vectorized_elementwise_kernel>(void*)", + "vectorized_elementwise_kernel [clone .isra.0]", + ): + with self.subTest(decorated=decorated): + self.assertEqual(ks.canonical_kernel_name(decorated), + "vectorized_elementwise_kernel") + self.assertTrue(ks.kernel_matches("vectorized_elementwise_kernel", decorated)) + + def test_an_unbalanced_delimiter_does_not_survive_into_the_token(self): + self.assertEqual(ks.canonical_kernel_name("gemm_kernel"), "") + class TestSelectionVerdict(unittest.TestCase): def meta(self, calls=7, target=TARGET): @@ -417,5 +436,109 @@ def process_events(with_inner): sorted(verdict["live_candidate_targets"]), sorted([mid, inner])) +class TestTheVerdictRefusesMalformedInput(unittest.TestCase): + """These are the fail-closed edges. Each one is a way an extraction could arrive incomplete and + still be read as "nothing to check here", which is the vacuous pass the contract exists to stop.""" + + def meta(self, calls=7, target=TARGET): + module, attr = target.split(":", 1) + return {"module": module, "attr": attr, "total_calls_observed": calls} + + def test_a_prose_target_is_named_as_such_rather_than_probed(self): + verdict = ks.verify("the attention wrapper", KERNEL, self.meta(), trace()) + self.assertFalse(verdict["ok"]) + self.assertIn("invalid_target_callable", verdict["failed"]) + + def test_a_head_with_no_device_symbol_cannot_certify_anything(self): + verdict = ks.verify(TARGET, "", self.meta(), trace()) + self.assertFalse(verdict["ok"]) + self.assertIn("missing_device_kernel", verdict["failed"]) + + def test_one_malformed_candidate_sinks_the_whole_probe(self): + """The coverage check compares the declared candidate set against what was probed. A member + that cannot be probed at all must fail rather than quietly shrink the set being compared.""" + verdict = ks.verify(TARGET, KERNEL, self.meta(), trace(), + candidate_targets=["live_call_seam (see notes)"]) + self.assertFalse(verdict["ok"]) + self.assertIn("invalid_candidate_target", verdict["failed"]) + + def test_an_empty_kernel_name_never_matches_by_accident(self): + self.assertFalse(ks.kernel_matches("", KERNEL)) + self.assertFalse(ks.kernel_matches(KERNEL, "")) + self.assertFalse(ks.kernel_matches("<>", KERNEL)) + + +class TestSpansAreReadFromEitherTraceShape(unittest.TestCase): + def test_begin_end_pairs_bound_a_span_the_way_a_complete_event_does(self): + """Kineto writes a marker as a complete `X` event, but a trace that was cut short (or came + from a different exporter) carries the same span as a B/E pair. Reading only `X` would report + a live seam as never having run.""" + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "B", "pid": 1, "tid": 2, "ts": 100}, + {"cat": "cpu_op", "name": "launch", "ph": "X", "pid": 1, "tid": 2, + "ts": 110, "dur": 5, "args": {"External id": 7}}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "E", "pid": 1, "tid": 2, "ts": 200}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "ts": 300, "dur": 10, + "args": {"External id": 7}}, + ] + module, attr = TARGET.split(":", 1) + verdict = ks.verify( + TARGET, KERNEL, + {"module": module, "attr": attr, "total_calls_observed": 1}, events) + self.assertTrue(verdict["ok"], verdict) + self.assertEqual(verdict["matched_kernel_calls"], 1) + + def test_an_end_with_no_begin_is_dropped_instead_of_inventing_a_span(self): + spans = ks._complete_spans([ + {"cat": "cpu_op", "name": "m", "ph": "E", "pid": 1, "tid": 2, "ts": 200}, + ], "m") + self.assertEqual(spans, []) + + def test_an_event_with_no_timestamp_is_not_inside_any_span(self): + self.assertFalse(ks._within_any_span({"cat": "cpu_op"}, [(0.0, 10.0, {})])) + + def test_a_device_row_does_not_donate_its_own_id_to_the_launch_set(self): + """Kernel rows sit inside the marker span on the device timeline too. Harvesting ids from + them would make a kernel vouch for itself, so only host events may establish causality.""" + events = [ + {"cat": "cpu_op", "name": ks.INSTALL_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 90, "dur": 1}, + {"cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 100, "dur": 100}, + {"cat": "kernel", "name": KERNEL, "ph": "X", "pid": 1, "tid": 2, + "ts": 120, "dur": 10, "args": {"External id": 7}}, + ] + module, attr = TARGET.split(":", 1) + verdict = ks.verify( + TARGET, KERNEL, + {"module": module, "attr": attr, "total_calls_observed": 1}, events) + self.assertFalse(verdict["ok"]) + self.assertIn("device_kernel_not_under_target", verdict["failed"]) + + +class TestMergingProcessTraces(unittest.TestCase): + def test_a_trace_file_carrying_non_event_entries_is_merged_without_raising(self): + """`traceEvents` from a truncated export can contain nulls/strings. The verifier reads other + people's files, so a malformed entry must be skipped rather than abort every rank's merge.""" + with tempfile.TemporaryDirectory() as root: + path = os.path.join(root, "selection.pid-1.rank-0.call-1.json") + with open(path, "w") as fh: + json.dump({"traceEvents": [ + None, + "not an event", + {"cat": "cpu_op", "name": "m", "ph": "X", "pid": 1, "tid": 2, + "ts": 1, "dur": 1, "args": {"External id": 5, "correlation": 6}}, + ]}, fh) + merged = ks.merge_process_traces([path]) + self.assertEqual(len(merged), 1) + self.assertEqual(merged[0]["pid"], "trace-0:1") + self.assertEqual(merged[0]["args"]["External id"], "trace-0:5") + self.assertEqual(merged[0]["args"]["correlation"], "trace-0:6") + + if __name__ == "__main__": unittest.main() diff --git a/e2e_workflow/scripts/tests/test_parse_profile.py b/e2e_workflow/scripts/tests/test_parse_profile.py index 8b4931a78..cf6b34011 100644 --- a/e2e_workflow/scripts/tests/test_parse_profile.py +++ b/e2e_workflow/scripts/tests/test_parse_profile.py @@ -306,6 +306,30 @@ def test_c9_norm_key_collision_resolves_each_row_by_exact_name(self): self.assertEqual(stats["dispatcher_op"], 1) self.assertEqual(stats["unresolved"], 1) + def test_a_row_naming_nothing_the_profiler_saw_is_unresolved_not_assumed(self): + """The head track routes only gpu_kernel rows. A name the profile never contains has no + evidence at all, and defaulting it to "kernel" is exactly how an unverifiable head got in.""" + rows, stats = pp.annotate_rows( + [{"name": "a_kernel_nobody_profiled"}], + pp.merge_entity_evidence({"real_kernel": {"calls": 1, "total_us": 5.0, + "cat_counts": {"kernel": 1}}}), + "torch-trace") + self.assertEqual(rows[0]["entity_kind"], "unresolved") + self.assertEqual(rows[0]["entity_evidence"]["basis"], "name_not_found_in_profile") + self.assertEqual(stats["unresolved"], 1) + + def test_a_decorated_row_name_still_resolves_through_the_normalized_key(self): + """Top-N rows arrive with demangled/decorated names ('void ns::k(int)'). When the fold + is unambiguous that row IS the profiled kernel, and refusing it would drop a real head.""" + device = {"void ns::gemm_kernel(int)": {"calls": 3, "total_us": 9.0, + "cat_counts": {"kernel": 3}}} + rows, stats = pp.annotate_rows( + [{"name": "gemm_kernel"}], pp.merge_entity_evidence(device), "torch-trace") + self.assertEqual(rows[0]["entity_kind"], "gpu_kernel") + self.assertEqual(rows[0]["entity_evidence"]["matched_profiled_kernel"], + "void ns::gemm_kernel(int)") + self.assertEqual(stats["gpu_kernel"], 1) + def test_c9_collision_resolution_is_merge_order_independent(self): # The setdefault bug made the collision winner depend on which aggregate was merged first. # Reversing the merge input order must yield IDENTICAL kinds for every row: neither the @@ -385,6 +409,40 @@ def test_existing_device_row_is_not_duplicated_by_dispatcher_expansion(self): [row["name"] for row in rows].count("kernel_paged_attention_2d"), 1) self.assertEqual(sum(row["pct_gpu_time"] for row in rows), 20.0) + def test_a_dispatcher_whose_children_cost_nothing_is_left_alone(self): + """Expansion divides the row's Amdahl mass by the children's measured time. With no measured + time there is nothing to divide by, and inventing a split would be worse than not expanding.""" + row = {"name": "vllm::attention", "pct_gpu_time": 20.0, "total_ms": 10.0} + rows, records = pp.expand_dispatcher_rows([row], { + "vllm::attention": {"kernel_paged_attention_2d": {"calls": 4, "total_us": 0.0}}, + }) + self.assertEqual(rows, [row]) + self.assertEqual(records, []) + + def test_a_trace_that_cannot_be_read_yields_no_edges_rather_than_raising(self): + """The trace path is supplied by the caller and may be absent or truncated. parse_profile is + the step that TRIAGES a run; it must degrade to "no evidence", never take the run down.""" + self.assertEqual(pp.index_dispatch_edges(self._trace_file(raw="{not json")), {}) + self.assertEqual(pp.index_dispatch_edges("/nonexistent/trace.json"), {}) + + def test_sibling_spans_do_not_inherit_each_others_kernels(self): + """Ancestry is what attributes a kernel to an outer dispatcher. A span that already CLOSED is + not an ancestor, so a launch under the second sibling must not be credited to the first.""" + events = [ + {"cat": "cpu_op", "name": "outer", "pid": 1, "tid": 2, + "ts": 100, "dur": 200, "args": {"External id": 1}}, + {"cat": "cpu_op", "name": "first_child", "pid": 1, "tid": 2, + "ts": 110, "dur": 10, "args": {"External id": 2}}, + {"cat": "cpu_op", "name": "second_child", "pid": 1, "tid": 2, + "ts": 150, "dur": 10, "args": {"External id": 3}}, + {"cat": "kernel", "name": "second_kernel", "ts": 400, "dur": 5, + "args": {"External id": 3}}, + ] + edges = pp.index_dispatch_edges(self._trace_file(events)) + self.assertIn("second_kernel", edges["second_child"]) + self.assertIn("second_kernel", edges["outer"]) + self.assertNotIn("first_child", edges) + def test_exact_host_device_collision_is_not_expanded(self): row = {"name": "shared", "pct_gpu_time": 20.0} evidence = pp.merge_entity_evidence( @@ -1207,5 +1265,130 @@ def test_module_entrypoint_invokes_main(self): self.assertIn("# Profile Top-1", out.getvalue()) +# --------------------------------------------------------------------------- # +# host-side entity index + the --annotate CLI the architect's head rows go through +# --------------------------------------------------------------------------- # +class TestHostEntityIndex(_TmpMixin, unittest.TestCase): + """Host spans are deliberately kept OUT of the timing aggregate (a cpu_op's duration subsumes the + kernels it dispatched). They are indexed separately so --annotate can say "this claimed head is a + dispatcher" instead of the much weaker "not found".""" + + def test_host_spans_are_counted_by_category_and_device_rows_are_not(self): + idx = pp.index_host_entities(self._trace_file([ + {"cat": "cpu_op", "name": "vllm::attention", "dur": 10.0}, + {"cat": "cpu_op", "name": "vllm::attention", "dur": 5.0}, + {"cat": "user_annotation", "name": "step", "dur": 100.0}, + {"cat": "hip_runtime", "name": "hipLaunchKernel", "dur": 1.0}, + {"cat": "kernel", "name": "gemm_kernel", "dur": 80.0}, + "not an event", + ])) + self.assertEqual(idx["vllm::attention"], { + "calls": 2, "total_us": 15.0, "cat_counts": {"cpu_op": 2}}) + self.assertEqual(idx["step"]["cat_counts"], {"user_annotation": 1}) + self.assertIn("hipLaunchKernel", idx) + self.assertNotIn("gemm_kernel", idx) + + def test_an_unreadable_trace_indexes_nothing_rather_than_raising(self): + self.assertEqual(pp.index_host_entities("/nonexistent/trace.json"), {}) + self.assertEqual(pp.index_host_entities(self._trace_file(raw="{not json")), {}) + + +class TestAnnotateCli(_TmpMixin, unittest.TestCase): + """`--annotate` is how a Top-N doc becomes routable head candidates. It is the step that must + refuse a row it cannot prove is a GPU kernel, and its EXIT CODE is what the workflow reads.""" + + def _main(self, argv): + out, err = io.StringIO(), io.StringIO() + saved = sys.argv + sys.argv = ["parse_profile.py"] + argv + try: + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + with self.assertRaises(SystemExit) as caught: + pp.main() + finally: + sys.argv = saved + return caught.exception.code, out.getvalue(), err.getvalue() + + def _doc(self, top_kernels, name="profile_topN.json"): + return self._write(name, json.dumps({"top_kernels": top_kernels})) + + def test_a_dispatcher_row_is_replaced_by_the_kernels_it_launched(self): + trace = self._trace_file([ + {"cat": "cpu_op", "name": "vllm::attention", "pid": 1, "tid": 2, + "ts": 100, "dur": 100, "args": {"External id": 17}}, + {"cat": "kernel", "name": "kernel_paged_attention_2d", "pid": 1, "tid": 7, + "ts": 110, "dur": 80, "args": {"External id": 17}}, + ]) + doc = self._doc([{"name": "vllm::attention", "short_name": "attention", + "pct_gpu_time": 20.0, "total_ms": 10.0, "calls": 2}]) + out_path = os.path.join(self._dir(), "annotated.json") + code, stdout, stderr = self._main( + ["--torch-trace", trace, "--annotate", doc, "--annotate-out", out_path]) + + self.assertEqual(code, 0) + with open(out_path) as fh: + annotated = json.load(fh) + self.assertEqual(annotated["entity_kind_contract"], "v1") + self.assertEqual([r["name"] for r in annotated["top_kernels"]], + ["kernel_paged_attention_2d"]) + self.assertEqual([r["entity_kind"] for r in annotated["top_kernels"]], ["gpu_kernel"]) + self.assertEqual(annotated["entity_kind_counts"], {"gpu_kernel": 1}) + self.assertEqual(annotated["dispatcher_expansions"][0]["dispatcher"], "vllm::attention") + self.assertIn("gpu_kernel=1", stderr) + self.assertEqual(json.loads(stdout)["annotated"], 1) + with open(doc) as fh: # --annotate-out leaves the input untouched + self.assertNotIn("entity_kind_contract", json.load(fh)) + + def test_a_row_that_is_not_a_profiled_kernel_exits_non_zero(self): + """The workflow branches on this exit code: a claimed head the profiler never dispatched must + stop the head track rather than be routed as if it had been measured.""" + trace = self._trace_file([ + {"cat": "kernel", "name": "gemm_kernel", "ts": 10, "dur": 80, + "args": {"External id": 1}}, + ]) + doc = self._doc([{"name": "gemm_kernel", "pct_gpu_time": 20.0}, + {"name": "live_call_seam", "pct_gpu_time": 5.0}]) + code, stdout, stderr = self._main(["--torch-trace", trace, "--annotate", doc]) + + self.assertEqual(code, 1) + with open(doc) as fh: # no --annotate-out: annotated in place + annotated = json.load(fh) + kinds = {r["name"]: r["entity_kind"] for r in annotated["top_kernels"]} + self.assertEqual(kinds, {"gemm_kernel": "gpu_kernel", "live_call_seam": "unresolved"}) + self.assertEqual(annotated["entity_kind_counts"], {"gpu_kernel": 1, "unresolved": 1}) + self.assertIn("unresolved=1", stderr) + self.assertEqual(json.loads(stdout)["entity_kind_counts"]["unresolved"], 1) + + def test_a_host_only_row_is_labelled_a_dispatcher_rather_than_reported_missing(self): + """A dispatcher that launched nothing this trace can attribute is still IDENTIFIED, which is + a successful classification -- the exit code marks rows with no evidence at all. Refusing a + dispatcher from the head track is admitHeads' job, and it needs this label to do it.""" + trace = self._trace_file([ + {"cat": "cpu_op", "name": "vllm::opaque", "pid": 1, "tid": 2, + "ts": 100, "dur": 100}, + {"cat": "kernel", "name": "gemm_kernel", "ts": 10, "dur": 80}, + ]) + doc = self._doc([{"name": "vllm::opaque", "pct_gpu_time": 20.0}]) + code, _, stderr = self._main(["--torch-trace", trace, "--annotate", doc]) + + self.assertEqual(code, 0) + with open(doc) as fh: + annotated = json.load(fh) + self.assertEqual(annotated["top_kernels"][0]["entity_kind"], "dispatcher_op") + self.assertEqual(annotated["entity_kind_counts"], {"dispatcher_op": 1}) + self.assertIn("dispatcher_op=1", stderr) + + def test_annotating_with_rocprof_evidence_needs_no_torch_trace(self): + doc = self._doc([{"name": "gemm_kernel", "pct_gpu_time": 20.0}]) + code, _, _ = self._main( + ["--rocprof-dir", self._rocprof_dir(), "--annotate", doc]) + + self.assertEqual(code, 0) + with open(doc) as fh: + annotated = json.load(fh) + self.assertEqual(annotated["top_kernels"][0]["entity_kind"], "gpu_kernel") + self.assertEqual(annotated["dispatcher_expansions"], []) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/e2e_workflow/scripts/tests/test_seam_trace.py b/e2e_workflow/scripts/tests/test_seam_trace.py index 2f227ca3e..45a5357ca 100644 --- a/e2e_workflow/scripts/tests/test_seam_trace.py +++ b/e2e_workflow/scripts/tests/test_seam_trace.py @@ -145,6 +145,160 @@ def test_process_local_call_traces_do_not_overwrite(self): self.assertIn(".call-1.json", traces[0]) self.assertIn(".call-2.json", traces[1]) + def test_installing_the_same_target_twice_keeps_the_first_wrapper(self): + """A candidate may be named twice (once by the architect, once by the extractor). Re-wrapping + would nest a marker inside itself and double-count the seam's calls.""" + st.install("_seam_trace_fixture:outer") + wrapper = self.module.outer + st.install("_seam_trace_fixture:outer") + self.assertIs(self.module.outer, wrapper) + self.assertEqual(self.module.outer(3), 8) + markers = [value for action, value in self.events + if action == "enter" and value.startswith(st.MARKER_PREFIX)] + self.assertEqual(markers, [st.MARKER_PREFIX + "_seam_trace_fixture:outer"]) + + def test_a_callable_whose_signature_cannot_be_read_is_still_marked(self): + """`inspect.signature` refuses some C callables reached through a partial. The seam is still a + pure-Python object to replace, so the probe must install rather than abort the whole capture.""" + import functools + + self.module.opaque = functools.partial(range) + st.install("_seam_trace_fixture:opaque") + self.assertEqual(list(self.module.opaque(3)), [0, 1, 2]) + names = [value for action, value in self.events if action == "enter"] + self.assertIn(st.MARKER_PREFIX + "_seam_trace_fixture:opaque", names) + + def test_the_seam_still_runs_when_the_process_has_no_torch(self): + """Markers are installed from sitecustomize, which runs before the server imports torch. A + seam that raised (or silently skipped the call) there would break the server being profiled.""" + st.install("_seam_trace_fixture:outer") + sys.modules["torch"] = None # makes `import torch` raise, as it does before torch is built + self.assertEqual(self.module.outer(3), 8) + self.assertEqual([value for action, value in self.events if action == "enter"], []) + + +class TestTracePathIsProcessLocal(unittest.TestCase): + def setUp(self): + for key in ("GEAK_SELECTION_TRACE", "GEAK_SELECTION_TRACE_UNIQUE", "RANK"): + self.addCleanup(os.environ.pop, key, None) + os.environ.pop(key, None) + + def test_no_trace_env_means_no_path_and_no_profile(self): + """The marker overlay is also loaded by processes that are not the capture (a tuner, a + one-shot import check). With no destination they must not start a profiler at all.""" + self.assertEqual(st._trace_path(1), "") + self.assertFalse(st._start_profile()) + + def test_an_explicit_template_is_filled_with_pid_and_rank(self): + """A TP deployment runs one marked process per rank. Whether the operator places {pid}/{rank} + by hand or leaves it to the default suffix, two ranks must never be handed one filename.""" + os.environ["GEAK_SELECTION_TRACE"] = "/tmp/sel-{pid}-{rank}.json" + os.environ["RANK"] = "3" + self.assertEqual(st._trace_path(7), f"/tmp/sel-{os.getpid()}-3.call-7.json") + + def test_a_path_with_no_placeholders_gets_the_pid_and_rank_appended(self): + os.environ["GEAK_SELECTION_TRACE"] = "/tmp/sel.json" + os.environ["RANK"] = "3" + self.assertEqual(st._trace_path(7), f"/tmp/sel.pid-{os.getpid()}.rank-3.call-7.json") + + def test_the_unique_opt_out_takes_the_path_verbatim(self): + """The escape hatch for a single-process debug capture: write exactly where I said. It is not + safe for a multi-rank run, which is why per-process naming is what happens by default.""" + os.environ["GEAK_SELECTION_TRACE"] = "/tmp/sel.json" + os.environ["GEAK_SELECTION_TRACE_UNIQUE"] = "0" + self.assertEqual(st._trace_path(7), "/tmp/sel.json") + + def test_the_rank_comes_from_the_launcher_env_when_it_declares_one(self): + os.environ["RANK"] = "2" + self.assertEqual(st._rank(), "2") + + def test_an_unranked_process_is_still_named_rather_than_dropped(self): + self.assertEqual(st._rank(), "unknown") + + +class TestProfileLifecycleFailsSoft(unittest.TestCase): + """Every failure here happens inside the SERVER under measurement. A probe that raises would + take the capture down with it, so each path degrades to "no trace" plus a stderr line.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + os.environ["GEAK_SELECTION_TRACE"] = os.path.join(self.tmp.name, "selection.json") + os.environ["GEAK_SELECTION_TRACE_UNIQUE"] = "0" + self.addCleanup(os.environ.pop, "GEAK_SELECTION_TRACE", None) + self.addCleanup(os.environ.pop, "GEAK_SELECTION_TRACE_UNIQUE", None) + self.addCleanup(os.environ.pop, "GEAK_SELECTION_PROFILE_CALLS", None) + self.saved_torch = sys.modules.get("torch") + self.addCleanup(self._restore_torch) + st._PROFILE.update(active=False, done=False, owner=None, profiler=None, + active_calls=0, root_calls=0, out="", trace_index=0, + atexit_registered=False) + + def _restore_torch(self): + if self.saved_torch is None: + sys.modules.pop("torch", None) + else: + sys.modules["torch"] = self.saved_torch + + def _install_torch(self, profile): + torch = types.ModuleType("torch") + torch.profiler = types.SimpleNamespace( + ProfilerActivity=types.SimpleNamespace(CPU="cpu"), + profile=profile, + record_function=lambda name: _Context([], name), + ) + sys.modules["torch"] = torch + + def test_a_profiler_that_refuses_to_start_disables_further_attempts(self): + def refuse(activities): + raise RuntimeError("no profiling on this device") + + self._install_torch(refuse) + self.assertFalse(st._start_profile()) + self.assertTrue(st._PROFILE["done"]) + self.assertFalse(st._start_profile()) + + def test_a_failed_export_does_not_propagate_into_the_served_call(self): + events = [] + + class _Unexportable(_Profiler): + def export_chrome_trace(self, path): + raise OSError("disk full") + + self._install_torch(lambda activities: _Unexportable(events)) + self.assertTrue(st._start_profile()) + st._finish_profile() + self.assertFalse(st._PROFILE["active"]) + + def test_finishing_a_profile_that_never_started_is_a_no_op(self): + st._finish_profile() + self.assertEqual(st._PROFILE["trace_index"], 0) + + def test_the_call_budget_stops_the_probe_rather_than_tracing_the_whole_run(self): + """An unbounded probe on a serving process writes a trace per root call for the life of the + server. The budget is what keeps a capture window bounded.""" + os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "1" + self._install_torch(lambda activities: _Profiler([])) + self.assertTrue(st._start_profile()) + st._finish_profile() + self.assertTrue(st._PROFILE["done"]) + self.assertFalse(st._start_profile()) + + +class TestWrappableRefusesWhatItCannotPreserve(unittest.TestCase): + def test_a_builtin_is_refused_because_a_python_wrapper_changes_its_identity(self): + self.assertFalse(st._wrappable(len)) + + def test_a_plain_callable_object_is_accepted(self): + class Seam: + def __call__(self): + return 1 + + self.assertTrue(st._wrappable(Seam())) + + def test_a_non_callable_attribute_is_refused(self): + self.assertFalse(st._wrappable(object())) + if __name__ == "__main__": unittest.main() From cc3008e7854fc788856eab1633a1be5abb8480e0 Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Thu, 20 Aug 2026 13:41:04 +0000 Subject: [PATCH 7/8] Identify a kernel by its own name on real ROCm symbols --- .github/workflows/ci-l0-checks.yml | 8 +- e2e_workflow/e2e_workflow.js | 73 ++++++-- e2e_workflow/scripts/kernel_selection.py | 118 +++++++++--- e2e_workflow/scripts/parse_profile.py | 12 +- e2e_workflow/scripts/seam_trace.py | 21 ++- .../test_kernel_canonicalization_parity.js | 59 ++++++ .../scripts/tests/kernel_symbols.json | 145 ++++++++++++++ .../scripts/tests/test_kernel_selection.py | 177 +++++++++++++++++- .../scripts/tests/test_parse_profile.py | 13 ++ e2e_workflow/scripts/tests/test_seam_trace.py | 75 +++++++- 10 files changed, 649 insertions(+), 52 deletions(-) create mode 100644 e2e_workflow/scripts/test_kernel_canonicalization_parity.js create mode 100644 e2e_workflow/scripts/tests/kernel_symbols.json diff --git a/.github/workflows/ci-l0-checks.yml b/.github/workflows/ci-l0-checks.yml index 9a370ac07..179deccc1 100644 --- a/.github/workflows/ci-l0-checks.yml +++ b/.github/workflows/ci-l0-checks.yml @@ -214,7 +214,7 @@ jobs: /tmp/gitleaks dir . --no-banner --redact --config .gitleaks.toml node-regression: - name: Node regression (expert_skills) + name: Node regression (expert_skills, kernel canonicalization) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -225,6 +225,12 @@ jobs: # Pure node (fs/path only) — no npm install needed. - name: expert_skills OFF-identical regression run: node e2e_workflow/scripts/test_expert_skills_off_identical.js + # The JS gate and the Python selection verdict canonicalize the same kernel symbols. Only this + # job can run the JS half, so without it the two implementations are pinned on the Python side + # alone and are free to drift — which lets a kernel pass one side and be refused by the other. + # The Python half is TestTheSharedFixtureHoldsOnThisSide in tests/test_kernel_selection.py. + - name: kernel canonicalization JS/Python parity + run: node e2e_workflow/scripts/test_kernel_canonicalization_parity.js dry-run: name: run_e2e dry-run mapping diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index 7cc88b60a..fd0700e07 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -747,27 +747,74 @@ async function ensureFlydslGate() { // two machine fields separate and require runtime evidence before bake-off or authoring. const CALLABLE_SPEC_RX = /^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$/; const validCallableSpec = (s) => CALLABLE_SPEC_RX.test(String(s || '').trim()); -// Template arguments are removed innermost-first until the symbol stops changing. One greedy pass -// over `<.*>` spans from the FIRST '<' to the LAST '>', so `k(t)` loses the '(' that marks the -// end of the name; and a nested `k>` leaves the leftover delimiter behind. Removing only -// balanced groups, repeatedly, is what makes a templated symbol reduce to the same token as its bare -// spelling -- which is the whole job here, since a mismatch refuses a real head. -const stripTemplateArgs = (symbol) => { +// parse_profile.short_name truncates display names at 60 chars; a declared name that hit the limit +// is our own doing, so a prefix match at it is accepted rather than read as a mismatch. +const SHORT_NAME_LIMIT = 60; +// Balanced groups are removed innermost-first until the symbol stops changing. One greedy pass +// spans from the FIRST opener to the LAST closer, so `k(t)` loses the '(' that marks the end +// of the name, and a nested `k>` leaves the leftover delimiter behind. +const stripBalanced = (symbol, opener, closer) => { + const esc = (c) => c.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const rx = new RegExp(`${esc(opener)}[^${esc(opener)}${esc(closer)}]*${esc(closer)}`, 'g'); let text = symbol; for (let previous = null; previous !== text;) { previous = text; - text = text.replace(/<[^<>]*>/g, ''); + text = text.replace(rx, ' '); } return text; }; -const canonicalDeviceKernel = (s) => stripTemplateArgs( - String(s || '').trim().toLowerCase().replace(/\[clone[^\]]*\]/g, '')) - .split('(', 1)[0].split('::').pop().replace(/[^a-z0-9_]+/g, ''); +// Must stay identical to canonical_kernel_name in kernel_selection.py -- the JS gate and the Python +// verdict compare the same two symbols, so any drift lets a kernel pass one side and fail the other. +// Parentheses are stripped as balanced groups rather than by cutting at the first '(': ROCm spells +// the unnamed namespace `(anonymous namespace)`, which puts a parenthesis BEFORE the kernel, so +// cutting there collapsed every such symbol to the return type `void`, and two unrelated kernels +// that both collapsed to `void` then certified each other. Taking the last whitespace-separated +// token drops that return type without fusing it into names that carry no namespace. +function canonicalDeviceKernel(s) { + const stripped = stripBalanced( + stripBalanced(String(s || '').replace(/\[clone[^\]]*\]/gi, ''), '<', '>'), '(', ')'); + // Whatever opener is left never closes, so the symbol was cut off inside it -- profile artifacts + // store kernel names elided mid-template. The name is what precedes that opener; taking the last + // whitespace token instead lifts a fragment out of the template arguments and states it with the + // same confidence as a real name, which then matches the wrong kernel rather than refusing. + const parts = stripped.split(/[<(]/)[0].split('::').pop().split(/\s+/).filter(Boolean); + return parts.length ? parts[parts.length - 1].toLowerCase().replace(/[^a-z0-9_]+/g, '') : ''; +} +// A name that hit the display limit ends mid-token, so no word boundary can follow it. Tested both +// ways because either side may be the shortened one: heads carry a short_name while traces carry +// the full symbol, and which is which is not fixed. +const truncatedPrefix = (needle, haystack) => + needle.length >= SHORT_NAME_LIMIT && haystack.startsWith(needle); +// [arguments, closed] for the symbol's first `<...>`. Reading only what follows the `<` keeps this +// independent of the return type and namespace, which one side routinely spells and the other does +// not. `closed` is false when the symbol was cut off inside the template -- profile artifacts elide +// long names, and only the visible part of an elided argument list can be held against anything. +// Separators become `_` rather than vanishing, so `<128, 4, ...>` cannot read as a prefix of +// `<128, 48, ...>`. +function templateArguments(value) { + const text = String(value || ''), start = text.indexOf('<'); + const fold = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '_'); + if (start < 0) return ['', true]; + let depth = 0; + for (let i = start; i < text.length; i++) { + if (text[i] === '<') depth++; + else if (text[i] === '>' && --depth === 0) return [fold(text.slice(start + 1, i)), true]; + } + return [fold(text.slice(start + 1)), false]; +} function kernelIdentitiesMatch(a, b) { const x = canonicalDeviceKernel(a), y = canonicalDeviceKernel(b); - return !!x && !!y && (x === y || - (x.length >= 6 && new RegExp(`(?:^|_)${x.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:$|_)`).test(y)) || - (y.length >= 6 && new RegExp(`(?:^|_)${y.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:$|_)`).test(x))); + if (!x || !y) return false; + if (x !== y && !truncatedPrefix(x, y) && !truncatedPrefix(y, x)) return false; + // The base token deliberately drops template arguments so a bare declared name can match its + // decorated spelling. When BOTH sides carry them the information is present on both, and ignoring + // it certifies the wrong kernel: one capture here held 20 distinct kernels named + // at::native::vectorized_elementwise_kernel, separated only by their functor. + const [xa, xc] = templateArguments(a), [ya, yc] = templateArguments(b); + if (!xa || !ya) return true; + if (xc && yc) return xa === ya; + // An elided list still has to agree as far as both sides actually spell it out. + return xa.startsWith(ya) || ya.startsWith(xa); } function requiredDeviceKernel(h) { if (!h || h.entity_kind !== 'gpu_kernel') return ''; diff --git a/e2e_workflow/scripts/kernel_selection.py b/e2e_workflow/scripts/kernel_selection.py index a4d4e2cfa..2e7f3d82c 100644 --- a/e2e_workflow/scripts/kernel_selection.py +++ b/e2e_workflow/scripts/kernel_selection.py @@ -67,30 +67,85 @@ def merge_process_traces(paths): return merged -def _strip_template_args(text): - """Remove balanced ``<...>`` groups innermost-first until the symbol stops changing. - - One greedy pass over ``<.*>`` spans from the FIRST ``<`` to the LAST ``>``, so ``k(t)`` - loses the ``(`` that marks the end of the name, and a nested ``k>`` leaves the leftover - delimiter behind. This must stay identical to ``canonicalDeviceKernel`` in e2e_workflow.js: the JS - gate and this verdict compare the same two symbols, and a canonicalization that drifted between - them would let a kernel pass one side and be refused by the other. +# parse_profile.short_name truncates display names here. A declared name that hit the limit is our +# own doing, so the matcher accepts a prefix at it rather than reading the truncation as a mismatch. +SHORT_NAME_LIMIT = 60 + + +def _strip_balanced(text, opener, closer): + """Remove balanced ``opener...closer`` groups innermost-first until the symbol stops changing. + + A single greedy pass spans from the FIRST opener to the LAST closer, so ``k(t)`` loses the + ``(`` that ends the name and a nested ``k>`` leaves a stray delimiter behind. Removing + the innermost group repeatedly is the only way to survive both. """ + pattern = re.compile( + re.escape(opener) + "[^" + re.escape(opener + closer) + "]*" + re.escape(closer)) previous = None while previous != text: previous = text - text = re.sub(r"<[^<>]*>", "", text) + text = pattern.sub(" ", text) return text def canonical_kernel_name(value): - """Return a stable token for matching mangled/demangled kernel symbols.""" - text = str(value or "").strip().lower() - text = re.sub(r"\[clone[^\]]*\]", "", text) - text = _strip_template_args(text) - text = text.split("(", 1)[0] - text = text.rsplit("::", 1)[-1] - return re.sub(r"[^a-z0-9_]+", "", text) + """Return a stable token for matching mangled/demangled kernel symbols. + + This must stay identical to ``canonicalDeviceKernel`` in e2e_workflow.js: the JS gate and this + verdict compare the same two symbols, and a canonicalization that drifted between them would let + a kernel pass one side and be refused by the other. + + Parentheses are stripped as balanced groups rather than by cutting at the first ``(``. ROCm names + the unnamed namespace ``(anonymous namespace)``, which puts a parenthesis *before* the kernel, so + cutting there collapsed every such symbol to the return type ``void`` -- and two unrelated + kernels that both collapse to ``void`` then certified each other. Taking the last whitespace- + separated token drops that return type without fusing it into names that carry no namespace. + """ + text = re.sub(r"\[clone[^\]]*\]", "", str(value or ""), flags=re.IGNORECASE) + text = _strip_balanced(text, "<", ">") + text = _strip_balanced(text, "(", ")") + # Whatever opener is left never closes, so the symbol was cut off inside it -- profile artifacts + # store kernel names elided mid-template. The name is what precedes that opener; taking the last + # whitespace token instead lifts a fragment out of the template arguments and states it with the + # same confidence as a real name ('...elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k' + # answered 'gpu_k'), which then matches the wrong kernel rather than refusing to answer. + text = re.split(r"[<(]", text, 1)[0] + text = text.split("::")[-1].split() + return re.sub(r"[^a-z0-9_]+", "", text[-1].lower()) if text else "" + + +def _truncated_prefix(needle, haystack): + """True when ``needle`` is ``haystack`` cut at the display limit. + + A name that hit the limit ends mid-token, so no word boundary can follow it. Tested both ways + because either side may be the shortened one: heads carry a short_name while traces carry the + full symbol, and which is which is not fixed. + """ + return len(needle) >= SHORT_NAME_LIMIT and haystack.startswith(needle) + + +def _template_arguments(value): + """``(arguments, closed)`` for the symbol's first ``<...>``. + + Reading only what follows the ``<`` keeps this independent of the return type and namespace, + which one side routinely spells and the other does not. ``closed`` is False when the symbol was + cut off inside the template -- profile artifacts elide long names, and only the visible part of + an elided argument list can be held against anything. Separators become ``_`` rather than + vanishing, so ``<128, 4, ...>`` cannot read as a prefix of ``<128, 48, ...>``. + """ + text = str(value or "") + start = text.find("<") + if start < 0: + return "", True + depth = 0 + for index in range(start, len(text)): + if text[index] == "<": + depth += 1 + elif text[index] == ">": + depth -= 1 + if depth == 0: + return re.sub(r"[^a-z0-9]+", "_", text[start + 1:index].lower()), True + return re.sub(r"[^a-z0-9]+", "_", text[start + 1:].lower()), False def kernel_matches(expected, observed): @@ -98,10 +153,20 @@ def kernel_matches(expected, observed): got = canonical_kernel_name(observed) if not want or not got: return False - return want == got or ( - len(want) >= 6 - and re.search(r"(?:^|_)" + re.escape(want) + r"(?:$|_)", got) is not None - ) + if want != got and not _truncated_prefix(want, got) and not _truncated_prefix(got, want): + return False + # The base token deliberately drops template arguments so a bare declared name can match its + # decorated spelling. When BOTH sides carry them the information is present on both, and ignoring + # it certifies the wrong kernel: one capture here held 20 distinct kernels named + # at::native::vectorized_elementwise_kernel, separated only by their functor. + want_args, want_closed = _template_arguments(expected) + got_args, got_closed = _template_arguments(observed) + if not want_args or not got_args: + return True + if want_closed and got_closed: + return want_args == got_args + # An elided list still has to agree as far as both sides actually spell it out. + return want_args.startswith(got_args) or got_args.startswith(want_args) def _device_projection(event): @@ -348,7 +413,10 @@ def main(argv=None): verify(args.target, args.device_kernel, meta, merge_process_traces(matching_traces), args.candidate_target), )) - selected_meta_path, selected_paths, verdict = max( + # The strongest single process seeds the verdict shape, but every field that survives below is + # recomputed across ALL attempts. Only the meta path stays process-specific, so it is named for + # the one process it describes rather than presented as the run's selection. + best_process_meta_file, _, verdict = max( attempts, key=lambda item: ( item[2]["matched_kernel_calls"], @@ -406,10 +474,10 @@ def main(argv=None): } for meta_path, paths, item in attempts ] - selected_paths = [path for _, paths, _ in attempts for path in paths] - verdict["capture_meta_file"] = selected_meta_path - verdict["trace_file"] = selected_paths[0] if len(selected_paths) == 1 else "" - verdict["trace_files"] = selected_paths + all_paths = [path for _, paths, _ in attempts for path in paths] + verdict["best_process_meta_file"] = best_process_meta_file + verdict["trace_file"] = all_paths[0] if len(all_paths) == 1 else "" + verdict["trace_files"] = all_paths verdict["trace_files_considered"] = trace_paths payload = json.dumps(verdict, indent=2) if args.out: diff --git a/e2e_workflow/scripts/parse_profile.py b/e2e_workflow/scripts/parse_profile.py index ecc617940..ab8545c22 100644 --- a/e2e_workflow/scripts/parse_profile.py +++ b/e2e_workflow/scripts/parse_profile.py @@ -87,16 +87,26 @@ def classify(name): return "other", "unknown", True, "Unclassified — inspect source to route." +SHORT_NAME_LIMIT = 60 + + def short_name(name): """Best-effort readable short name from a mangled C++/triton symbol.""" n = name # drop leading 'void ' and template/return noise n = re.sub(r"^void\s+", "", n) + # A symbol from an unnamed namespace opens with '(' where the identifier should be, so the + # '[\w:]+' probe below finds nothing and the whole signature falls through as the "short" name + # -- ROCm's '(anonymous namespace)::kda_packed_decode_kernel<...>(...)' used to shorten to the + # trailing parameter type rather than the kernel. + n = re.sub(r"\(anonymous namespace\)", "", n).lstrip(": ") # take the first identifier-ish token before '(' or '<' m = re.match(r"[\w:]+", n) base = m.group(0) if m else n base = base.split("::")[-1] - return base[:60] + # Truncation is for display only. kernel_selection.canonical_kernel_name knows about this limit + # and tolerates a prefix match at it, because mangled and Tensile symbols routinely exceed it. + return base[:SHORT_NAME_LIMIT] # --------------------------------------------------------------------------- # diff --git a/e2e_workflow/scripts/seam_trace.py b/e2e_workflow/scripts/seam_trace.py index 28859a6a3..5dd1acb9d 100644 --- a/e2e_workflow/scripts/seam_trace.py +++ b/e2e_workflow/scripts/seam_trace.py @@ -47,7 +47,11 @@ def _trace_path(trace_index): root, ext = os.path.splitext(template) path = f"{root}.pid-{pid}.rank-{_rank()}{ext or '.json'}" if os.environ.get("GEAK_SELECTION_TRACE_UNIQUE", "1") == "0": - return template + # UNIQUE=0 opts out of the per-CALL suffix only. The per-PROCESS part has to stay: returning + # the raw template leaves a literal '{pid}' in the name, and kernel_selection pairs a capture + # to its trace by matching '.pid-' -- without it every rank races on one path and the + # verdict degrades to capture_process_trace_missing. + return path root, ext = os.path.splitext(path) return f"{root}.call-{trace_index}{ext or '.json'}" @@ -73,6 +77,7 @@ def _start_profile(): if not out: return False ident = threading.get_ident() + profiler = None try: import torch activities = [torch.profiler.ProfilerActivity.CPU] @@ -80,16 +85,26 @@ def _start_profile(): activities.append(torch.profiler.ProfilerActivity.CUDA) profiler = torch.profiler.profile(activities=activities) profiler.__enter__() + _record_install_markers() _PROFILE.update(active=True, owner=ident, profiler=profiler, out=out, active_calls=0, root_calls=0) - _record_install_markers() if not _PROFILE["atexit_registered"]: atexit.register(_finish_profile) _PROFILE["atexit_registered"] = True return True except Exception as exc: - _PROFILE["done"] = True + # Anything that fails after __enter__ has to close the profiler here. _finish_profile + # returns early once done is set, so a profiler left entered on this path would stay + # entered -- and collecting -- for the life of the process. + _PROFILE.update(active=False, done=True, owner=None, profiler=None) sys.stderr.write(f"[seam_trace] profiler start failed: {exc!r}\n") + if profiler is not None: + try: + profiler.__exit__(None, None, None) + except Exception as close_exc: + # It stays entered, and collecting, for the life of the server. Nothing here can + # undo that, but an unexplained slowdown later is worth one line now. + sys.stderr.write(f"[seam_trace] profiler close failed: {close_exc!r}\n") return False diff --git a/e2e_workflow/scripts/test_kernel_canonicalization_parity.js b/e2e_workflow/scripts/test_kernel_canonicalization_parity.js new file mode 100644 index 000000000..8a9a76138 --- /dev/null +++ b/e2e_workflow/scripts/test_kernel_canonicalization_parity.js @@ -0,0 +1,59 @@ +#!/usr/bin/env node +// Regression guard for kernel symbol canonicalization (no GPU, no model needed). +// +// Invariant under test: canonicalDeviceKernel / kernelIdentitiesMatch in e2e_workflow.js agree with +// canonical_kernel_name / kernel_matches in scripts/kernel_selection.py. The JS gate and the Python +// verdict compare the SAME two symbols, so any drift between them lets a kernel pass one side and be +// refused by the other. Both sides are pinned to tests/kernel_symbols.json; the Python half of this +// is TestTheSharedFixtureHoldsOnThisSide in tests/test_kernel_selection.py. +// +// The functions are extracted from the real workflow source rather than reimplemented here -- a copy +// would pass this test while the shipped code drifted, which is the failure it exists to catch. +// +// Run: node e2e_workflow/scripts/test_kernel_canonicalization_parity.js +'use strict'; +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..', '..'); // .../GEAK +const WORKFLOW = path.join(ROOT, 'e2e_workflow', 'e2e_workflow.js'); +const FIXTURE = path.join(ROOT, 'e2e_workflow', 'scripts', 'tests', 'kernel_symbols.json'); + +let failures = 0; +const ok = (cond, msg) => { if (!cond) { console.error(' FAIL:', msg); failures++; } else console.log(' ok:', msg); }; + +const src = fs.readFileSync(WORKFLOW, 'utf8'); +const start = src.indexOf('const SHORT_NAME_LIMIT'); +const end = src.indexOf('function requiredDeviceKernel'); +ok(start !== -1 && end !== -1 && start < end, 'canonicalization block located in e2e_workflow.js'); +if (failures) process.exit(1); + +const build = new Function( + `${src.slice(start, end)}\nreturn { canonicalDeviceKernel, kernelIdentitiesMatch, SHORT_NAME_LIMIT };`); +const { canonicalDeviceKernel, kernelIdentitiesMatch, SHORT_NAME_LIMIT } = build(); + +const fixture = JSON.parse(fs.readFileSync(FIXTURE, 'utf8')); + +console.log('\n# canonical tokens'); +for (const c of fixture.canonical) { + const got = canonicalDeviceKernel(c.symbol); + ok(got === c.token, `${JSON.stringify(c.symbol.slice(0, 56))} -> ${JSON.stringify(c.token)}` + + (got === c.token ? '' : ` (got ${JSON.stringify(got)})`)); +} + +console.log('\n# match verdicts'); +for (const c of fixture.matches) { + const forward = kernelIdentitiesMatch(c.a, c.b), back = kernelIdentitiesMatch(c.b, c.a); + ok(forward === c.match && back === c.match, + `${c.why} (got ${forward}/${back}, want ${c.match})`); +} + +console.log('\n# constants shared with the Python side'); +ok(SHORT_NAME_LIMIT === 60, 'SHORT_NAME_LIMIT matches parse_profile.SHORT_NAME_LIMIT'); + +// The fixture is only worth anything while it still carries symbols that real captures produced. +const real = fixture.canonical.filter((c) => c.real).length; +ok(real >= 5, `fixture keeps ${real} symbols taken verbatim from ROCm captures`); + +console.log(failures ? `\nFAILED (${failures})` : '\nPASS'); +process.exit(failures ? 1 : 0); diff --git a/e2e_workflow/scripts/tests/kernel_symbols.json b/e2e_workflow/scripts/tests/kernel_symbols.json new file mode 100644 index 000000000..2d90e55f2 --- /dev/null +++ b/e2e_workflow/scripts/tests/kernel_symbols.json @@ -0,0 +1,145 @@ +{ + "_comment": [ + "Shared fixture for kernel symbol canonicalization. Two implementations must agree on it:", + " - canonical_kernel_name / kernel_matches in e2e_workflow/scripts/kernel_selection.py", + " - canonicalDeviceKernel / kernelIdentitiesMatch in e2e_workflow/e2e_workflow.js", + "The JS gate and the Python verdict compare the SAME two symbols, so any drift between them lets", + "a kernel pass one side and be refused by the other. Asserted from Python by", + "tests/test_kernel_selection.py and from node by scripts/test_kernel_canonicalization_parity.js.", + "", + "Symbols marked 'real' were lifted verbatim from ROCm captures. Hand-written C++ never produced", + "the shapes that broke this: an unnamed namespace putting a '(' in front of the kernel, and a", + "bare kernel with no namespace to separate it from the 'void' return type." + ], + "canonical": [ + { + "real": true, + "symbol": "void (anonymous namespace)::kda_packed_decode_kernel<8, false>((anonymous namespace)::KdaPackedDecodeParams)", + "token": "kda_packed_decode_kernel" + }, + { + "real": true, + "symbol": "void aiter::greedy_sample_kernel(float const*, int*, int, int)", + "token": "greedy_sample_kernel" + }, + { + "real": true, + "symbol": "void wvSplitKrc_<__hip_bfloat16, 8, 4>(void*, int)", + "token": "wvsplitkrc_" + }, + { + "real": true, + "symbol": "void paged_attention_ll4mi_QKV_mfma16_kernel<0, __hip_bfloat16, (vllm::Fp8KVCacheDataType)0, 256>(float*, int*)", + "token": "paged_attention_ll4mi_qkv_mfma16_kernel" + }, + { + "real": true, + "symbol": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bLi256EEEvPT_i", + "token": "_zn5aiter24add_rmsnorm_quant_kernelidf16bli256eeevpt_i" + }, + { + "real": true, + "symbol": "reshape_and_cache_shuffle_5d", + "token": "reshape_and_cache_shuffle_5d" + }, + { + "symbol": "at::native::vectorized_elementwise_kernel<4, at::native::AddFunctor, at::detail::Array >(int, float)", + "token": "vectorized_elementwise_kernel" + }, + { + "symbol": "vectorized_elementwise_kernel [clone .isra.0]", + "token": "vectorized_elementwise_kernel" + }, + { + "real": true, + "_note": "profile artifacts store long kernel names elided mid-template, leaving an opener that never closes", + "symbol": "void at::native::elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k...", + "token": "elementwise_kernel_manual_unroll" + }, + { + "real": true, + "symbol": "_ZN7ck_tile6kentryILi2ENS_15MoeFlatmmKernelINS_33GemmSpatiallyLocalTilePart...", + "token": "_zn7ck_tile6kentryili2ens_15moeflatmmkernelins_33gemmspatiallylocaltilepart" + }, + { "symbol": "gemm_kernel", "token": "" }, + { "symbol": "", "token": "" } + ], + "matches": [ + { + "why": "the bare name and its decorated spelling are one kernel", + "a": "paged_attention_ll4mi_QKV_mfma4_kernel", + "b": "void paged_attention_ll4mi_QKV_mfma4_kernel<__hip_bfloat16, 128, 256>(float*, int)", + "match": true + }, + { + "why": "a bare declared name has no template arguments to compare, so it matches the decorated spelling", + "a": "vectorized_elementwise_kernel", + "b": "void at::native::vectorized_elementwise_kernel<4, AddFunctor >(int)", + "match": true + }, + { + "why": "two instantiations of one template are two kernels once both sides carry the arguments", + "a": "void at::native::vectorized_elementwise_kernel<16, at::native::FillFunctor >(int)", + "b": "void at::native::vectorized_elementwise_kernel<4, at::native::MulFunctor >(int)", + "match": false + }, + { + "why": "real head row: a short_name cut inside the ARGUMENT list keeps a whole template, and only one side spells the return type and namespace", + "a": "clamp_position_kernel(long*, long const*, unsigned lon", + "b": "void (anonymous namespace)::clamp_position_kernel(long*, long const*, unsigned long, int)", + "match": true + }, + { + "why": "one build leaves the namespace unnamed and another declares it; same kernel, same instantiation", + "a": "void (anonymous namespace)::store_kvcache<512l, 512l, 1, false, long>(int)", + "b": "void sglang::store_kvcache<512l, 512l, 1, false, long>(sglang::StoreKVCacheParams)", + "match": true + }, + { + "why": "both argument lists are elided, so they must agree as far as they are spelled: 128, 4 against 128, 8", + "a": "void at::native::elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k...", + "b": "void at::native::elementwise_kernel_manual_unroll<128, 8, at::native::gpu_k...", + "match": false + }, + { + "why": "folding separators to _ stops '128, 4,' from reading as a prefix of '128, 48,'", + "a": "k<128, 4, at::native::gpu_k", "b": "k<128, 48, at::native::gpu_k", "match": false + }, + { + "why": "real sglang neighbours: a name appearing inside another is a different kernel", + "a": "_fwd_kernel", "b": "_fwd_kernel_stage2", "match": false + }, + { + "why": "a name embedded in another is not evidence of the same kernel", + "a": "gemm_kernel", "b": "fused_gemm_kernel_v2", "match": false + }, + { + "why": "two unnamed-namespace kernels used to both reduce to 'void' and certify each other", + "a": "void (anonymous namespace)::clamp_position_kernel(long*, long)", + "b": "void (anonymous namespace)::kda_packed_decode_kernel<8, false>(int)", + "match": false + }, + { + "why": "a token this short appears inside unrelated kernels and is not evidence", + "a": "gemm", "b": "gemm_kernel_v2", "match": false + }, + { + "why": "a prefix that is not the display-limit truncation is a different kernel", + "a": "attention_kernel", "b": "attention_kernelbackward", "match": false + }, + { + "why": "a name cut at the 60-char display limit ends mid-token, so only a prefix rule can match it", + "a": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bDF16bLi256ELi16ELb1", + "b": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bDF16bLi256ELi16ELb1ELb0ELb1ELi1EEEvPT0_i", + "match": true + }, + { + "why": "one character short of the display limit is just an unrelated fragment", + "a": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bDF16bLi256ELi16ELb", + "b": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bDF16bLi256ELi16ELb1ELb0ELb1ELi1EEEvPT0_i", + "match": false + }, + { "why": "an empty side is never evidence", "a": "", "b": "gemm_kernel", "match": false } + ] +} diff --git a/e2e_workflow/scripts/tests/test_kernel_selection.py b/e2e_workflow/scripts/tests/test_kernel_selection.py index 0a699c3f0..a8df18d8f 100644 --- a/e2e_workflow/scripts/tests/test_kernel_selection.py +++ b/e2e_workflow/scripts/tests/test_kernel_selection.py @@ -14,6 +14,12 @@ ks = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(ks) +# parse_profile owns the display-name limit that canonical_kernel_name's prefix rule depends on. +PP_SPEC = importlib.util.spec_from_file_location( + "parse_profile", os.path.join(SCRIPTS, "parse_profile.py")) +pp = importlib.util.module_from_spec(PP_SPEC) +PP_SPEC.loader.exec_module(pp) + TARGET = ( "vllm.v1.attention.ops.chunked_prefill_paged_decode:" "chunked_prefill_paged_decode" @@ -67,10 +73,177 @@ def test_nested_template_arguments_reduce_to_the_bare_kernel_name(self): "vectorized_elementwise_kernel") self.assertTrue(ks.kernel_matches("vectorized_elementwise_kernel", decorated)) - def test_an_unbalanced_delimiter_does_not_survive_into_the_token(self): - self.assertEqual(ks.canonical_kernel_name("gemm_kernel(c10::BFloat1"), + "grouped_topk_kernel") + self.assertEqual(ks.canonical_kernel_name("gemm_kernel"), "") + def test_an_elided_name_still_matches_the_full_symbol_it_was_cut_from(self): + elided = "_ZN7ck_tile6kentryILi2ENS_15MoeFlatmmKernelINS_33GemmSpatiallyLocalTilePart..." + full = ("_ZN7ck_tile6kentryILi2ENS_15MoeFlatmmKernelINS_33GemmSpatiallyLocalTilePartitioner" + "INS_13TileGemmShapeINS_8sequenceIJLi16") + self.assertTrue(ks.kernel_matches(elided, full)) + + +# Symbols lifted verbatim from ROCm captures under /shared_nfs/hyperloom-claw. They are here because +# hand-written C++ never produced the shapes that broke this: an unnamed namespace putting a '(' in +# front of the kernel, and a bare kernel with no namespace at all to separate it from 'void'. +REAL_ROCM_SYMBOLS = { + "void (anonymous namespace)::kda_packed_decode_kernel<8, false>" + "((anonymous namespace)::KdaPackedDecodeParams)": "kda_packed_decode_kernel", + "void aiter::greedy_sample_kernel(float const*, int*, int, int)": + "greedy_sample_kernel", + "void paged_attention_ll4mi_QKV_mfma16_kernel<0, __hip_bfloat16, " + "(vllm::Fp8KVCacheDataType)0, 256>(float*, int*)": "paged_attention_ll4mi_qkv_mfma16_kernel", + "void wvSplitKrc_<__hip_bfloat16, 8, 4>(void*, int)": "wvsplitkrc_", + "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bLi256EEEvPT_i": "_zn5aiter24add_rmsnorm_quant_kernelidf16bli256eeevpt_i", + "reshape_and_cache_shuffle_5d": "reshape_and_cache_shuffle_5d", +} + + +class TestRealRocmSymbolsCanonicalizeToTheirKernel(unittest.TestCase): + def test_each_symbol_reduces_to_the_kernel_it_names(self): + for symbol, token in REAL_ROCM_SYMBOLS.items(): + with self.subTest(symbol=symbol[:48]): + self.assertEqual(ks.canonical_kernel_name(symbol), token) + + def test_an_unnamed_namespace_does_not_collapse_the_symbol_to_its_return_type(self): + """ROCm spells the unnamed namespace '(anonymous namespace)', which puts a parenthesis BEFORE + the kernel. Cutting the symbol at its first '(' left the return type as the whole token, so + every such kernel reduced to 'void' -- and two unrelated ones then certified each other, + which is exactly the unearned credit this verdict exists to refuse.""" + one = "void (anonymous namespace)::clamp_position_kernel(long*, long)" + two = "void (anonymous namespace)::kda_packed_decode_kernel<8, false>(int)" + self.assertEqual(ks.canonical_kernel_name(one), "clamp_position_kernel") + self.assertEqual(ks.canonical_kernel_name(two), "kda_packed_decode_kernel") + self.assertFalse(ks.kernel_matches(one, two)) + self.assertFalse(ks.kernel_matches(two, one)) + + def test_a_kernel_with_no_namespace_is_not_fused_with_its_return_type(self): + """Stripping separators rather than splitting on them glued 'void' onto any kernel that had + no '::' to fall back on, so the bare name never matched its own decorated spelling.""" + bare = "paged_attention_ll4mi_QKV_mfma4_kernel" + decorated = f"void {bare}<__hip_bfloat16, 128, 256>(float*, int)" + self.assertEqual(ks.canonical_kernel_name(decorated), bare.lower()) + self.assertTrue(ks.kernel_matches(bare, decorated)) + + def test_a_name_truncated_by_the_display_limit_still_matches_its_full_symbol(self): + """parse_profile.short_name caps display names, and mangled and Tensile symbols run well past + the cap. The truncated name ends mid-token, so no word boundary can follow it -- without an + explicit prefix rule our own shortening reads as a different kernel.""" + symbol = "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bDF16bLi256ELi16ELb1ELb0ELb1ELi1EEEvPT0_i" + declared = symbol[:ks.SHORT_NAME_LIMIT] + self.assertGreater(len(symbol), ks.SHORT_NAME_LIMIT) + self.assertTrue(ks.kernel_matches(declared, symbol)) + self.assertFalse(ks.kernel_matches(declared[:ks.SHORT_NAME_LIMIT - 1], symbol)) + + def test_a_name_embedded_inside_another_is_a_different_kernel(self): + """The first two pairs are real neighbours inside one sglang capture. Accepting a name + because it appears inside the other certified _fwd_kernel as _fwd_kernel_stage2 -- a + different kernel, which is precisely the credit this verdict exists to withhold.""" + for a, b in ( + ("_fwd_kernel", "_fwd_kernel_stage2"), + ("reshape_and_cache_kernel", "reshape_and_cache_kernel_flash"), + ("gemm_kernel", "fused_gemm_kernel_v2"), + ("gemm", "gemm_kernel_v2"), + ("attention_kernel", "attention_kernelbackward"), + ): + with self.subTest(a=a, b=b): + self.assertFalse(ks.kernel_matches(a, b)) + self.assertFalse(ks.kernel_matches(b, a)) + + def test_two_instantiations_of_one_template_are_not_one_kernel(self): + """The base token drops template arguments so a bare declared name can match its decorated + spelling. When both sides carry those arguments the information is present on both, and one + capture here held 20 distinct kernels whose only difference was the functor.""" + fill = ("void at::native::vectorized_elementwise_kernel<16, at::native::FillFunctor, " + "std::array >(int, at::native::FillFunctor)") + power = ("void at::native::vectorized_elementwise_kernel<4, at::native::(anonymous " + "namespace)::pow_tensor_scalar_kernel_impl, std::array >(int)") + self.assertEqual(ks.canonical_kernel_name(fill), ks.canonical_kernel_name(power)) + self.assertFalse(ks.kernel_matches(fill, power)) + self.assertTrue(ks.kernel_matches(fill, fill)) + + def test_a_bare_declared_name_still_matches_a_templated_symbol(self): + """The tightening above applies only when BOTH sides carry template arguments. A head that + declares the bare name has none to compare, and refusing it would reject every real head.""" + self.assertTrue(ks.kernel_matches( + "vectorized_elementwise_kernel", + "void at::native::vectorized_elementwise_kernel<4, AddFunctor >(int)")) + + def test_arguments_are_read_past_the_return_type_and_namespace(self): + """Only one side spells the return type and namespace. This pair is a real head row whose + stored short_name had been cut inside the ARGUMENT list while its template stayed whole -- + comparing the symbols entire refused a kernel against itself.""" + self.assertTrue(ks.kernel_matches( + "clamp_position_kernel(long*, long const*, unsigned lon", + "void (anonymous namespace)::clamp_position_kernel(long*, long const*, unsigned " + "long, int)")) + + def test_one_kernel_matches_across_builds_that_name_its_namespace_differently(self): + """Both spellings occur across the captures: one build leaves the namespace unnamed, another + declares it. Same kernel, same instantiation.""" + self.assertTrue(ks.kernel_matches( + "void (anonymous namespace)::store_kvcache<512l, 512l, 1, false, long>(int)", + "void sglang::store_kvcache<512l, 512l, 1, false, long>(sglang::StoreKVCacheParams)")) + + def test_an_elided_argument_list_still_has_to_agree_as_far_as_it_is_spelled(self): + """Artifacts elide long names mid-template, so neither list is complete. Skipping the check + entirely would let every elided instantiation of one template certify the others -- these two + differ only in the visible '128, 4' against '128, 8'.""" + four = "void at::native::elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k..." + eight = "void at::native::elementwise_kernel_manual_unroll<128, 8, at::native::gpu_k..." + self.assertEqual(ks.canonical_kernel_name(four), ks.canonical_kernel_name(eight)) + self.assertFalse(ks.kernel_matches(four, eight)) + self.assertTrue(ks.kernel_matches(four, four)) + + def test_a_separator_cannot_make_one_argument_list_a_prefix_of_another(self): + """Deleting separators instead of folding them read '<128, 4,' as a prefix of '<128, 48,'.""" + self.assertFalse(ks.kernel_matches("k<128, 4, at::native::gpu_k", + "k<128, 48, at::native::gpu_k")) + + def test_the_display_limit_agrees_with_the_module_that_applies_it(self): + """The prefix rule above is only sound while both sides mean the same number of characters.""" + self.assertEqual(ks.SHORT_NAME_LIMIT, pp.SHORT_NAME_LIMIT) + + +class TestTheSharedFixtureHoldsOnThisSide(unittest.TestCase): + """kernel_symbols.json is the one place both canonicalizers are pinned. The node half of this is + scripts/test_kernel_canonicalization_parity.js; a rule added here without adding it there is + exactly the drift that lets a kernel pass the JS gate and be refused by this verdict.""" + + @classmethod + def setUpClass(cls): + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), + "kernel_symbols.json")) as handle: + cls.fixture = json.load(handle) + + def test_every_symbol_reduces_to_its_recorded_token(self): + for case in self.fixture["canonical"]: + with self.subTest(symbol=case["symbol"][:48]): + self.assertEqual(ks.canonical_kernel_name(case["symbol"]), case["token"]) + + def test_every_recorded_pair_gets_the_recorded_verdict(self): + for case in self.fixture["matches"]: + with self.subTest(why=case["why"]): + self.assertEqual(ks.kernel_matches(case["a"], case["b"]), case["match"]) + self.assertEqual(ks.kernel_matches(case["b"], case["a"]), case["match"]) + + def test_the_fixture_still_covers_symbols_taken_from_real_captures(self): + """Hand-written C++ is what let the unnamed-namespace and bare-kernel defects through.""" + self.assertGreaterEqual(sum(1 for c in self.fixture["canonical"] if c.get("real")), 5) + class TestSelectionVerdict(unittest.TestCase): def meta(self, calls=7, target=TARGET): diff --git a/e2e_workflow/scripts/tests/test_parse_profile.py b/e2e_workflow/scripts/tests/test_parse_profile.py index cf6b34011..f7641ec14 100644 --- a/e2e_workflow/scripts/tests/test_parse_profile.py +++ b/e2e_workflow/scripts/tests/test_parse_profile.py @@ -241,6 +241,19 @@ def test_strips_void_template_and_namespace(self): def test_truncates_to_60_chars(self): self.assertEqual(pp.short_name("k" * 70), "k" * 60) + self.assertEqual(pp.SHORT_NAME_LIMIT, 60) + + def test_an_unnamed_namespace_shortens_to_the_kernel_not_its_parameter(self): + """ROCm spells the unnamed namespace '(anonymous namespace)', so the symbol opens with '(' + where the identifier should be. The '[\\w:]+' probe found nothing, the whole signature fell + through as the "short" name, and splitting it on '::' handed back the trailing parameter + type -- this real symbol used to shorten to 'KdaPackedDecodeParams)'.""" + self.assertEqual( + pp.short_name("void (anonymous namespace)::kda_packed_decode_kernel<8, false>" + "((anonymous namespace)::KdaPackedDecodeParams)"), + "kda_packed_decode_kernel") + self.assertEqual(pp.norm_key("void (anonymous namespace)::clamp_position_kernel(int)"), + "clamppositionkernel") def test_non_identifier_name_is_returned_as_is(self): self.assertEqual(pp.short_name(""), "") diff --git a/e2e_workflow/scripts/tests/test_seam_trace.py b/e2e_workflow/scripts/tests/test_seam_trace.py index 45a5357ca..b1a2e4201 100644 --- a/e2e_workflow/scripts/tests/test_seam_trace.py +++ b/e2e_workflow/scripts/tests/test_seam_trace.py @@ -48,8 +48,9 @@ def setUp(self): self.events = [] self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - self.trace = os.path.join(self.tmp.name, "selection.json") - os.environ["GEAK_SELECTION_TRACE"] = self.trace + # UNIQUE=0 drops only the per-call suffix, so an explicit {pid} keeps the path predictable. + os.environ["GEAK_SELECTION_TRACE"] = os.path.join(self.tmp.name, "selection.{pid}.json") + self.trace = os.path.join(self.tmp.name, f"selection.{os.getpid()}.json") os.environ["GEAK_SELECTION_TRACE_UNIQUE"] = "0" os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "1" self.addCleanup(os.environ.pop, "GEAK_SELECTION_TRACE", None) @@ -140,7 +141,7 @@ def test_process_local_call_traces_do_not_overwrite(self): self.module.outer(2) traces = sorted( name for name in os.listdir(self.tmp.name) - if name.startswith("selection.pid-") and name.endswith(".json")) + if name.startswith(f"selection.{os.getpid()}.call-") and name.endswith(".json")) self.assertEqual(len(traces), 2) self.assertIn(".call-1.json", traces[0]) self.assertIn(".call-2.json", traces[1]) @@ -201,12 +202,20 @@ def test_a_path_with_no_placeholders_gets_the_pid_and_rank_appended(self): os.environ["RANK"] = "3" self.assertEqual(st._trace_path(7), f"/tmp/sel.pid-{os.getpid()}.rank-3.call-7.json") - def test_the_unique_opt_out_takes_the_path_verbatim(self): - """The escape hatch for a single-process debug capture: write exactly where I said. It is not - safe for a multi-rank run, which is why per-process naming is what happens by default.""" + def test_the_unique_opt_out_drops_the_call_suffix_but_not_the_process(self): + """The opt-out is for operators who want one file per process instead of one per call. It + cannot also drop the pid: kernel_selection pairs a capture to its trace by matching + '.pid-', so a shared filename loses the pairing and every rank races on one path.""" os.environ["GEAK_SELECTION_TRACE"] = "/tmp/sel.json" os.environ["GEAK_SELECTION_TRACE_UNIQUE"] = "0" - self.assertEqual(st._trace_path(7), "/tmp/sel.json") + os.environ["RANK"] = "3" + self.assertEqual(st._trace_path(7), f"/tmp/sel.pid-{os.getpid()}.rank-3.json") + + def test_the_unique_opt_out_still_substitutes_an_explicit_placeholder(self): + """Returning the raw template here used to leave a literal '{pid}' in the filename.""" + os.environ["GEAK_SELECTION_TRACE"] = "/tmp/sel-{pid}.json" + os.environ["GEAK_SELECTION_TRACE_UNIQUE"] = "0" + self.assertEqual(st._trace_path(7), f"/tmp/sel-{os.getpid()}.json") def test_the_rank_comes_from_the_launcher_env_when_it_declares_one(self): os.environ["RANK"] = "2" @@ -216,6 +225,10 @@ def test_an_unranked_process_is_still_named_rather_than_dropped(self): self.assertEqual(st._rank(), "unknown") +def _raise_record_function(name): + raise RuntimeError("record_function unavailable on this build") + + class TestProfileLifecycleFailsSoft(unittest.TestCase): """Every failure here happens inside the SERVER under measurement. A probe that raises would take the capture down with it, so each path degrades to "no trace" plus a stderr line.""" @@ -270,6 +283,54 @@ def export_chrome_trace(self, path): st._finish_profile() self.assertFalse(st._PROFILE["active"]) + def test_a_failure_after_entering_the_profiler_still_exits_it(self): + """The profiler is entered before the install markers are written. When that write raised, + the handler set done=True while active was still True -- and _finish_profile returns early on + done, so the profiler stayed entered, and collecting, for the life of the server.""" + exited = [] + + class _EnteredProfiler(_Profiler): + def __exit__(self, *exc): + exited.append(True) + return False + + self._install_torch(lambda activities: _EnteredProfiler([])) + sys.modules["torch"].profiler.record_function = _raise_record_function + st._INSTALLED["mod:fn"] = None + self.addCleanup(st._INSTALLED.pop, "mod:fn", None) + self.assertFalse(st._start_profile()) + self.assertEqual(exited, [True]) + self.assertFalse(st._PROFILE["active"]) + self.assertIsNone(st._PROFILE["profiler"]) + + def test_a_profiler_that_also_refuses_to_exit_does_not_take_the_call_down(self): + """Everything on this path runs inside the served call, including the cleanup.""" + class _UnstoppableProfiler(_Profiler): + def __exit__(self, *exc): + raise RuntimeError("profiler already torn down") + + self._install_torch(lambda activities: _UnstoppableProfiler([])) + sys.modules["torch"].profiler.record_function = _raise_record_function + st._INSTALLED["mod:fn"] = None + self.addCleanup(st._INSTALLED.pop, "mod:fn", None) + self.assertFalse(st._start_profile()) + self.assertFalse(st._PROFILE["active"]) + + def test_a_process_with_no_trace_destination_never_enters_a_profiler(self): + """The marker overlay is loaded by processes that are not the capture. _start_profile has to + stop at the missing destination rather than profile a run nothing will read.""" + os.environ.pop("GEAK_SELECTION_TRACE", None) + self._install_torch(lambda activities: _Profiler([])) + self.assertFalse(st._start_profile()) + self.assertFalse(st._PROFILE["done"]) + + def test_the_budget_is_rechecked_before_each_start_not_only_after_a_finish(self): + os.environ["GEAK_SELECTION_PROFILE_CALLS"] = "1" + st._PROFILE["trace_index"] = 1 + self._install_torch(lambda activities: _Profiler([])) + self.assertFalse(st._start_profile()) + self.assertTrue(st._PROFILE["done"]) + def test_finishing_a_profile_that_never_started_is_a_no_op(self): st._finish_profile() self.assertEqual(st._PROFILE["trace_index"], 0) From 2253f32558e0d3b3c1a1be6c8a34ccc71677b862 Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Thu, 20 Aug 2026 14:30:42 +0000 Subject: [PATCH 8/8] Read a kernel name from the front of the symbol, not the back --- e2e_workflow/e2e_workflow.js | 26 +++++--- e2e_workflow/scripts/kernel_selection.py | 62 ++++++++++++++---- .../test_kernel_canonicalization_parity.js | 5 +- .../scripts/tests/kernel_symbols.json | 63 ++++++++++++++----- .../scripts/tests/test_kernel_selection.py | 26 +++++++- 5 files changed, 143 insertions(+), 39 deletions(-) diff --git a/e2e_workflow/e2e_workflow.js b/e2e_workflow/e2e_workflow.js index fd0700e07..117dd015c 100644 --- a/e2e_workflow/e2e_workflow.js +++ b/e2e_workflow/e2e_workflow.js @@ -768,17 +768,25 @@ const stripBalanced = (symbol, opener, closer) => { // Parentheses are stripped as balanced groups rather than by cutting at the first '(': ROCm spells // the unnamed namespace `(anonymous namespace)`, which puts a parenthesis BEFORE the kernel, so // cutting there collapsed every such symbol to the return type `void`, and two unrelated kernels -// that both collapsed to `void` then certified each other. Taking the last whitespace-separated -// token drops that return type without fusing it into names that carry no namespace. +// that both collapsed to `void` then certified each other. Brackets go the same way, which subsumes +// the `[clone .1]` rule: this pipeline appends its own annotations to a kernel name +// (`_fwd_grouped_kernel_stage1 [sliding_attention]`, `main_kernel[prefill]`), and rocprof spells +// memory ops `Memcpy DtoD (Device -> Device)`. The return type is then dropped by name and the FIRST +// identifier taken, as parse_profile.short_name does; taking the last whitespace token also drops +// `void`, but on any of those annotated spellings it answers with the annotation. function canonicalDeviceKernel(s) { - const stripped = stripBalanced( - stripBalanced(String(s || '').replace(/\[clone[^\]]*\]/gi, ''), '<', '>'), '(', ')'); + const stripped = stripBalanced(stripBalanced( + stripBalanced(String(s || ''), '[', ']'), '<', '>'), '(', ')'); // Whatever opener is left never closes, so the symbol was cut off inside it -- profile artifacts - // store kernel names elided mid-template. The name is what precedes that opener; taking the last - // whitespace token instead lifts a fragment out of the template arguments and states it with the - // same confidence as a real name, which then matches the wrong kernel rather than refusing. - const parts = stripped.split(/[<(]/)[0].split('::').pop().split(/\s+/).filter(Boolean); - return parts.length ? parts[parts.length - 1].toLowerCase().replace(/[^a-z0-9_]+/g, '') : ''; + // store kernel names elided mid-template. The name is what precedes that opener; reading on + // instead lifts a fragment out of the template arguments and states it with the same confidence + // as a real name, which then matches the wrong kernel rather than refusing. + // Removing a group also leaves a gap where it stood, and an unnamed namespace sits mid- + // qualification: `at::native::(anonymous namespace)::CatArrayBatchedCopy` becomes + // `at::native:: ::CatArray...`, where the identifier probe stops at the space. + const cut = stripped.split(/[<([]/)[0].trim().replace(/\s*::\s*/g, '::'); + const match = /^[\w:]+/.exec(cut.replace(/^void\s+/, '')); + return match ? match[0].split('::').pop().toLowerCase().replace(/[^a-z0-9_]+/g, '') : ''; } // A name that hit the display limit ends mid-token, so no word boundary can follow it. Tested both // ways because either side may be the shortened one: heads carry a short_name while traces carry diff --git a/e2e_workflow/scripts/kernel_selection.py b/e2e_workflow/scripts/kernel_selection.py index 2e7f3d82c..89a6fa2d6 100644 --- a/e2e_workflow/scripts/kernel_selection.py +++ b/e2e_workflow/scripts/kernel_selection.py @@ -98,20 +98,33 @@ def canonical_kernel_name(value): Parentheses are stripped as balanced groups rather than by cutting at the first ``(``. ROCm names the unnamed namespace ``(anonymous namespace)``, which puts a parenthesis *before* the kernel, so cutting there collapsed every such symbol to the return type ``void`` -- and two unrelated - kernels that both collapse to ``void`` then certified each other. Taking the last whitespace- - separated token drops that return type without fusing it into names that carry no namespace. + kernels that both collapse to ``void`` then certified each other. + + Brackets go the same way, which subsumes the ``[clone .1]`` rule: this pipeline appends its own + annotations to a kernel name (``_fwd_grouped_kernel_stage1 [sliding_attention]``, + ``main_kernel[prefill]``, ``hgemm_..._SPK1 [qkv_proj]``), and rocprof spells memory ops + ``Memcpy DtoD (Device -> Device)``. + + The return type is then dropped by name and the FIRST identifier is taken, exactly as + ``parse_profile.short_name`` does. Taking the last whitespace token instead also drops ``void``, + but on any of the annotated spellings above it answers with the annotation -- ``DtoD``, + ``sliding_attention``, ``qkv_proj`` -- and a head then fails to match its own kernel. """ - text = re.sub(r"\[clone[^\]]*\]", "", str(value or ""), flags=re.IGNORECASE) + text = _strip_balanced(str(value or ""), "[", "]") text = _strip_balanced(text, "<", ">") text = _strip_balanced(text, "(", ")") # Whatever opener is left never closes, so the symbol was cut off inside it -- profile artifacts - # store kernel names elided mid-template. The name is what precedes that opener; taking the last - # whitespace token instead lifts a fragment out of the template arguments and states it with the - # same confidence as a real name ('...elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k' - # answered 'gpu_k'), which then matches the wrong kernel rather than refusing to answer. - text = re.split(r"[<(]", text, 1)[0] - text = text.split("::")[-1].split() - return re.sub(r"[^a-z0-9_]+", "", text[-1].lower()) if text else "" + # store kernel names elided mid-template. The name is what precedes that opener; reading on + # instead lifts a fragment out of the template arguments and states it with the same confidence + # as a real name ('...elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k' answered + # 'gpu_k'), which then matches the wrong kernel rather than refusing to answer. + text = re.split(r"[<(\[]", text, maxsplit=1)[0] + # Removing a group leaves a gap where it stood, and an unnamed namespace sits mid-qualification: + # `at::native::(anonymous namespace)::CatArrayBatchedCopy` becomes `at::native:: ::CatArray...`, + # where the identifier probe below stops at the space and the last `::` segment is empty. + text = re.sub(r"\s*::\s*", "::", text.strip()) + match = re.match(r"[\w:]+", re.sub(r"^void\s+", "", text)) + return re.sub(r"[^a-z0-9_]+", "", match.group(0).split("::")[-1].lower()) if match else "" def _truncated_prefix(needle, haystack): @@ -203,6 +216,30 @@ def _complete_spans(events, name): return spans +def _outermost_spans(spans): + """Return one logical call for same-thread spans nested inside an identical marker. + + ``capture_shapes`` and ``seam_trace`` can both wrap the selected callable with + ``GEAK_TARGET::``. One real invocation then produces the same marker nested inside itself. + Keep every raw span for launch-causality analysis, but count only the outermost copy when reporting + calls. Different threads and non-nested calls remain distinct. + """ + outermost = [] + for index, span in sorted( + enumerate(spans), + key=lambda item: (item[1][0], -item[1][1], item[0])): + start, end, event = span + contained = any( + outer_start <= start and end <= outer_end + and event.get("pid") == outer_event.get("pid") + and event.get("tid") == outer_event.get("tid") + for outer_start, outer_end, outer_event in outermost + ) + if not contained: + outermost.append(span) + return outermost + + def _within_any_span(event, spans, same_thread=False): if event.get("ts") is None: return False @@ -253,6 +290,7 @@ def _marker_kernel_evidence(trace_events, target_callable, device_kernel): "target": target_callable, "marker": marker, "spans": spans, + "marker_calls": len(_outermost_spans(spans)), "related_external_ids": related_external_ids, "related_correlations": related_correlations, "matched": matched, @@ -314,7 +352,7 @@ def verify(target_callable, device_kernel, capture_meta, trace_events, candidate if missing_candidate_markers: failed.append("candidate_marker_not_installed") selected_evidence = evidence.get(target_callable) or { - "marker": MARKER_PREFIX + target_callable, "spans": [], + "marker": MARKER_PREFIX + target_callable, "spans": [], "marker_calls": 0, "related_external_ids": set(), "related_correlations": set(), "matched": [], } marker = selected_evidence["marker"] @@ -355,7 +393,7 @@ def verify(target_callable, device_kernel, capture_meta, trace_events, candidate "capture_target": meta_target, "total_calls_observed": observed_calls, "target_marker": marker, - "target_marker_calls": len(spans), + "target_marker_calls": selected_evidence["marker_calls"], "matched_kernel_calls": len(matched), "matched_kernel_names": sorted(set(matched)), "correlated_external_ids": len(related_external_ids), diff --git a/e2e_workflow/scripts/test_kernel_canonicalization_parity.js b/e2e_workflow/scripts/test_kernel_canonicalization_parity.js index 8a9a76138..89455d6ac 100644 --- a/e2e_workflow/scripts/test_kernel_canonicalization_parity.js +++ b/e2e_workflow/scripts/test_kernel_canonicalization_parity.js @@ -37,8 +37,9 @@ const fixture = JSON.parse(fs.readFileSync(FIXTURE, 'utf8')); console.log('\n# canonical tokens'); for (const c of fixture.canonical) { const got = canonicalDeviceKernel(c.symbol); - ok(got === c.token, `${JSON.stringify(c.symbol.slice(0, 56))} -> ${JSON.stringify(c.token)}` + - (got === c.token ? '' : ` (got ${JSON.stringify(got)})`)); + ok(got === c.canonical, + `${JSON.stringify(c.symbol.slice(0, 56))} -> ${JSON.stringify(c.canonical)}` + + (got === c.canonical ? '' : ` (got ${JSON.stringify(got)})`)); } console.log('\n# match verdicts'); diff --git a/e2e_workflow/scripts/tests/kernel_symbols.json b/e2e_workflow/scripts/tests/kernel_symbols.json index 2d90e55f2..4cae4657f 100644 --- a/e2e_workflow/scripts/tests/kernel_symbols.json +++ b/e2e_workflow/scripts/tests/kernel_symbols.json @@ -8,62 +8,85 @@ "tests/test_kernel_selection.py and from node by scripts/test_kernel_canonicalization_parity.js.", "", "Symbols marked 'real' were lifted verbatim from ROCm captures. Hand-written C++ never produced", - "the shapes that broke this: an unnamed namespace putting a '(' in front of the kernel, and a", - "bare kernel with no namespace to separate it from the 'void' return type." + "the shapes that broke this: an unnamed namespace putting a '(' in front of the kernel, a bare", + "kernel with no namespace to separate it from the 'void' return type, and the annotations this", + "pipeline appends to a name after the fact -- a phase tag, a cluster label, a memcpy direction." ], "canonical": [ { "real": true, "symbol": "void (anonymous namespace)::kda_packed_decode_kernel<8, false>((anonymous namespace)::KdaPackedDecodeParams)", - "token": "kda_packed_decode_kernel" + "canonical": "kda_packed_decode_kernel" }, { "real": true, "symbol": "void aiter::greedy_sample_kernel(float const*, int*, int, int)", - "token": "greedy_sample_kernel" + "canonical": "greedy_sample_kernel" }, { "real": true, "symbol": "void wvSplitKrc_<__hip_bfloat16, 8, 4>(void*, int)", - "token": "wvsplitkrc_" + "canonical": "wvsplitkrc_" }, { "real": true, "symbol": "void paged_attention_ll4mi_QKV_mfma16_kernel<0, __hip_bfloat16, (vllm::Fp8KVCacheDataType)0, 256>(float*, int*)", - "token": "paged_attention_ll4mi_qkv_mfma16_kernel" + "canonical": "paged_attention_ll4mi_qkv_mfma16_kernel" }, { "real": true, "symbol": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bLi256EEEvPT_i", - "token": "_zn5aiter24add_rmsnorm_quant_kernelidf16bli256eeevpt_i" + "canonical": "_zn5aiter24add_rmsnorm_quant_kernelidf16bli256eeevpt_i" }, { "real": true, "symbol": "reshape_and_cache_shuffle_5d", - "token": "reshape_and_cache_shuffle_5d" + "canonical": "reshape_and_cache_shuffle_5d" }, { "symbol": "at::native::vectorized_elementwise_kernel<4, at::native::AddFunctor, at::detail::Array >(int, float)", - "token": "vectorized_elementwise_kernel" + "canonical": "vectorized_elementwise_kernel" }, { "symbol": "vectorized_elementwise_kernel [clone .isra.0]", - "token": "vectorized_elementwise_kernel" + "canonical": "vectorized_elementwise_kernel" }, { "real": true, "_note": "profile artifacts store long kernel names elided mid-template, leaving an opener that never closes", "symbol": "void at::native::elementwise_kernel_manual_unroll<128, 4, at::native::gpu_k...", - "token": "elementwise_kernel_manual_unroll" + "canonical": "elementwise_kernel_manual_unroll" }, { "real": true, "symbol": "_ZN7ck_tile6kentryILi2ENS_15MoeFlatmmKernelINS_33GemmSpatiallyLocalTilePart...", - "token": "_zn7ck_tile6kentryili2ens_15moeflatmmkernelins_33gemmspatiallylocaltilepart" + "canonical": "_zn7ck_tile6kentryili2ens_15moeflatmmkernelins_33gemmspatiallylocaltilepart" }, - { "symbol": "gemm_kernel", "token": "" }, - { "symbol": "", "token": "" } + { + "real": true, + "_note": "the annotations below are appended by this pipeline, not by the compiler. Reading the name from the END of the symbol answers with the annotation and the head stops matching its own kernel.", + "symbol": "_fwd_grouped_kernel_stage1 [sliding_attention]", + "canonical": "_fwd_grouped_kernel_stage1" + }, + { "real": true, "symbol": "main_kernel[prefill]", "canonical": "main_kernel" }, + { "real": true, "symbol": "main_kernel [cluster B (long, ~34us)]", "canonical": "main_kernel" }, + { "real": true, "symbol": "hgemm_bf16_32x64x256x3_SPK1 [qkv_proj]", "canonical": "hgemm_bf16_32x64x256x3_spk1" }, + { "real": true, "symbol": "allreduce_prototype_twoshot@prefill", "canonical": "allreduce_prototype_twoshot" }, + { + "real": true, + "_note": "rocprof spells memory ops with the direction after the operation", + "symbol": "Memcpy DtoD (Device -> Device)", + "canonical": "memcpy" + }, + { + "real": true, + "_note": "an unnamed namespace can also sit mid-qualification, leaving 'at::native:: ::CatArray...' once the group is removed", + "symbol": "void at::native::(anonymous namespace)::CatArrayBatchedCopy", "canonical": "" }, + { "symbol": "", "canonical": "" } ], "matches": [ { @@ -140,6 +163,16 @@ "b": "_ZN5aiter24add_rmsnorm_quant_kernelIDF16bDF16bLi256ELi16ELb1ELb0ELb1ELi1EEEvPT0_i", "match": false }, + { + "why": "real head row: the phase tag is ours, so the row still has to match the kernel it names", + "a": "_fwd_grouped_kernel_stage1 [sliding_attention]", + "b": "_fwd_grouped_kernel_stage1", + "match": true + }, + { + "why": "real head row: a memcpy row and its own rocprof spelling", + "a": "Memcpy", "b": "Memcpy DtoD (Device -> Device)", "match": true + }, { "why": "an empty side is never evidence", "a": "", "b": "gemm_kernel", "match": false } ] } diff --git a/e2e_workflow/scripts/tests/test_kernel_selection.py b/e2e_workflow/scripts/tests/test_kernel_selection.py index a8df18d8f..9679e964e 100644 --- a/e2e_workflow/scripts/tests/test_kernel_selection.py +++ b/e2e_workflow/scripts/tests/test_kernel_selection.py @@ -232,7 +232,7 @@ def setUpClass(cls): def test_every_symbol_reduces_to_its_recorded_token(self): for case in self.fixture["canonical"]: with self.subTest(symbol=case["symbol"][:48]): - self.assertEqual(ks.canonical_kernel_name(case["symbol"]), case["token"]) + self.assertEqual(ks.canonical_kernel_name(case["symbol"]), case["canonical"]) def test_every_recorded_pair_gets_the_recorded_verdict(self): for case in self.fixture["matches"]: @@ -256,6 +256,30 @@ def test_0720_live_inner_launcher_is_selection_success(self): self.assertEqual(verdict["matched_kernel_calls"], 1) self.assertEqual(verdict["failed"], []) + def test_capture_and_seam_markers_count_one_logical_call(self): + """capture_shapes and seam_trace both emit GEAK_TARGET for the selected callable. The trace + therefore nests the same marker inside itself, but the report must count the invocation once.""" + events = trace() + events.insert(2, { + "cat": "cpu_op", "name": ks.MARKER_PREFIX + TARGET, + "ph": "X", "pid": 1, "tid": 2, "ts": 105, "dur": 80, + }) + verdict = ks.verify(TARGET, KERNEL, self.meta(calls=1), events) + self.assertTrue(verdict["ok"], verdict) + self.assertEqual(verdict["target_marker_calls"], 1) + self.assertEqual(verdict["matched_kernel_calls"], 1) + + def test_nested_markers_on_other_threads_and_later_calls_stay_distinct(self): + marker = ks.MARKER_PREFIX + TARGET + spans = ks._outermost_spans([ + (100.0, 200.0, {"pid": 1, "tid": 2, "name": marker}), + (110.0, 190.0, {"pid": 1, "tid": 2, "name": marker}), + (120.0, 180.0, {"pid": 1, "tid": 3, "name": marker}), + (300.0, 320.0, {"pid": 1, "tid": 2, "name": marker}), + ]) + self.assertEqual([(start, end) for start, end, _ in spans], + [(100.0, 200.0), (120.0, 180.0), (300.0, 320.0)]) + def test_kernel_seen_elsewhere_is_not_enough(self): verdict = ks.verify(TARGET, KERNEL, self.meta(), trace(under_target=False)) self.assertFalse(verdict["ok"])