diff --git a/admin/src/dashboard.ts b/admin/src/dashboard.ts index fd29ca1..6702cc5 100644 --- a/admin/src/dashboard.ts +++ b/admin/src/dashboard.ts @@ -91,12 +91,14 @@ function scrubReason(reason: any): string { } /** - * The control-byte class for text this pane renders, C0 + DEL + C1 (issue #337). + * The control-byte class for text this pane renders (issue #337, widened under #402). * * WHICH CLASS, because the project has two and the first version of this picked the wrong one. The * narrow one is `triggers.mjs`'s VALIDATOR, C0 + DEL, which decides whether an operator-authored file - * is acceptable. The wider one is `panel.mjs`'s `CONTROL_CHARS`, C0 + DEL + C1, whose own comment calls - * it a "defensive strip of C0/C1 control chars from untrusted input" and which already backs `clip`, + * is acceptable. The wider one is `panel.mjs`'s `interpreted`, which since issue #402 asks what a code + * point DRAWS rather than listing shapes -- what a terminal executes, plus what breaks a line, draws + * nothing or draws a blank a reader cannot tell from a space, minus what COMPOSES a neighbouring glyph -- + * and which already backs `clip`, * so the PLAIN and ASCII render paths have stripped C1 out of these same rows all along. A themed row * and a plain row of the same record going through two different classes is the drift this must not be, * and the wider one is the right one for the job: this is text on its way to a terminal, not a file @@ -2336,11 +2338,26 @@ function renderRunList(rows: any[], selected: number, w: number): string[] { /** The indices of tail lines containing `query` (case-insensitive substring) -- the LIVE_TAIL search * model, computed over the held tail bytes alone, so search reads nothing the view does not already. */ -function tailMatches(lines: any[], query: string): number[] { - const q = String(query).toLowerCase(); +// Exported for its own pin, as `targetUrl` is: what an operator can SEARCH FOR has to stay tied to what +// the pane DRAWS, and that relation is not visible from a rendered frame. +export function tailMatches(lines: any[], query: string): number[] { + // SEARCH WHAT THE PANE DRAWS, not the raw bytes behind it (issue #402). The pane substitutes the whole + // control class, and the search box DELETES it from what the operator types, so once the class grew to + // hold the invisible characters that real log output carries, a line drawn as `repo -prod` could not be + // found by any query at all: typing what is on screen missed the raw byte, and pasting the original + // missed because the box had dropped it. Scrubbing both sides is the only arrangement where what an + // operator reads is what they can search for. + // ONE READING, THE DRAWN ONE, on both sides. The pane substitutes the class and the search box now + // substitutes it too, so typing what is on screen and pasting the original bytes produce the same query + // and both find the line. A first repair compared the drawn reading OR the deleted one, and a review pass + // showed what that costs: the deleted reading RE-MERGES the collision this class was widened to expose, + // so a search for `deploy-prod` matched both it and `deploy` plus a zero-width space plus `-prod`. The + // pane tells those two rows apart; the search must not put them back together. + const drawn = (v: string) => scrubControls(v).toLowerCase(); + const q = drawn(query); const out: number[] = []; for (let i = 0; i < lines.length; i++) { - if (String(lines[i]).toLowerCase().includes(q)) out.push(i); + if (drawn(String(lines[i])).includes(q)) out.push(i); } return out; } diff --git a/admin/src/panel.mjs b/admin/src/panel.mjs index 07d5350..c998176 100644 --- a/admin/src/panel.mjs +++ b/admin/src/panel.mjs @@ -60,8 +60,190 @@ export function setGlyphs(ascii) { const MIN_WIDTH = 8; -// eslint-disable-next-line no-control-regex -- defensive strip of C0/C1 control chars from untrusted input -const CONTROL_CHARS = /[\x00-\x1f\x7f-\x9f]/g; +/** + * THE CLASS, and the line it draws is INTERPRETED against COMPOSING (issue #402). + * + * It used to stop at what a terminal EXECUTES: C0, DEL and C1, with U+009B in it because a CSI introducer + * needs no ESC in front. That left every code point which changes what a reader SEES without executing + * anything, and two of those are worth naming: + * + * - a bidi override or isolate REORDERS the text after it, so `repo/safe` plus U+202E plus `gnp.txt` is + * drawn as a name ending in `.png`. The job id, the target and the branch are all attacker- or + * model-writable, and a run record is what an operator reads before deciding what to do about it. + * - a character that draws as NOTHING, or as a blank that is not a space, makes two DIFFERENT strings + * draw identically: `deploy-prod` against `deploy` plus U+200B plus `-prod`, and equally against a + * Hangul filler or a no-break space. Eleven columns each, and not the same trigger. The pickers select + * by the string, so the operator can edit or delete the row they did not mean. + * + * ASKED OF THE RENDERER, NOT LISTED AND NOT CATEGORISED, and it took two review rounds to get there. A + * hand-written list of the shapes someone thought of covered 98 code points; this covers 4,024, and the + * old list is a strict SUBSET of it -- nothing it held has been let go. Missing were + * the fillers (one of which this project's own `env-file.mjs` already calls a deception character), the + * tag block, the Arabic and Egyptian format controls, the musical controls, the reserved code points and every + * blank a reader cannot tell from a space. + * + * A SECOND VERSION derived it from `\p{Cf}|\p{Zl}|\p{Zp}|\p{Zs}`, which is a different rule wearing this + * one's clothes: it still left U+FFF0-U+FFF8, which THIS FILE's width table already treats as characters + * the renderer draws as nothing, and U+2800, which draws a blank cell. A list is what the carve-out in + * issue #382 was and it was wrong twice; four categories standing in for "what does this draw" is the same + * shape a third time. So membership asks what a code point DRAWS, through `columnsOf` and through the one + * place that table deliberately guesses, which the clause below the predicate explains. + * + * WHAT IS DELIBERATELY NOT IN IT: U+0020, which is the space this substitutes TO; and a code point that + * COMPOSES the character beside it. U+200D joins an emoji sequence into one glyph, U+200C is orthography + * in Persian and the Indic scripts, and a variation selector chooses a character's form (`Mn`, so no + * property here reaches it anyway). Substituting those changes a CHARACTER where substituting the rest + * reveals a CONTROL, and a gate that cannot tell them apart corrupts the text it was added to protect. + * + * ISSUE #401 ANSWERED THE OTHER HALF of that issue's argument, and saying it precisely took three goes. + * None of these corrupts the geometry, because every measurement site scrubs BEFORE it measures. "They + * measure zero columns now" was the claim, and by this module's own table it is false of 3,843 of the + * 4,024: only 181 measure zero. The renderer reads it the other way round, drawing all but 19 of them as + * nothing, and the gap between those two readings is the whole subject of the predicate below. What was + * left was the reading, which is why the class moves and the width table does not. + * + * SUBSTITUTION IS THEREFORE NO LONGER COLUMN-PRESERVING, and it used to be free: every member of the old + * class measured one column, so replacing it with a space changed no geometry. Most members of this one + * measure zero, so a substituted line is WIDER than the line that arrived. Nothing breaks, because every + * measurement site scrubs BEFORE it measures -- but that is an ordering those sites keep now, rather than + * an identity the counts used to give for nothing. + * + * THE TAG BLOCK IS BOTH, which is why it is the one thing settled by SEQUENCE. A tag after U+1F3F4 composes a + * subdivision flag, and a tag anywhere else is invisible text: a whole ASCII message at zero columns. So a + * tag is kept inside a flag sequence and substituted outside one. Measured both ways. + * + * WHAT IT COSTS, measured rather than waved at, because this is a trade and not a free win: + * + * - A CORRECTLY ISOLATED RTL NAME NOW DISPLAYS WORSE. `\u2067` + a Hebrew project name + `\u2069` + + * `/main` was isolating that name so it read correctly beside the LTR path, and it comes out as the + * name between two spaces with the isolation gone. The panel cannot tell that isolate from the one an + * attacker used, because they are the same code point doing the same thing, so this is the price of + * closing the deception rather than an oversight. + * - A soft-hyphenated word and a BOM-led log line each gain a space. + * - The gate closes the EXPLICIT deception only. The bidi algorithm reorders neutrals beside a strong + * RTL character with no control present at all, so `acme/repo` + a Hebrew letter + `gnp.txt` still + * reads differently from how it is stored. Substituting cannot reach that without refusing Hebrew. + * - THE COMPOSING CARVE-OUT IS ALSO A CHANNEL, and it is the one this file argues hardest for keeping: + * 265 default-ignorable code points stay out because they compose, and the renderer draws every one of + * them as nothing. `deploy` plus U+E0100 plus `-prod` is eleven columns either way after the gate, + * which is the same collision the tag block gets a sequence matcher for, at about 2.8 times its size. + * Closing it would need the same sequence-awareness the flag has, per script rather than per block. + * - THE FAST PATH ONLY RESCUES ASCII. A tail of 200 lines of 100 KB costs about 200 ms a render at 0% + * non-ASCII and about 4 seconds at 100% CJK. That is not a regression, the branch only ever saves + * work, but this panel's own argument is that it renders CJK as a matter of course. + * + * THIS PROJECT NOW HAS THREE CLASSES FOR ONE QUESTION, and the differences are deliberate rather than + * drift. `triggers.mjs`'s VALIDATOR is C0 + DEL and decides whether an operator's file is acceptable. + * `env-file.mjs`'s `QUOTED_CONTROL` is this rule almost exactly -- its own docblock says "the bidi + * controls and isolates, the zero-width characters, and the line and paragraph separators" -- and it also + * holds U+200C, U+200D and the variation selectors, because a `.env` VALUE has no legitimate emoji + * sequence in it and any invisible byte there is suspect. This panel renders a CJK repository name and a + * container's log output as a matter of course, so it keeps what composes. `endpointShown` ESCAPES rather + * than substituting, on an allowlist of printable ASCII, which is right for a DNS name or a socket path + * and would escape the content issue #401 had just taught this module to measure. + */ +// A SUBDIVISION FLAG, matched as a WHOLE VALID SEQUENCE rather than guessed at from one end: the base, +// one to six tag letters, and the cancel tag that closes it. A first version exempted "a tag preceded by +// the base and any number of tags", which is not the same claim and is not a validity check at all -- it +// exempted every tag FOREVER AFTER a flag, cancel tag included, so one legitimate flag emoji anywhere in +// a model-writable field restored the entire hidden-message hazard this class exists to close. It was +// also a variable-length lookbehind, which rescans backwards at every position: 200 KB of tags in one +// `.log` line took 2.3 seconds, and `readLogTail` bounds the number of lines but not their length. +const FLAG_SEQUENCE = /\u{1f3f4}[\u{e0020}-\u{e007e}]{1,6}\u{e007f}/u; + +// Composing, so never substituted: EVERY mark, spacing marks included. `\p{Mn}|\p{Me}` was the first +// spelling and it was wrong in a way only this class could expose: issue #401 made a spacing mark measure +// ZERO columns, agreeing with the renderer, so once membership started asking "does it draw nothing" the +// Mc vowel signs of the Indic scripts fell straight into it. `\u0915\u093f` would have been substituted +// to `\u0915 `, which is not revealing a control, it is deleting a vowel. +const COMPOSES = /\p{M}/u; +// A blank that is not U+0020 and draws in ONE column, so a reader cannot tell it from a space. +const BLANK_LIKE = /\p{Zs}|\u2800/u; +// A line or paragraph separator: it is read as a BREAK, whatever width it happens to draw. +const BREAKS = /\p{Zl}|\p{Zp}/u; +// The two places the pinned renderer draws an UNASSIGNED code point as nothing, where the width table +// guesses one column. Measured against the pin rather than reasoned from the standard. +const INVISIBLE_UNASSIGNED = /[\u2065]|[\u{e0000}-\u{e0fff}]/u; + +/** + * IS THIS CODE POINT ONE THE PANEL SUBSTITUTES? The rule, as a predicate, rather than a category list. + * + * A first version derived it from `\p{Cf}|\p{Zl}|\p{Zp}|\p{Zs}` and a review pass showed that is a + * different rule wearing this one's clothes: it left U+FFF0-U+FFF8, which THIS FILE's own width table + * already treats as code points the renderer draws as nothing, and U+2800, which draws a blank + * cell. A test that restates the implementation's own expression cannot catch a wrong rule, which is what + * the round before that got wrong. So membership is asked of the renderer's own answer -- what does this + * DRAW -- and the categories are gone. + */ +function interpreted(ch) { + const cp = ch.codePointAt(0); + // What a terminal EXECUTES. U+009B is a CSI introducer needing no ESC, which is why C1 is here. + if (cp <= 0x1f || cp === 0x7f || (cp >= 0x80 && cp <= 0x9f)) return true; + // PRINTABLE ASCII IS THE ANSWER FOR ALMOST EVERY CHARACTER A PANE EVER HOLDS, and saying so here rather + // than reaching `columnsOf` below is what keeps this affordable: every caller runs it per character over + // whole `.log` lines, and a review pass measured a search over a 200-line tail of 100 KB lines at 2.1 + // SECONDS per render without it. None of U+0021-U+007E is a mark, a blank or a separator. + if (cp < 0x80) return false; + // The space this substitutes TO, and the two joiners, which compose rather than hide. + // + // THE SPACE IS ALREADY COVERED by the ASCII line above, and both are kept on purpose: each makes the + // other's mutation equivalent, which is worth saying so the next reader does not chase either as a gap. + // The fast path exists for cost and the named check for legibility, and deleting the fast path alone + // would silently put U+0020 into the class if this line ever went with it. + if (ch === " " || ch === "\u200c" || ch === "\u200d") return false; + if (COMPOSES.test(ch)) return false; + // BREAKS A LINE, which is an interpretation rather than a drawing, and the one arm that is not about + // what a code point looks like: U+2028 and U+2029 draw ONE column, so neither test below reaches them. + if (BREAKS.test(ch)) return true; + // Draws as NOTHING: the format characters, the bidi controls, the fillers, and the unassigned code + // points the renderer blanks (U+FFF0-U+FFF8 among them, which are RESERVED rather than noncharacters -- + // the real noncharacters, U+FDD0-U+FDEF and the plane-enders, are not in this class and draw a glyph). + // + // THE SECOND TEST IS NOT REDUNDANT, and leaving it out is the defect a third review round found. Asking + // `columnsOf` alone looked like "ask the renderer", and it is not: issue #401's sweep pins that table as + // never NARROWER than the renderer, and its docblock says it deliberately answers ONE for an unassigned + // code point, because guessing wider is the safe direction for a WIDTH. For MEMBERSHIP the safe + // direction is the other one, so that guess is a MISS -- 3,760 of them, measured against the pin: U+2065 + // and the special-purpose plane, which is a hidden-ASCII channel 39 times the size of the tag block this + // file builds a whole sequence matcher for. The variation selectors inside that plane are marks and have + // already been kept above; the tags are settled by sequence. + if (columnsOf(ch) === 0 || INVISIBLE_UNASSIGNED.test(ch)) return true; + // Or draws as a blank a reader cannot tell from a space. U+3000 is deliberately excluded: it draws TWO + // columns, it is an ordinary full-width space in Japanese text, and substituting it would narrow the + // line as well as rewrite the content. The collision it can still make is a stated residual. + return BLANK_LIKE.test(ch) && columnsOf(ch) === 1; +} + +/** + * Walk `s`, hand every substituted code point to `replace`, and keep everything else. + * + * A whole valid flag sequence is taken in one step, which is how a tag can be kept inside one and + * substituted outside one without a lookbehind and without rescanning. + */ +function mapInterpreted(s, replace) { + const text = String(s ?? ""); + let out = ""; + let i = 0; + while (i < text.length) { + if (text.codePointAt(i) === 0x1f3f4) { + const m = FLAG_SEQUENCE.exec(text.slice(i, i + 32)); + if (m && m.index === 0) { + out += m[0]; + i += m[0].length; + continue; + } + } + const ch = String.fromCodePoint(text.codePointAt(i)); + out += interpreted(ch) ? replace : ch; + i += ch.length; + } + return out; +} + +/** Does `s` hold anything this panel would substitute? */ +function anyInterpreted(s) { + return mapInterpreted(s, "\u0000").indexOf("\u0000") !== -1; +} // The ONLY escape sequence a styled line may keep: an SGR run. Anything else that starts with ESC is data // that reached a pane, not decoration this project wrote. @@ -76,9 +258,15 @@ const CONTROL_CHARS = /[\x00-\x1f\x7f-\x9f]/g; // eslint-disable-next-line no-control-regex -- the allowlist half of the class above const STYLE_TOKENS = /\x1b\[[0-9;]*m/g; -/** Remove C0/C1 control characters (shared by `clip` and `makeLineInput`). */ -function stripControls(s) { - return String(s ?? "").replace(CONTROL_CHARS, ""); +/** + * DELETE the class, where `scrubControls` substitutes (shared by `clip` and `makeLineInput`). + * + * Exported for the tail search, which has to compare against BOTH readings: the pane substitutes, the + * search box deletes, and a query that finds nothing either way is the regression issue #402 introduced + * and a review pass measured. + */ +export function stripControls(s) { + return mapInterpreted(s, ""); } /** @@ -105,7 +293,7 @@ function stripControls(s) { * it. The first version of this comment claimed a pin that does not exist. */ export function scrubControls(s) { - return String(s ?? "").replace(CONTROL_CHARS, " "); + return mapInterpreted(s, " "); } /** @@ -155,7 +343,7 @@ export function scrubControlsPerLine(s) { /** Does this string carry one? `search` rather than `.test`, because a `/g` regex carries `lastIndex`. */ export function hasControls(s) { - return String(s ?? "").search(CONTROL_CHARS) !== -1; + return anyInterpreted(s); } /** @@ -207,7 +395,7 @@ export function clipData(line, w) { * WHAT IT COUNTS: * * - zero for a mark (`\p{M}`) and for a format character (`\p{Cf}`), plus the fillers and - * noncharacters the renderer also draws as nothing; + * reserved code points the renderer also draws as nothing; * - two for every code point in `WIDE`, which is that renderer's own double-width set; * - two for a narrow character followed by U+FE0F, which asks for its emoji form, and two for a keycap, * which is one of twelve bases plus U+FE0F plus U+20E3 drawn as a single key; @@ -317,7 +505,7 @@ const VS16 = "\ufe0f"; // of this comment said the opposite, that Mc is excluded because it occupies a column, and both were false // when written. The argument for excluding it came from Unicode; the renderer that draws this pane gives Mc // zero, and between a standard and the thing painting the characters the painter wins. Cf covers the -// zero-width and bidi format characters, and the bracketed tail is the Hangul fillers and the noncharacters +// zero-width and bidi format characters, and the bracketed tail is the Hangul fillers and the reserved code points // the renderer also draws as nothing. const ZERO_WIDTH = /\p{M}|\p{Cf}|[ᅟᅠ᠎ㅤﾠ￰-]/u; @@ -640,11 +828,21 @@ function charAfter(s, at) { export function makeLineInput(initial = "") { // THE SAME RULE AT THE VALUE'S OWN DOORS. `backspace` and `del` were fixed to step by character, and a // review pass pointed out that the constructor, `insert` (which takes a whole PASTE) and `setValue` were - // left open: `stripControls` removes C0 and C1, not half a character. The argument that made the edit-side + // left open: `stripControls` removes the whole class, and it removes half a character too. The argument that made the edit-side // fix necessary applies unchanged here, because `value()` is what gets SAVED, and it also keeps `render` // honest -- `cursor` is an index into this string, so a value with no orphans in it means the window's // offsets and the cursor cannot disagree about what they are counting. - const enter = (text) => dropOrphans([...stripControls(text)]).join(""); + // SUBSTITUTE, NOT DELETE, so this value reads like the pane (issue #402). This box is the LIVE_TAIL + // search, and the pane substitutes the class: deleting here meant an operator who pasted a log line's + // own bytes produced a query matching NEITHER what is drawn nor what is stored. It is safe to substitute + // because the cursor sentinels are added at RENDER, not held in the value, so nothing here depends on + // them vanishing. + // + // `dropOrphans` is belt rather than braces on this path: `scrubControls` does not remove half a pair, it + // substitutes the class, and a lone surrogate is not in the class -- it is removed by `widthSteps` when + // anything measures or cuts this value. Keeping it here means the VALUE never holds one either, which is + // what issue #401 needed of the string that gets saved. + const enter = (text) => dropOrphans([...scrubControls(text)]).join(""); let value = enter(initial); let cursor = value.length; return { diff --git a/admin/test/control-bytes.test.mjs b/admin/test/control-bytes.test.mjs index 313e020..efc8b5a 100644 --- a/admin/test/control-bytes.test.mjs +++ b/admin/test/control-bytes.test.mjs @@ -2,10 +2,18 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; -import { box, clip, clipData, hasControls, scrubControls } from "../src/panel.mjs"; +import { box, clip, clipData, columnsOf, hasControls, scrubControls } from "../src/panel.mjs"; import { frame, makeStyler, PLAIN_THEME, stripAnsi, visibleLen } from "../src/style.mjs"; import { renderRuns, renderTriggers } from "../src/render.mjs"; +/** The TypeScript dashboard, through the loader the other suites use: `dashboard.ts` is not plain ESM. */ +async function tsLoader() { + const { createRequire } = await import("node:module"); + const piRequire = createRequire(import.meta.resolve("@earendil-works/pi-coding-agent")); + const { createJiti } = piRequire("jiti"); + return createJiti(import.meta.url); +} + // ONE CLASS, ONE OPERATION, and this file is where that claim is held (issue #382, item 1). // // Five copies of `[\u0000-\u001f\u007f-\u009f]` lived across four modules and did not agree about what to @@ -15,23 +23,149 @@ import { renderRuns, renderTriggers } from "../src/render.mjs"; // flipping `cell` and `cellOf` to deletion were killed by the suite; nothing noticed a third renderer // already deleting. -test("the class covers C0, DEL and C1, and nothing else up to U+017F", () => { - // A SWEEP rather than a handful of examples, because the boundary is the whole point: U+009B is a CSI - // introducer that needs no ESC in front of it, so a class that stops at DEL leaves a working escape. - for (let cp = 0; cp <= 0x17f; cp++) { +/** + * The pinned renderer's own width, as the oracle for what "draws as nothing" means. + * + * Loaded the way `width.test.mjs` loads it, with the VERSION asserted rather than assumed: pi depends on + * pi-tui by a range, and a class checked against the wrong renderer is not checked. + */ +async function loadVisibleWidth() { + const { createRequire } = await import("node:module"); + const { pathToFileURL } = await import("node:url"); + const pi = createRequire(import.meta.resolve("@earendil-works/pi-coding-agent")); + assert.equal(pi("@earendil-works/pi-tui/package.json").version, "0.80.7", "the oracle must be the pinned renderer"); + const { visibleWidth } = await import(pathToFileURL(pi.resolve("@earendil-works/pi-tui")).href); + assert.equal(typeof visibleWidth, "function", "and it must actually load"); + return visibleWidth; +} + +test("the class IS its rule, swept over every code point there is (#402)", async () => { + const visibleWidth = await loadVisibleWidth(); + // A SWEEP BY THE RULE, and the rule is stated in terms of what the module says it DRAWS rather than by + // restating the class's own expression. That distinction is the correction a review pass forced twice. + // The first version was a hand-written list of 98, where this holds 4,024. The second stated the + // rule as the same four Unicode categories the implementation used, which a reviewer pointed out can + // only catch a typo, never a wrong rule -- and it was still wrong, leaving U+FFF0-U+FFF8 and U+2800, + // which draws a blank cell. A third round found the largest miss of all, 3,760 code points, by noticing + // that asking the width table alone is not asking the renderer: that table GUESSES one column for an + // unassigned code point, which is the safe direction for a width and a miss for membership. + // + // `columnsOf` is the oracle here, and using it is not circular: issue #401 bolts it to the pinned + // renderer over every code point there is, so it is an independently held answer to "what does this + // draw", which is the only question this class actually asks. + // EVERY mark, spacing marks included: issue #401 made those zero columns, so a narrower spelling here + // would put the Indic vowel signs in the class. + const composes = (ch) => ch === "\u200c" || ch === "\u200d" || /\p{M}/u.test(ch); + const blankLike = (ch) => /\p{Zs}|\u2800/u.test(ch); + const breaks = (ch) => /\p{Zl}|\p{Zp}/u.test(ch); + const executes = (cp) => cp <= 0x1f || (cp >= 0x7f && cp <= 0x9f); + let drawsNothing = 0; + let blanks = 0; + for (let cp = 0; cp <= 0x10ffff; cp++) { + if (cp >= 0xd800 && cp <= 0xdfff) continue; + // A TAG is settled by its SEQUENCE rather than by its code point, and is swept in its own test. + if (cp >= 0xe0020 && cp <= 0xe007f) continue; const ch = String.fromCodePoint(cp); - const control = cp <= 0x1f || (cp >= 0x7f && cp <= 0x9f); - assert.equal(hasControls(`a${ch}b`), control, `U+${cp.toString(16).padStart(4, "0")}: hasControls`); - assert.equal(scrubControls(`a${ch}b`), control ? "a b" : `a${ch}b`, `U+${cp.toString(16).padStart(4, "0")}: scrubControls`); - assert.equal(clip(`a${ch}b`, 10), control ? "ab" : `a${ch}b`, `U+${cp.toString(16).padStart(4, "0")}: clip still DELETES`); + // THE RENDERER ITSELF is the oracle for this arm, not a second copy of the implementation's range. + // A review pass pointed out that restating `INVISIBLE_UNASSIGNED` here is the shape this test's own + // comment forbids: it catches a typo and never a wrong bound. `visibleWidth` is the pinned renderer, + // which is the thing the class is trying to agree with, so it is asked directly. + const invisibleUnassigned = visibleWidth(ch) === 0; + let inClass = executes(cp); + if (!inClass && ch !== " " && !composes(ch)) { + // A break is read as a break whatever it draws, which is why it is not a width question. + if (breaks(ch)) { + inClass = true; + } else if (columnsOf(ch) === 0 || invisibleUnassigned) { + inClass = true; + drawsNothing += 1; + } else if (blankLike(ch) && columnsOf(ch) === 1) { + inClass = true; + blanks += 1; + } + } + assert.equal(hasControls(`a${ch}b`), inClass, `U+${cp.toString(16).toUpperCase().padStart(4, "0")}: membership`); + assert.equal(scrubControls(`a${ch}b`), inClass ? "a b" : `a${ch}b`, `U+${cp.toString(16).toUpperCase().padStart(4, "0")}: what is done with it`); } - // `hasControls` uses `search`, not `.test`: a `/g` regex carries `lastIndex` between calls, so the same - // string would answer differently on the second ask. + // Both non-executing arms are real, so neither clause is quietly dead. + // Both counts are pinned rather than merely non-zero, so the class cannot quietly grow or shrink: 85 + // code points draw as nothing: the format characters, the bidi controls, the fillers, and the two + // unassigned regions the renderer draws as nothing where the width table guesses one column. 16 are + // blanks a reader cannot tell from a space. With the 65 a terminal executes and the two line breaks + // that is 3,928, and the tag block adds 96 more, settled by sequence below. + assert.equal(drawsNothing, 3845, `the draws-nothing arm covers ${drawsNothing} code points`); + assert.equal(blanks, 16, `the blank-like arm covers ${blanks}`); + // U+3000 is the stated exclusion: a full-width space is TWO columns and ordinary Japanese text. + assert.equal(hasControls("a\u3000b"), false, "an ideographic space is content, not a control"); + // `hasControls` must answer the same twice: the old implementation was a `/g` regex carrying lastIndex. const dirty = "a\u0007b"; assert.equal(hasControls(dirty), true); assert.equal(hasControls(dirty), true, "and it answers the same the second time"); }); +test("a tag composes a subdivision flag and deceives anywhere else (#402)", () => { + // THE ONE CODE POINT CLASS THAT IS BOTH, so it is the one the class settles by SEQUENCE. After U+1F3F4 + // a tag builds a flag; anywhere else it is invisible text, and a run of them is a whole ASCII message + // at zero columns. + const flag = "\u{1f3f4}\u{e0067}\u{e0062}\u{e0065}\u{e006e}\u{e0067}\u{e007f}"; + assert.equal(scrubControls(flag), flag, "a flag sequence survives whole"); + assert.equal(scrubControls(`job${flag}id`), `job${flag}id`, "including inside other text"); + assert.equal(scrubControls("job\u{e0041}\u{e0042}id"), "job id", "and a bare tag run is substituted"); + assert.equal(scrubControls(`${flag}\u200b`), `${flag} `, "a flag does not exempt what follows it"); + for (let cp = 0xe0020; cp <= 0xe007f; cp++) { + const ch = String.fromCodePoint(cp); + assert.equal(hasControls(`a${ch}b`), true, `U+${cp.toString(16).toUpperCase()} outside a flag`); + } + + // THE EXEMPTION IS A VALIDITY CHECK, NOT A PREFIX, and this is the assertion that says so. A first + // version exempted "a tag preceded by the base and any number of tags", which exempts every tag FOREVER + // AFTER a flag -- cancel tag included -- so one legitimate flag emoji anywhere in a model-writable field + // restored the whole hazard. A review pass recovered `rm -rf /` verbatim from a drawn line that way. + const hide = (msg) => [...msg].map((c) => String.fromCodePoint(0xe0000 + c.charCodeAt(0))).join(""); + const payload = hide("rm -rf /"); + assert.notEqual(scrubControls(`\u{1f3f4}${payload}`), `\u{1f3f4}${payload}`, "a base plus a tag run is not a flag, so the run is substituted"); + assert.notEqual(scrubControls(`${flag}${payload}`), `${flag}${payload}`, "and a COMPLETE flag does not exempt the run after it"); + assert.ok(scrubControls(`${flag}${payload}`).startsWith(flag), "while the flag itself still survives"); + // A run longer than any real subdivision is not a flag either. + const tooLong = `\u{1f3f4}${hide("abcdefgh")}\u{e007f}`; + assert.notEqual(scrubControls(tooLong), tooLong, "a tag run longer than a subdivision code is not a flag"); + + // EVERY TAG OUTSIDE A COMPLETE SEQUENCE GOES, asserted rather than "the result differs". A review pass + // showed what `notEqual` alone buys: making the cancel tag optional left a SIX-character hidden message + // whole, and the assertion above was satisfied by the eight-character one being partly substituted. + for (const around of [`\u{1f3f4}${payload}`, `${flag}${payload}`, `${payload}${flag}`, `a${payload}b`]) { + const scrubbed = scrubControls(around); + const tags = [...scrubbed].filter((c) => c.codePointAt(0) >= 0xe0020 && c.codePointAt(0) <= 0xe007f); + const kept = around.includes(flag) ? 6 : 0; // the six a complete flag is allowed to keep + assert.equal(tags.length, kept, `tags surviving in ${JSON.stringify(around)}`); + } + // AND A FLAG IS NOT DUPLICATED OR ALLOWED TO EAT WHAT SITS BETWEEN TWO OF THEM, which is what dropping + // the match-at-this-position guard did: `base tagA tagB base tagC cancel` came back as the second flag + // twice, with the bytes between them gone. + const twoBases = `\u{1f3f4}${hide("ab")}\u{1f3f4}${hide("gbeng")}\u{e007f}`; + const out = scrubControls(twoBases); + assert.equal([...out].filter((c) => c.codePointAt(0) === 0x1f3f4).length, 2, "both bases survive as themselves"); + assert.ok(out.endsWith(`\u{1f3f4}${hide("gbeng")}\u{e007f}`), "the valid flag is kept whole at the end"); + assert.equal([...out].filter((c) => c.codePointAt(0) >= 0xe0020 && c.codePointAt(0) <= 0xe007f).length, 6, "and only its own tags survive"); +}); + +test("the width table's guess is not the class's answer (#402)", () => { + // THE THIRD ROUND'S BLOCKER. Asking `columnsOf` alone reads like "ask the renderer" and is not: issue + // #401 pins that table as never NARROWER than the renderer and it deliberately answers ONE for an + // unassigned code point, which is the safe direction for a width and a MISS for membership. 3,760 code + // points the renderer draws as nothing were outside the class, a hidden-ASCII channel 39 times the size + // of the tag block this file builds a sequence matcher for. + const hidden = [...("rm -rf /")].map((c) => String.fromCodePoint(0xe0080 + c.charCodeAt(0) - 32)).join(""); + assert.equal(columnsOf(hidden), 8, "the width table guesses one column each, which is why it cannot decide this"); + assert.equal(scrubControls(`job${hidden}id`), `job${" "}id`, "and every one of them is substituted anyway"); + for (const cp of [0x2065, 0xe0000, 0xe0002, 0xe001f, 0xe0080, 0xe00ff, 0xe01f0, 0xe0fff]) { + const ch = String.fromCodePoint(cp); + assert.equal(hasControls(`a${ch}b`), true, `U+${cp.toString(16).toUpperCase()} is invisible at the renderer`); + } + // The variation selectors inside the same plane are MARKS, and stay out for the composing reason. + assert.equal(hasControls("a\u{e0100}b"), false, "a variation selector composes, even in that plane"); +}); + test("clipData substitutes and then clips, so width is what the operator sees", () => { assert.equal(clipData("a\u0001b", 10), "a b", "one byte, one column"); assert.equal(clip("a\u0001b", 10), "ab", "while clip itself still deletes, which panel.test.mjs pins"); @@ -216,3 +350,118 @@ test("every cutter drops a half surrogate, not just `clip`", () => { } } }); + +test("the class draws its line at INTERPRETED against COMPOSING (#402)", () => { + // THE RULE, not the list, because a list is what the carve-out in #382 was and it was wrong twice. A code + // point is in the class when a terminal or a reader INTERPRETS it -- as an escape, a reordering, or a + // break -- and out of it when it COMPOSES the character beside it. + const INTERPRETED = [ + ["U+202E right-to-left override", "‮"], + ["U+202D left-to-right override", "‭"], + ["U+2066 left-to-right isolate", "⁦"], + ["U+2069 pop directional isolate", "⁩"], + ["U+061C arabic letter mark", "؜"], + ["U+200E left-to-right mark", "‎"], + ["U+200F right-to-left mark", "‏"], + ["U+200B zero width space", "​"], + ["U+2060 word joiner", "⁠"], + ["U+FEFF byte order mark", ""], + ["U+00AD soft hyphen", "­"], + ["U+180E mongolian vowel separator", "᠎"], + ["U+2028 line separator", "
"], + ["U+2029 paragraph separator", "
"], + ["U+FFF9 interlinear annotation anchor", ""], + ["U+FFF0, reserved and drawn as nothing", "￰"], + ["U+2800 braille blank, which draws a blank cell", "⠀"], + ["U+00A0 no-break space", " "], + ["U+2007 figure space", " "], + ]; + const COMPOSING = [ + ["U+200D zero width joiner", "‍"], + ["U+200C zero width non-joiner", "‌"], + ["U+FE0F variation selector-16", "️"], + ["U+FE0E variation selector-15", "︎"], + ["U+0301 combining acute", "́"], + ["U+3000 ideographic space, two columns and ordinary Japanese text", "　"], + ]; + for (const [name, ch] of INTERPRETED) { + assert.equal(hasControls(`a${ch}b`), true, `${name} is interpreted, so it is in the class`); + assert.equal(scrubControls(`a${ch}b`), "a b", `${name} becomes one space`); + } + for (const [name, ch] of COMPOSING) { + assert.equal(hasControls(`a${ch}b`), false, `${name} composes, so it is not in the class`); + assert.equal(scrubControls(`a${ch}b`), `a${ch}b`, `${name} passes through untouched`); + } +}); + +test("widening the class does not break a glyph it was not meant to touch (#402)", () => { + // THE COST OF GETTING THE BOUNDARY WRONG, asserted from the other side. A class that swept up every + // zero-width code point would split a family emoji into three and drop the emoji form of a heart, which + // is changing a CHARACTER rather than revealing a CONTROL. + const family = "\u{1f468}‍\u{1f469}‍\u{1f466}"; + assert.equal(scrubControls(family), family, "a family emoji survives whole"); + assert.equal(scrubControls("❤️"), "❤️", "and so does the emoji form of a heart"); + assert.equal(columnsOf(scrubControls("❤️")), 2, "which still measures as the glyph it is"); + // Persian orthography, where the non-joiner is the spelling rather than a control. + assert.equal(scrubControls("می‌خواهم"), "می‌خواهم", "a non-joiner inside a word is content"); +}); + +test("a bidi override cannot make a record read as something else (#402)", () => { + // THE REPRODUCTION FROM THE ISSUE. A target ending in an override plus `gnp.txt` is drawn as a name + // ending in `.png`, and a run record is what an operator reads before deciding what to do about it. + const target = "acme/repo‮gnp.txt"; + assert.equal(scrubControls(target), "acme/repo gnp.txt", "the override becomes a space and the tail reads forwards"); + assert.doesNotMatch(clipData(target, 40), /[‪-‮⁦-⁩]/u, "and no reordering control reaches the pane"); +}); + +test("two identifiers that draw alike cannot stay distinct through the gate (#402)", () => { + // THE SELECTION HAZARD. `deploy-prod` and `deploy` + U+200B + `-prod` are eleven columns each and are + // not the same string, so a picker that selects by the string can act on the row the operator did not + // mean. Substituting makes the difference visible, which is the only honest answer available: the panel + // cannot know which of the two was intended. + const plain = "deploy-prod"; + const hidden = "deploy​-prod"; + assert.notEqual(plain, hidden, "the fixture is two different strings"); + assert.equal(columnsOf(plain), columnsOf(hidden), "which today draw the same width"); + assert.notEqual(scrubControls(plain), scrubControls(hidden), "and after the gate they no longer look alike"); +}); + +test("the tail search finds the line the pane draws (#402)", async () => { + // A REGRESSION THIS CHANGE INTRODUCED AND A REVIEW PASS MEASURED. The pane substitutes the class, the + // search box DELETES it from what the operator types, and the haystack used to be the raw bytes. Once + // the class held the invisible characters that real log output carries, a line drawn as `repo -prod` + // could be found by NO query: what is on screen missed the raw byte, and the original missed because + // the box had dropped it. Both sides are scrubbed now, so what is read is what can be searched. + const { makeLineInput } = await import("../src/panel.mjs"); + const jiti = await tsLoader(); + const { tailMatches } = await jiti.import("../src/dashboard.ts"); + const lines = ["pulled:acme/repo​-prod:ok", "unrelated line", "pulled:acme/other:ok"]; + // What the operator sees, typed back in. The editor drops the invisible byte from the query, exactly as + // it does for anything an operator pastes. + const typed = makeLineInput(""); + typed.insert("repo -prod"); + assert.deepEqual(tailMatches(lines, typed.value()), [0], "typing what is drawn finds it"); + // And the original bytes, pasted, still find it: the haystack is scrubbed, so both spellings land. + const pasted = makeLineInput(""); + pasted.insert("repo​-prod"); + assert.deepEqual(tailMatches(lines, pasted.value()), [0], "pasting the original finds it too"); + assert.deepEqual(tailMatches(lines, "nothing here"), [], "and a miss is still a miss"); +}); + +test("a flag needs at least one tag, and the search window is what keeps this linear (#402)", () => { + // TWO GUARDS THE SUITE DID NOT HOLD, both found by mutating them rather than by reading. + // + // THE LOWER BOUND: a base followed only by the cancel tag is not a subdivision flag, and `{0,6}` kept + // its cancel tag instead of substituting it. + assert.notEqual(scrubControls("\u{1f3f4}\u{e007f}"), "\u{1f3f4}\u{e007f}", "a base and a cancel tag alone is not a flag"); + // THE WINDOW: the matcher looks at a bounded slice because a valid sequence is at most 16 code units. + // Widening it to the rest of the string is output-identical and QUADRATIC, which is the same shape as + // the lookbehind this replaced: 100,000 bare flag bases went from 30 ms to over five seconds. A timing + // assertion would be flaky, so what is pinned is the behaviour that makes the bound safe, which is that + // a sequence longer than the window is not recognised and a valid one at the window's edge still is. + const hide = (msg) => [...msg].map((c) => String.fromCodePoint(0xe0000 + c.charCodeAt(0))).join(""); + const longest = `\u{1f3f4}${hide("abcdef")}\u{e007f}`; + assert.equal([...longest].length, 8, "the longest valid sequence is eight code points"); + assert.equal(scrubControls(longest), longest, "and it is recognised whole"); + assert.equal(scrubControls(`x${longest}y`), `x${longest}y`, "including when it is not at the start"); +}); diff --git a/admin/test/dashboard.test.mjs b/admin/test/dashboard.test.mjs index 44fbd53..e073151 100644 --- a/admin/test/dashboard.test.mjs +++ b/admin/test/dashboard.test.mjs @@ -3043,7 +3043,7 @@ test("the deps layer projects a failed Job to five host-chosen fields -- .data n const line = JSON.stringify(snap.failed); assert.ok(!line.includes("SECRET TITLE") && !line.includes("SECRET BODY") && !line.includes("secret-login") && !line.includes("secretFrame"), "payload and stacktrace stay out of the snapshot"); // C1 is in the class too since issue #337, because `scrubReason` shares `scrubControl` with the - // renderer and that moved to `panel.mjs`'s wider `CONTROL_CHARS`. Nothing this project produces puts a + // renderer and that moved to `panel.mjs`'s wider class, now its `interpreted` predicate. Nothing this project produces puts a // C1 code point in a worker throw's message, so this pins a contract rather than a behaviour. for (const code of [0x1b, 0x07, 0x9b, 0x80]) { assert.ok(!row.failedReason.includes(String.fromCharCode(code)), `control byte ${code} is scrubbed`); diff --git a/admin/test/panel.test.mjs b/admin/test/panel.test.mjs index 481ca3a..8a27d00 100644 --- a/admin/test/panel.test.mjs +++ b/admin/test/panel.test.mjs @@ -263,15 +263,19 @@ test("makeLineInput edits round-trip: insert, cursor moves, backspace, del, setV assert.equal(li.cursor(), 5, "setValue parks the cursor at the end"); }); -test("makeLineInput strips control characters on the way in, typed or pasted", () => { +test("makeLineInput substitutes control characters on the way in, typed or pasted", () => { + // SUBSTITUTES, where it used to delete (issue #402). This box is the LIVE_TAIL search, the pane + // substitutes the class, and deleting here produced a query matching neither what is drawn nor what is + // stored. What has NOT changed is the invariant this test was written for: a sentinel can never enter + // the value, because `render` adds them rather than the value holding them. const li = makeLineInput("a\x00b"); - assert.equal(li.value(), "ab", "the initial value is stripped too"); + assert.equal(li.value(), "a b", "the initial value is scrubbed too"); li.insert("\x07"); - assert.equal(li.value(), "ab", "a lone control char inserts nothing"); + assert.equal(li.value(), "a b ", "a lone control char becomes a space"); li.insert("cd\x1bef"); - assert.equal(li.value(), "abcdef", "a pasted string is stripped, then inserted whole"); + assert.equal(li.value(), "a b cd ef", "a pasted string is scrubbed, then inserted whole"); li.insert(LINE_INPUT_CURSOR[0] + "x" + LINE_INPUT_CURSOR[1]); - assert.equal(li.value(), "abcdefx", "the cursor sentinels are C0 controls and can never enter the value"); + assert.doesNotMatch(li.value(), /[\x01\x02]/, "the cursor sentinels are in the class and can never enter the value"); }); test("makeLineInput render is exactly width columns and windows around the cursor", () => { diff --git a/admin/test/width.test.mjs b/admin/test/width.test.mjs index 0956a60..730008a 100644 --- a/admin/test/width.test.mjs +++ b/admin/test/width.test.mjs @@ -532,7 +532,7 @@ test("a keycap is consumed whole, and never duplicates its own enclosing mark (# test("the editor's value never admits half a character, through any door (#401)", () => { // `backspace` and `del` were fixed to step by character; a review pass found the three doors left open. - // `stripControls` removes C0 and C1, not half a character, and `insert` takes a whole PASTE. The argument + // `stripControls` removes the whole class, and it removes half a character too, and `insert` takes a whole PASTE. The argument // that made the edit-side fix necessary is that `value()` is the string that gets SAVED, and it applies // here unchanged. const lone = /[\ud800-\udbff](?![\udc00-\udfff])|(? `gateDialogs`. | | 2026-09-24 | Issue #403. **`DES-ADMIN-VIA-PI-EXTENSION` AMENDED**: the unframed degrade honours the width it was given. It was a layout promise that held on one branch of an `if` -- the run rows are bounded to their own fixed 24-column pane, everything else was returned as built (the live tail's unframed branch had no clip at all, and said so) -- so a single render showed rows cut to the pane beside section headers, settings lines and hints running past it (measured: 28 of 59 lines over the width at `render(4)`, the widest 52 columns). Same shape as the logs viewer's #399, which this file had already fixed. **A width that is MISSING or non-finite is left alone**, deliberately: the degrade is what those get too, and `clip(x, NaN)` returns the empty string, so inventing a number there would blank the pane rather than degrade it. **The clip is by LENGTH and that rests on a measured assumption**, now pinned: this path is monochrome, zero of its lines carrying an SGR sequence under a real theme, because its own branches strip colour before returning. If that changes the result is worse than lost colour -- `clipData` substitutes the ESC and leaves `[31m` VISIBLE, charging four phantom columns against the width -- and the answer is an ANSI-aware cut. The width remains a UTF-16 count, wrong for CJK exactly as it is everywhere else in this module, which is #401 and is not made worse here. The TRUNCATION is centralised beside the comparison, not only the comparison: a review pass showed that splitting them is invisible, since `Math.round(7.9)` frames nothing and then clips to 8. **Code evidence**: `admin/src/dashboard.ts` -> `renderPanel`, `framedAt`, `degradeWidth`. | | 2026-09-24 | Issue #401. **`DES-ADMIN-VIA-PI-EXTENSION` AMENDED**: the panel measures COLUMNS, and the entry's own sentence saying it does not is what changed. Every width promise in this module was a UTF-16 code-unit count -- `clip`, `pad`, `styler.cell`, `divider`, `clipPlain`, `visibleLen`, both frame builders' top rules, the line editor's window and the model-visible runs table -- so a CJK job id framed a pane this module reported as exactly 80 columns and the renderer drew at 90, and a combining mark ragged it the other way. Measured against the pinned renderer, as this module counted against what it draws: a CJK job id 5 against 10, fullwidth latin 3 against 6, Hangul 5 against 10, combining marks 5 against 3. **ONE TABLE, IN `panel.mjs`, AND THE STYLER COMES TO IT.** Importing pi-tui's `visibleWidth` into `style.mjs` was the alternative, and that module is overlay-only and already depends on pi: rejected because `panel.mjs` owns `clip` and the monochrome renderer, its purity pin forbids it reaching the world at all, and the two renderers draw the same geometry. A width rule holding in the framed pane and not in the plain one is the shape that holds on one branch of an `if`, which #403 had just been about -- and a mutation of the MONOCHROME top rule survived a suite pinning only the coloured one, which is that same shape appearing inside this very fix. **THE CORRECTION THAT MATTERS: the table is TRANSCRIBED FROM THE RENDERER, not written out of UAX #11.** A first version was hand-written from the standard and checked against ten strings. It passed, and it UNDER-counted 139,820 code points -- CJK Extension B through H, Tangut, the Kana supplement, the Hangul Jamo extensions, every emoji block added since Unicode 13 -- so the fix carried the defect it was fixing, on exactly the CJK repository name the issue is about; and in two classes (an astral character, and a character followed by U+FE0F) it was WORSE than the `.length` it replaced, which had been right there by accident because such a sequence is two code units and two columns. Ten examples cannot see that, which is why the bolt is **two sweeps: every code point there is, and every character followed by U+FE0F and by a keycap**. On every one of those the table may never measure NARROWER than the renderer, and every place it measures wider must be a declared departure. **That is per code point and per those pairs, not for every string**, and a third round measured where it stops: the renderer computes a cluster's base after stripping leading non-printing characters, so a cluster BEGINNING with one of 2,616 zero-width characters and continuing with one of exactly four tails has that tail counted twice. It is an UNDER-count and it amplifies with a repeated breaker, so it is filed as #417 rather than left implied. A LONE SURROGATE is a leader of the same kind and that one is fixed here, because the containment was one line: `clip` steps its fitted line rather than returning it unchanged, so an orphan can no longer reach the drawn text after the count has already removed it. The pair sweep exists because a first version of this row CLAIMED it and the test shipped a six-entry list instead, and the hole was one selector further along: a KEYCAP, twelve bases plus U+FE0F plus U+20E3 drawn as one key, measured 1 against the renderer's 2. That was an UNDER-count, so it is fixed rather than stated. The DIRECTION is the whole property, and "over-counting is harmless" would be too kind: BOTH directions rag a frame, differently. An over-count pads a body line as though it were wider than it is, so the line comes out SHORT and the right border sits left of the one above it, contained by the pane -- a review pass measured 200 such lines from ZWJ sequences, Hangul Jamo clusters and Devanagari. An under-count runs the line PAST the pane and past the terminal, where it wraps and takes the border with it, which is the defect this issue names. The sweep is one-sided on purpose, and what is left is the safer of two bad shapes rather than a harmless one. **WHAT IT COUNTS**: zero for a mark and for a format character plus the fillers, two for the renderer's own double-width set, one more when U+FE0F asks for the emoji form of a narrow base, one for everything else including an unassigned code point. Mc is no longer excepted: a first version reasoned from Unicode that a spacing mark occupies a column, and the renderer that draws the pane says otherwise, so the renderer wins. **WHAT IS LEFT IS A CLASS, NOT ONE SHAPE, and saying otherwise was the other thing a review pass refuted**: the table sums steps, so every run the renderer collapses into one glyph comes out wider than it draws -- an emoji ZWJ sequence (6 against 2), a skin-tone modifier, a regional-indicator flag pair, a Hangul jamo cluster and a Devanagari cluster. All of them OVER-count, so all of them draw short inside the border rather than through it; terminals disagree with each other on all of them. **THE ORACLE IS pi ITSELF**, loaded from the pinned installation per `CONST-PI-VERSION-PINNED` with its VERSION asserted, not just its presence, because pi depends on pi-tui by a range and a table pinned to the wrong renderer is not pinned; and asserted to be a function rather than skipped on failure, since a sibling pin made exactly that silent-fallback mistake once. **Also repaired, as the same defect class rather than as scope creep**: the line editor windowed and padded by code units, so a CJK value measured half what the terminal drew and a window edge put a BARE LOW SURROGATE into the live trigger editor; `renderRuns`, the model-visible table, sized its columns by `.length`, so one CJK character in a `local:` target shifted every later column of that row; a cut to zero returned a floating combining mark, and a cut through a ZWJ sequence left the joiner dangling before the ellipsis; and two hostile-string caps in the graph model could split a surrogate pair. **The line editor's EDIT side needed the same fix as its render side**, which is the sharper half: `cursor` moved one CODE UNIT at a time, so two `left`s put it between the halves of an astral pair and the next `backspace` deleted ONE HALF, leaving the other in `value()` -- in the string that gets SAVED, where no amount of rendering can repair it. **`DES-PANEL-SEPARATE-FROM-RECEIVER` UNCHANGED, checked**: nothing about where the panel runs or what it binds moves. **The bidi and zero-width residual is UNCHANGED and still open**: those code points sit outside a control-byte class defined by what a terminal INTERPRETS, and they are a reader-deception question rather than a width one. **AND THE COUNT AND THE CUT NOW WALK ONE STEPPER**, which is the structural half of the repair: a first version put the U+FE0F state inside `columnsOf` alone, and `sliceColumns` and the line editor call it ONE CHARACTER AT A TIME, where a base is 1 and its selector is 0 -- so the cut spent a budget of 2 on a glyph drawn 3 wide and the pane overflowed. One generator yields `{text, cols}` steps, the count sums them and the cut takes whole ones, so the two cannot disagree. **Code evidence**: admin/src/panel.mjs -> columnsOf, widthSteps, WIDE, ZERO_WIDTH, sliceColumns, clip, pad, box, makeLineInput; admin/src/style.mjs -> visibleLen, cell, divider, frame, clipPlain; admin/src/render.mjs -> renderRuns; admin/src/graph-model.mjs -> clipName; admin/test/width.test.mjs. | +| 2026-09-24 | Issue #402. **`DES-ADMIN-VIA-PI-EXTENSION` AMENDED**: the control-byte class draws its line at INTERPRETED against COMPOSING, where it stopped at what a terminal EXECUTES. **The old boundary was the wrong reading of its own rule.** It excluded bidi and zero-width code points as things "a terminal does not INTERPRET", and a terminal does interpret a bidi override: it reorders everything after it, so `repo/safe` plus U+202E plus `gnp.txt` is drawn as a name ending in `.png`, and the job id, the target and the branch are all attacker- or model-writable while a run record is what an operator reads before deciding what to do about it. The other half is invisibility rather than reordering: `deploy-prod` and `deploy` plus U+200B plus `-prod` draw as eleven columns each and are NOT the same trigger, so a picker that selects by the string acts on the row the operator did not mean -- the dialog gate's own ambiguity note, from issue #404, is about exactly this and relies on `#N` prefixes to stay safe. **IN, AND IT IS A RULE RATHER THAN A LIST, which took three attempts to become one**: what a terminal executes (C0, DEL, C1), what breaks a line, what draws NOTHING (the format characters, the bidi controls and isolates, the Hangul and halfwidth fillers, the tag block, and the unassigned regions the renderer blanks), and what draws a BLANK a reader cannot tell from a space (U+00A0 and every other Unicode space, U+2800). 4,024 code points against the 98 the first attempt listed, and that list is a strict SUBSET of this, so nothing it held was let go. **DELIBERATELY OUT**: a code point that COMPOSES the character beside it -- U+200D joins an emoji sequence into one glyph, U+200C is orthography in Persian and the Indic scripts, and a variation selector chooses a character's form and carries a column with it under issue #401. Substituting those changes a CHARACTER where substituting the rest reveals a CONTROL, and a gate that cannot tell them apart corrupts the text it was added to protect; a class that swept up every zero-width code point would split a family emoji into three, which is pinned from that side too. **ONE CLASS AND ONE OPERATION, not a second named operation**, which is the shape the issue left open. The worker ESCAPES for the same question (`endpointShown`, issue #379) and that is NOT an inconsistency to reconcile: its gate is an allowlist of printable ASCII, correct for a DNS name or a socket path, and this panel renders a CJK repository name as a matter of course, so the same allowlist here would escape the content #401 had just taught it to measure. **The width half of the issue was already answered by #401 rather than pending**: none of these corrupts the geometry, because every measurement site scrubs BEFORE it measures. A first version of this row said "every one of these code points measures zero columns now" and that is false of nearly half of them -- by this module's own table 3,843 of the 4,024 measure ONE and only 181 measure zero, while the renderer draws all but 19 of them as nothing, and the gap between those two readings is exactly what the predicate is for -- so the conclusion held and the reason did not. **Substitution is therefore no longer column-preserving**, which it used to be for nothing: every member of the old class measured exactly one column. **What this does NOT answer, stated rather than implied**: substituting makes a difference VISIBLE, it does not say which of two look-alike strings was intended, because the panel cannot know. **`OQ-035` AMENDED with it**, because `scrubReason` shares the class. **The worker's VALIDATOR is UNCHANGED, checked**, for the reason `DES-ONE-SHOT-DISARM-IN-THE-FILE` already records. **Code evidence**: admin/src/panel.mjs -> interpreted, mapInterpreted, FLAG_SEQUENCE, scrubControls, hasControls, stripControls; admin/src/dashboard.ts -> tailMatches; admin/test/control-bytes.test.mjs. | diff --git a/specs/open-questions.md b/specs/open-questions.md index 2745f33..a6900c2 100644 --- a/specs/open-questions.md +++ b/specs/open-questions.md @@ -1385,13 +1385,17 @@ adversarial passes did. should not depend on every future write site holding the enum, when one scrub inside one `show()` holds it structurally. The `failedReason` cap is unchanged; its CLASS is not, and that is recorded here rather than left in a diff: `scrubReason` shares `scrubControl`, so widening the renderer's class widened this - belt's too, from C0 + DEL to C0 + DEL + C1. No `failedReason` this project can produce contains a C1 + belt's too, from C0 + DEL to C0 + DEL + C1 (and again under issue #402). No `failedReason` this project can produce contains a C1 code point (it is a worker throw's message, decoded as UTF-8), so nothing observable moved -- but a contract that moved without a row is the gap this project's own rule exists to close. **Issue #337 asked whether the class needs C1, and the answer turned out to be already written down.** This project has TWO control-byte classes, not one: `triggers.mjs`'s VALIDATOR is C0 + DEL and decides - whether an operator-authored file is acceptable, while `panel.mjs`'s `CONTROL_CHARS` is C0 + DEL + C1, - calls itself "a defensive strip of C0/C1 control chars from untrusted input", and already backs `clip` + whether an operator-authored file is acceptable, while `panel.mjs`'s class was C0 + DEL + C1 + (WIDER SINCE, under issue #402, where it also became a PREDICATE rather than a regex: `interpreted` asks + what a code point draws, so it holds the bidi controls, everything invisible, every blank a reader cannot + tell from a space and the line and paragraph separators, and it excludes what composes a neighbouring + glyph), + is derived from properties since issue #402, and already backs `clip` -- so the PLAIN and ASCII render paths have been stripping C1 out of these same rows all along. The renderer's own scrub was therefore the outlier rather than the convention, and a themed row and a plain row of the same record went through different classes depending on which file the pane used. It is now @@ -1403,7 +1407,10 @@ adversarial passes did. - **Position**: the FAILED panel section renders `failedReason` -- the message of the error the WORKER itself threw. The known payload-bearing sources were fixed at their sites in the same slice (`branch.mjs` answers with a type instead of the forge-payload value; `prepare-local.mjs` basenames - its path), and the display path wears a belt (control bytes REPLACED WITH SPACES -- one column each, so + its path), and the display path wears a belt (control bytes REPLACED WITH SPACES -- one column each for + the C0/DEL/C1 members this said when it was written, and NOT for every one issue #402 added: the renderer + draws all but 19 of that class as nothing, so a substituted line is wider than the raw one and every + measurement site scrubs BEFORE it measures rather than relying on the counts matching; so a belted pane and a plain one clip identically -- a 120-char cap, and the `job_failed` line's own bound). But the channel itself stays free-form: `InfraRetry` messages and any future untagged throw's `error.message` become `failedReason` verbatim, so the no-payload property is held by @@ -1639,3 +1646,4 @@ adversarial passes did. | 2026-09-22 | Issue #367. **`OQ-038` AMENDED**, three rows, all of them measurements this round took rather than questions it opened. The two genuine pre-start refusals that exit **1** and are therefore silent (`-i -t` with no TTY -- BOTH flags, since either alone exits 0, which is the argv the sandbox passes -- and a `DOCKER_CONTEXT` that does not resolve, where the `DOCKER_HOST` spelling of the same mistake exits 125 and IS covered), with the reason widening the code table is NOT the answer and what would be: pre-launch refusals on the model `sandbox-cli.mjs` already uses for a missing TTY. The fixed 2.5 s pause a healthy shell buys when its last command was a typo, since `--entrypoint bash -i` makes `docker run` return bash's 127, which is the stated trade for not letting a real `exec bash failed` go silent. And the fourth manifest-only refusal, `decideSandboxJobUser`, which is deliberately not folded into `sandboxSyncRefusal`: measured across twelve platform strings, one malformed stamp refuses on every one of them EXCEPT darwin and win32, so folding it in would make the panel's `b` depend on the operator's own OS for an identical run. That is the whole argument: "it is async and asks the daemon" is true of the function and NOT of this refusal, which is reached with zero calls to the endpoint resolver, the facts reader and the image preflight. **`OQ-035` AMENDED**, not "unchanged": its text said the belt is what RUN_DETAIL applies, and the reach is now every pane that prints a record, framed and plain, through one `cellOf` in the panel and one `cell` in the plain renderers. #337 recorded the identical widening as an amendment and this row first called it unchanged, which is the same mistake one surface over. The reason it gives for leaving the record's own enum fields alone is untouched and still why this is belt-and-braces rather than a boundary. **`OQ-016` UNCHANGED, checked**: nothing in the suspend-and-hand-over pair moves. | | 2026-09-22 | Issue #375. **`OQ-007` AMENDED** with one event name and one shape that MOVED between the two greps, and it is the naming rule from #337 applied rather than an exception to it: the session reaper now leaves an entry that is not a real directory and says `session_not_reaped {key, reason}`, a VERDICT from a look that succeeded, where `session_reaper_skipped` stays what it has been, the name for something a pass could not establish. Its sibling `sandbox_network_not_reaped` is the shape this follows. Worth the row on its own: a stray FILE in the store used to reach the per-entry catch as an ENOTDIR and appear under `session_reaper_skipped` on every pass, so this narrows that grep as well as widening the verdict one. **`OQ-037` UNCHANGED, checked**: rootless Podman is unaffected, since every file in a key directory is still worker-written. **Code evidence**: `worker/src/session-store.mjs` -> `reapSessions`. | | 2026-09-23 | Issue #382. **`OQ-035` AMENDED**, in its Position bullet: the control-byte class is written ONCE, in `admin/src/panel.mjs`, and every renderer substitutes through it -- `scrubReason` included. The bullet said "stripped" where the operation is a SUBSTITUTION, which is the distinction the whole issue turns on: `clip` deletes and the record cells substitute, so a pane composing with `clip` clipped one column narrower than its twin for the same record. `clip` still deletes, deliberately and for a pinned reason (`LINE_INPUT_CURSOR`'s sentinels are in the same class), and `clipData` is the composition data paths use. What the belt is no longer asked to do alone: a GATE now runs over every finished pane line, so a field that reaches a pane without a belt is still substituted before it is printed. The validator question this could have been read as raising is settled in `DES-ONE-SHOT-DISARM-IN-THE-FILE` rather than here, because a refusal at that writer leaves a one-shot armed and costs a second paid run. | +| 2026-09-24 | Issue #402. **`OQ-035` AMENDED**, in the same Position bullet issue #382 corrected: the renderer's control-byte class is no longer C0 + DEL + C1. It draws its line at INTERPRETED against COMPOSING, so the bidi controls, the invisible break characters (U+200B, U+2060 and the word joiner range, U+FEFF, U+00AD, U+180E) and the line and paragraph separators are in it, while U+200D, U+200C and the variation selectors are deliberately out because they compose the character beside them. **The `failedReason` belt moves with it, and that is why this row exists**: `scrubReason` shares the class, exactly as #382 recorded, so widening the renderer widened this belt again. Nothing observable moves for the same reason as last time -- a `failedReason` is a worker throw's message decoded as UTF-8 and this project produces none carrying a bidi control -- but a contract that moves without a row is the gap this rule exists to close. **The validator is UNCHANGED, checked**: `triggers.mjs` still refuses on C0 + DEL, and widening it was rejected for the reason `DES-ONE-SHOT-DISARM-IN-THE-FILE` already records, that a refusal at that writer leaves a one-shot armed and costs a second paid run. **Code evidence**: admin/src/panel.mjs -> interpreted, mapInterpreted, scrubControls, hasControls; admin/test/control-bytes.test.mjs. |