|
| 1 | +import { afterEach, describe, expect, test, vi } from "vitest"; |
| 2 | + |
| 3 | +import { countGraphemes, segmentGraphemes } from "../graphemes"; |
| 4 | +import { prepareText } from "../index"; |
| 5 | + |
| 6 | +const FONT_KEY = "Inter-400-14"; |
| 7 | + |
| 8 | +// MAN ZWJ WOMAN ZWJ GIRL ZWJ BOY — 7 code points, 11 code units, 1 grapheme. |
| 9 | +const FAMILY = "\u{1F468}\u{1F469}\u{1F467}\u{1F466}"; |
| 10 | +// Regional indicator pairs. Two code points, 4 code units, 1 grapheme each — |
| 11 | +// and the pairing is *positional*, so a mis-sliced run changes the count. |
| 12 | +const FLAG_US = "\u{1F1FA}\u{1F1F8}"; |
| 13 | +const FLAG_JP = "\u{1F1EF}\u{1F1F5}"; |
| 14 | +// THUMBS UP + EMOJI MODIFIER FITZPATRICK TYPE-4. 2 code points, 4 code units, |
| 15 | +// 1 grapheme. |
| 16 | +const THUMBS_UP_SKIN = "\u{1F44D}\u{1F3FD}"; |
| 17 | +// COMBINING ACUTE ACCENT. One code unit that attaches to whatever precedes it |
| 18 | +// — including, and this is the trap, a *space*. |
| 19 | +const ACUTE = "́"; |
| 20 | +const DIAERESIS = "̈"; |
| 21 | + |
| 22 | +/** |
| 23 | + * The corpus the length mapping is held to. |
| 24 | + * |
| 25 | + * Every entry is here to distinguish a correct grapheme→token mapping from a |
| 26 | + * plausible wrong one; see the "the corpus can actually disprove" test below, |
| 27 | + * which fails if the corpus stops exercising the traps. |
| 28 | + */ |
| 29 | +const CORPUS: readonly string[] = [ |
| 30 | + // Degenerate inputs. |
| 31 | + "", |
| 32 | + " ", |
| 33 | + " ", |
| 34 | + "\n", |
| 35 | + "\n\n", |
| 36 | + " \n \n ", |
| 37 | + "\t ", |
| 38 | + // Plain ASCII: the control. A wrong mapping must NOT show up here, which is |
| 39 | + // why the emoji entries exist. |
| 40 | + "alpha beta gamma", |
| 41 | + "alpha beta gamma delta epsilon zeta eta theta", |
| 42 | + "-hyphen/slash_underscore-", |
| 43 | + // A single token that IS one multi-code-unit cluster. |
| 44 | + FAMILY, |
| 45 | + FLAG_US, |
| 46 | + THUMBS_UP_SKIN, |
| 47 | + // Clusters at, inside, and around token boundaries. |
| 48 | + `${FAMILY} ${FAMILY}`, |
| 49 | + `${FAMILY} caf${ACUTE}`, |
| 50 | + `word${FAMILY}word`, |
| 51 | + `${THUMBS_UP_SKIN} ok`, |
| 52 | + `${FLAG_US}${FLAG_JP} flags`, |
| 53 | + // Three flags back to back: six regional indicators whose clustering depends |
| 54 | + // on pair *parity*, so slicing the run anywhere changes the answer. |
| 55 | + `${FLAG_US}${FLAG_US}${FLAG_US}`, |
| 56 | + `${FLAG_US} ${FLAG_US}${FLAG_JP}`, |
| 57 | + // Combining marks on letters. |
| 58 | + `cafe${ACUTE} nai${DIAERESIS}ve`, |
| 59 | + `e${ACUTE}${ACUTE}e`, |
| 60 | + // THE STRADDLE: `" " + U+0301` is ONE grapheme cluster but TWO tokens, because |
| 61 | + // the tokenizer splits on whitespace and the segmenter does not. |
| 62 | + `a ${ACUTE}b`, |
| 63 | + ` ${ACUTE}`, |
| 64 | + `${ACUTE}${ACUTE} ${ACUTE}`, |
| 65 | + `a ${ACUTE}${ACUTE}b`, |
| 66 | + // CRLF, which `prepareText` normalises before anything else touches it. |
| 67 | + "x\r\ny", |
| 68 | + `${FAMILY}\r\n${FLAG_US}`, |
| 69 | + "a\r\n\r\nb", |
| 70 | + // A lone CR is *not* normalised: it stays inline whitespace. |
| 71 | + "a\rb", |
| 72 | + // Mixed, long enough to have many tokens. |
| 73 | + `The ${FAMILY} family ate caf${ACUTE}e at ${FLAG_JP}\nand said ${THUMBS_UP_SKIN} twice.`, |
| 74 | +]; |
| 75 | + |
| 76 | +function prepare(text: string) { |
| 77 | + return prepareText({ text, fontKey: FONT_KEY }); |
| 78 | +} |
| 79 | + |
| 80 | +describe("token lengths derive from the single grapheme pass", () => { |
| 81 | + /** |
| 82 | + * The safety net for removing the second segmentation pass. |
| 83 | + * |
| 84 | + * Before this change every token's `length` was literally |
| 85 | + * `countGraphemes(token.value)`. That is now derived from the whole-string |
| 86 | + * segmentation instead, so this asserts the derivation reproduces the old |
| 87 | + * definition exactly, for every token of every corpus entry. |
| 88 | + */ |
| 89 | + test("every token length equals countGraphemes(token.value), the pre-change definition", () => { |
| 90 | + const mismatches: string[] = []; |
| 91 | + |
| 92 | + for (const text of CORPUS) { |
| 93 | + const prepared = prepare(text); |
| 94 | + |
| 95 | + prepared.tokens.forEach((token, index) => { |
| 96 | + // Newlines have never carried a length. |
| 97 | + const expected = |
| 98 | + token.kind === "newline" ? 0 : countGraphemes(token.value); |
| 99 | + |
| 100 | + if (token.length !== expected) { |
| 101 | + mismatches.push( |
| 102 | + `${JSON.stringify(text)} token#${String(index)} ${JSON.stringify( |
| 103 | + token.value, |
| 104 | + )} (${token.kind}): got ${String(token.length)}, want ${String( |
| 105 | + expected, |
| 106 | + )}`, |
| 107 | + ); |
| 108 | + } |
| 109 | + }); |
| 110 | + } |
| 111 | + |
| 112 | + expect(mismatches).toEqual([]); |
| 113 | + }); |
| 114 | + |
| 115 | + /** |
| 116 | + * A corpus that only holds ASCII would pass the test above no matter how the |
| 117 | + * mapping was written, because code units, code points and graphemes all |
| 118 | + * coincide there. This fails if that ever becomes true of the corpus. |
| 119 | + */ |
| 120 | + test("the corpus can actually disprove a code-unit mapping", () => { |
| 121 | + const tokensWhereCodeUnitsDiffer: string[] = []; |
| 122 | + const straddlingEntries: string[] = []; |
| 123 | + |
| 124 | + for (const text of CORPUS) { |
| 125 | + const prepared = prepare(text); |
| 126 | + |
| 127 | + for (const token of prepared.tokens) { |
| 128 | + if (token.kind !== "newline" && token.value.length !== token.length) { |
| 129 | + tokensWhereCodeUnitsDiffer.push(token.value); |
| 130 | + } |
| 131 | + } |
| 132 | + |
| 133 | + // A straddle is detectable from the outside: the token lengths sum to |
| 134 | + // MORE than the string's grapheme count, because one cluster was charged |
| 135 | + // to two tokens. |
| 136 | + const summed = prepared.tokens.reduce( |
| 137 | + (total, token) => total + token.length, |
| 138 | + 0, |
| 139 | + ); |
| 140 | + const newlines = prepared.tokens.filter( |
| 141 | + (token) => token.kind === "newline", |
| 142 | + ).length; |
| 143 | + |
| 144 | + if (summed + newlines > prepared.graphemeCount) { |
| 145 | + straddlingEntries.push(text); |
| 146 | + } |
| 147 | + } |
| 148 | + |
| 149 | + // Emoji, flags and combining marks: counting code units would be wrong here. |
| 150 | + expect(tokensWhereCodeUnitsDiffer.length).toBeGreaterThan(10); |
| 151 | + // And at least one entry has a cluster spanning a token boundary. |
| 152 | + expect(straddlingEntries.length).toBeGreaterThan(0); |
| 153 | + }); |
| 154 | + |
| 155 | + test("token values still tile the prepared text exactly", () => { |
| 156 | + for (const text of CORPUS) { |
| 157 | + const prepared = prepare(text); |
| 158 | + |
| 159 | + expect(prepared.tokens.map((token) => token.value).join("")).toBe( |
| 160 | + prepared.text, |
| 161 | + ); |
| 162 | + } |
| 163 | + }); |
| 164 | + |
| 165 | + test("token kinds are unchanged by the rewrite", () => { |
| 166 | + expect( |
| 167 | + prepare(`a\t ${FAMILY}\nb`).tokens.map((token) => token.kind), |
| 168 | + ).toEqual(["word", "space", "word", "newline", "word"]); |
| 169 | + |
| 170 | + // A lone CR is inline whitespace, not a newline. |
| 171 | + expect(prepare("a\rb").tokens.map((token) => token.kind)).toEqual([ |
| 172 | + "word", |
| 173 | + "space", |
| 174 | + "word", |
| 175 | + ]); |
| 176 | + }); |
| 177 | +}); |
| 178 | + |
| 179 | +describe("the named grapheme traps, with their expected numbers spelled out", () => { |
| 180 | + test("a ZWJ family emoji is one grapheme, alone and inside a word", () => { |
| 181 | + expect(prepare(FAMILY).tokens).toEqual([ |
| 182 | + { kind: "word", value: FAMILY, length: 1 }, |
| 183 | + ]); |
| 184 | + |
| 185 | + expect(prepare(`word${FAMILY}word`).tokens.map((t) => t.length)).toEqual([ |
| 186 | + 9, |
| 187 | + ]); |
| 188 | + }); |
| 189 | + |
| 190 | + test("regional indicators pair up — three flags are three graphemes, not six", () => { |
| 191 | + expect(prepare(`${FLAG_US}${FLAG_US}${FLAG_US}`).tokens[0]?.length).toBe(3); |
| 192 | + expect(prepare(`${FLAG_US}${FLAG_JP} flags`).tokens.map((t) => t.length)) // |
| 193 | + .toEqual([2, 1, 5]); |
| 194 | + }); |
| 195 | + |
| 196 | + test("a skin-tone modifier does not add a character", () => { |
| 197 | + expect(prepare(`${THUMBS_UP_SKIN} ok`).tokens.map((t) => t.length)).toEqual( |
| 198 | + [1, 1, 2], |
| 199 | + ); |
| 200 | + }); |
| 201 | + |
| 202 | + test("combining marks attach to the letter before them", () => { |
| 203 | + expect(prepare(`cafe${ACUTE} nai${DIAERESIS}ve`).tokens).toEqual([ |
| 204 | + { kind: "word", value: `cafe${ACUTE}`, length: 4 }, |
| 205 | + { kind: "space", value: " ", length: 1 }, |
| 206 | + { kind: "word", value: `nai${DIAERESIS}ve`, length: 5 }, |
| 207 | + ]); |
| 208 | + }); |
| 209 | + |
| 210 | + test("a cluster straddling a token boundary is charged to both tokens", () => { |
| 211 | + // `" " + U+0301` is ONE cluster spanning TWO tokens. Both sides count it, |
| 212 | + // which is what per-token `countGraphemes` used to produce: it saw `" "` |
| 213 | + // and `"́b"` as independent strings, 1 and 2 graphemes. |
| 214 | + const prepared = prepare(`a ${ACUTE}b`); |
| 215 | + |
| 216 | + expect(prepared.graphemeCount).toBe(3); |
| 217 | + expect(prepared.tokens).toEqual([ |
| 218 | + { kind: "word", value: "a", length: 1 }, |
| 219 | + { kind: "space", value: " ", length: 1 }, |
| 220 | + { kind: "word", value: `${ACUTE}b`, length: 2 }, |
| 221 | + ]); |
| 222 | + expect(segmentGraphemes(`a ${ACUTE}b`)).toEqual(["a", ` ${ACUTE}`, "b"]); |
| 223 | + }); |
| 224 | + |
| 225 | + test("whitespace-only text produces one space token of its grapheme count", () => { |
| 226 | + expect(prepare(" ").tokens).toEqual([ |
| 227 | + { kind: "space", value: " ", length: 3 }, |
| 228 | + ]); |
| 229 | + expect(prepare("").tokens).toEqual([]); |
| 230 | + }); |
| 231 | + |
| 232 | + test("CRLF is still normalised to LF before segmentation or tokenizing", () => { |
| 233 | + const prepared = prepare("x\r\ny"); |
| 234 | + |
| 235 | + expect(prepared.text).toBe("x\ny"); |
| 236 | + expect(prepared.text).not.toContain("\r"); |
| 237 | + expect(prepared.tokens).toEqual([ |
| 238 | + { kind: "word", value: "x", length: 1 }, |
| 239 | + { kind: "newline", value: "\n", length: 0 }, |
| 240 | + { kind: "word", value: "y", length: 1 }, |
| 241 | + ]); |
| 242 | + // Unnormalised, "\r\n" is a single grapheme cluster, so this would be 3. |
| 243 | + expect(prepared.graphemeCount).toBe(3); |
| 244 | + |
| 245 | + const emoji = prepare(`${FAMILY}\r\n${FLAG_US}`); |
| 246 | + |
| 247 | + expect(emoji.tokens.map((token) => token.length)).toEqual([1, 0, 1]); |
| 248 | + }); |
| 249 | +}); |
| 250 | + |
| 251 | +describe("the second segmentation pass is gone", () => { |
| 252 | + afterEach(() => { |
| 253 | + vi.unstubAllGlobals(); |
| 254 | + vi.resetModules(); |
| 255 | + }); |
| 256 | + |
| 257 | + /** |
| 258 | + * The perf pin. `prepareText` used to segment the whole string once and then |
| 259 | + * re-segment it token by token via `countGraphemes`; on an S2 wrapped-text |
| 260 | + * scroll that duplicate pass was 156ms of a 748ms window. One `segment()` |
| 261 | + * call per `prepareText` is the invariant that keeps it gone. |
| 262 | + */ |
| 263 | + test("prepareText segments each string exactly once, however many tokens it has", async () => { |
| 264 | + vi.resetModules(); |
| 265 | + |
| 266 | + const RealSegmenter = Intl.Segmenter; |
| 267 | + const segmented: string[] = []; |
| 268 | + const CountingSegmenter = function ( |
| 269 | + this: unknown, |
| 270 | + ...args: ConstructorParameters<typeof Intl.Segmenter> |
| 271 | + ) { |
| 272 | + const real = new RealSegmenter(...args); |
| 273 | + |
| 274 | + return { |
| 275 | + segment: (input: string) => { |
| 276 | + segmented.push(input); |
| 277 | + |
| 278 | + return real.segment(input); |
| 279 | + }, |
| 280 | + }; |
| 281 | + } as unknown as typeof Intl.Segmenter; |
| 282 | + |
| 283 | + vi.stubGlobal("Intl", { ...Intl, Segmenter: CountingSegmenter }); |
| 284 | + |
| 285 | + const { prepareText: prepareTextFresh } = await import("../prepare-text"); |
| 286 | + |
| 287 | + const text = `The ${FAMILY} family ate caf${ACUTE}e in ${FLAG_JP} today`; |
| 288 | + const prepared = prepareTextFresh({ text, fontKey: FONT_KEY }); |
| 289 | + |
| 290 | + // Fifteen tokens (eight words, seven spaces), one segmentation — of the |
| 291 | + // whole string, not of a token. Before the change this was sixteen calls. |
| 292 | + expect(prepared.tokens.length).toBe(15); |
| 293 | + expect(segmented).toEqual([text]); |
| 294 | + }); |
| 295 | +}); |
0 commit comments