Fix/linebreaks conversion quotes

src/annotator/anchoring/pdf.ts

@@ -71,6 +71,10 @@ function quotePositionCacheKey(quote: string, pos?: number) {
   return `${quote}:${pos}`;
 }
 
+function normalizePDFText(str: string): string {
+  return str.replace(/[\r\n]+/g, ' ').replace(/\s+/g, ' ');
+}
+
 /**
  * Return the text layer element of the PDF page containing `node`.
  */
@@ -422,64 +426,61 @@ async function anchorQuote(
     });
   }
 
-  // Search pages for the best match, ignoring whitespace differences.
-  const strippedPrefix =
+  // Normalize quote and context consistently with selector creation.
+  const normalizedPrefix =
     quoteSelector.prefix !== undefined
-      ? stripSpaces(quoteSelector.prefix)
+      ? normalizePDFText(quoteSelector.prefix)
       : undefined;
-  const strippedSuffix =
+  const normalizedSuffix =
     quoteSelector.suffix !== undefined
-      ? stripSpaces(quoteSelector.suffix)
+      ? normalizePDFText(quoteSelector.suffix)
       : undefined;
-  const strippedQuote = stripSpaces(quoteSelector.exact);
+  const normalizedQuote = normalizePDFText(quoteSelector.exact);
 
   let bestMatch;
   for (const page of pageIndexes) {
     const text = await getPageTextContent(page);
-    const strippedText = stripSpaces(text);
+    const normalizedText = normalizePDFText(text);
 
     // Determine expected offset of quote in current page based on position hint.
-    let strippedHint;
+    let normalizedHint;
     if (expectedPageIndex !== undefined && expectedOffsetInPage !== undefined) {
       if (page < expectedPageIndex) {
-        strippedHint = strippedText.length; // Prefer matches closer to end of page.
+        normalizedHint = normalizedText.length; // Prefer matches closer to end of page.
       } else if (page === expectedPageIndex) {
-        // Translate expected offset in whitespace-inclusive version of page
-        // text into offset in whitespace-stripped version of page text.
-        [strippedHint] = translateOffsets(
+        // Translate expected offset in original text into offset in normalized text.
+        [normalizedHint] = translateOffsets(
           text,
-          strippedText,
+          normalizedText,
           expectedOffsetInPage,
           expectedOffsetInPage,
-          isNotSpace,
-          // We don't need to normalize here since both input strings are
-          // derived from the same input.
-          { normalize: false },
+          char => char !== ' ',
+          { normalize: true },
         );
       } else {
-        strippedHint = 0; // Prefer matches closer to start of page.
+        normalizedHint = 0; // Prefer matches closer to start of page.
       }
     }
 
-    const match = matchQuote(strippedText, strippedQuote, {
-      prefix: strippedPrefix,
-      suffix: strippedSuffix,
-      hint: strippedHint,
+    const match = matchQuote(normalizedText, normalizedQuote, {
+      prefix: normalizedPrefix,
+      suffix: normalizedSuffix,
+      hint: normalizedHint,
     });
 
     if (!match) {
       continue;
     }
 
     if (!bestMatch || match.score > bestMatch.match.score) {
-      // Translate match offset from whitespace-stripped version of page text
-      // back to original text.
+      // Translate match offset from normalized text back to original text.
       const [start, end] = translateOffsets(
-        strippedText,
+        normalizedText,
         text,
         match.start,
         match.end,
-        isNotSpace,
+        char => char !== ' ',
+        { normalize: true },
       );
       bestMatch = {
         page,
@@ -500,21 +501,22 @@ async function anchorQuote(
       // helps to avoid incorrectly stopping the search early if the quote is
       // a word or phrase that is common in the document.
       const exactQuoteMatch =
-        strippedText.slice(match.start, match.end) === strippedQuote;
+        normalizedText.slice(match.start, match.end) === normalizedQuote;
 
       const exactPrefixMatch =
-        strippedPrefix !== undefined &&
-        strippedText.slice(
-          Math.max(0, match.start - strippedPrefix.length),
+        normalizedPrefix !== undefined &&
+        normalizedText.slice(
+          Math.max(0, match.start - normalizedPrefix.length),
           match.start,
-        ) === strippedPrefix;
+        ) === normalizedPrefix;
 
       const exactSuffixMatch =
-        strippedSuffix !== undefined &&
-        strippedText.slice(match.end, strippedSuffix.length) === strippedSuffix;
+        normalizedSuffix !== undefined &&
+        normalizedText.slice(match.end, match.end + normalizedSuffix.length) ===
+          normalizedSuffix;
 
       const hasContext =
-        strippedPrefix !== undefined || strippedSuffix !== undefined;
+        normalizedPrefix !== undefined || normalizedSuffix !== undefined;
 
       if (
         exactQuoteMatch &&
@@ -795,7 +797,20 @@ export async function describe(range: Range): Promise<Selector[]> {
     end: pageOffset + endPos.offset,
   } as TextPositionSelector;
 
-  const quote = TextQuoteAnchor.fromRange(pageView.div, textRange).toSelector();
+  const rawQuote = TextQuoteAnchor.fromRange(
+    pageView.div,
+    textRange,
+  ).toSelector();
+  const quote: TextQuoteSelector = {
+    ...rawQuote,
+    exact: normalizePDFText(rawQuote.exact).trim(),
+    prefix: rawQuote.prefix
+      ? normalizePDFText(rawQuote.prefix).trim()
+      : rawQuote.prefix,
+    suffix: rawQuote.suffix
+      ? normalizePDFText(rawQuote.suffix).trim()
+      : rawQuote.suffix,
+  };
   const pageSelector = createPageSelector(pageView, startPageIndex);
 
   return [position, quote, pageSelector];

src/annotator/anchoring/rendered-text.ts

@@ -0,0 +1,178 @@
+const BLOCK_TAGS = new Set([
+  'ADDRESS',
+  'ARTICLE',
+  'ASIDE',
+  'BLOCKQUOTE',
+  'DIV',
+  'DL',
+  'FIELDSET',
+  'FIGCAPTION',
+  'FIGURE',
+  'FOOTER',
+  'FORM',
+  'H1',
+  'H2',
+  'H3',
+  'H4',
+  'H5',
+  'H6',
+  'HEADER',
+  'HR',
+  'LI',
+  'MAIN',
+  'NAV',
+  'OL',
+  'P',
+  'PRE',
+  'SECTION',
+  'TABLE',
+  'TD',
+  'TH',
+  'TR',
+  'UL',
+]);
+
+const SPACE = ' ';
+
+/** Collapse any run of whitespace (including newlines) to a single space. */
+export function collapseWhitespace(text: string): string {
+  return text.replace(/\s+/g, SPACE);
+}
+
+function isBlock(node: Node): boolean {
+  return node.nodeType === Node.ELEMENT_NODE && BLOCK_TAGS.has(node.nodeName);
+}
+
+type RenderedText = {
+  text: string;
+  rawToNorm: number[];
+  normToRaw: number[];
+};
+
+function appendNormalizedChar(
+  ch: string,
+  rawIndex: number,
+  state: {
+    output: string;
+    rawToNorm: number[];
+    normToRaw: number[];
+  },
+) {
+  const isWs = /\s/.test(ch);
+  if (isWs) {
+    if (state.output.length === 0 || state.output.endsWith(SPACE)) {
+      return;
+    }
+    ch = SPACE;
+  }
+
+  state.output += ch;
+  state.normToRaw.push(rawIndex);
+  if (state.rawToNorm[rawIndex] === undefined) {
+    state.rawToNorm[rawIndex] = state.normToRaw.length - 1;
+  }
+}
+
+function buildRenderedText(root: Element): RenderedText {
+  const rawText = root.textContent ?? '';
+  const state = {
+    output: '',
+    rawToNorm: Array(rawText.length + 1).fill(undefined) as number[],
+    normToRaw: [] as number[],
+  };
+
+  let rawPos = 0;
+
+  const appendSpace = () => {
+    appendNormalizedChar(SPACE, rawPos, state);
+  };
+
+  const walk = (node: Node) => {
+    if (node.nodeType === Node.TEXT_NODE) {
+      const text = node.textContent ?? '';
+      for (let i = 0; i < text.length; i++) {
+        appendNormalizedChar(text[i], rawPos, state);
+        rawPos += 1;
+      }
+      return;
+    }
+
+    if (node.nodeType === Node.ELEMENT_NODE) {
+      const el = node as Element;
+
+      if (el.tagName === 'BR') {
+        appendSpace();
+        return;
+      }
+
+      const block = isBlock(el);
+      if (block) {
+        appendSpace();
+      }
+
+      for (const child of Array.from(node.childNodes)) {
+        walk(child);
+      }
+
+      if (block) {
+        appendSpace();
+      }
+    }
+  };
+
+  walk(root);
+
+  // Map end-of-string raw offset.
+  if (state.rawToNorm[rawPos] === undefined) {
+    state.rawToNorm[rawPos] = state.normToRaw.length;
+  }
+  state.normToRaw.push(rawPos);
+
+  const text = state.output; // Already normalized/collapsed
+  return { text, rawToNorm: state.rawToNorm, normToRaw: state.normToRaw };
+}
+
+function translateRawToNorm(rawToNorm: number[], rawOffset: number): number {
+  if (rawOffset < 0) {
+    return 0;
+  }
+  const clamped = Math.min(rawOffset, rawToNorm.length - 1);
+  for (let i = clamped; i >= 0; i--) {
+    const val = rawToNorm[i];
+    if (val !== undefined) {
+      return val;
+    }
+  }
+  return 0;
+}
+
+function translateNormToRaw(normToRaw: number[], normOffset: number): number {
+  if (normOffset <= 0) {
+    return 0;
+  }
+  const clamped = Math.min(normOffset, normToRaw.length - 1);
+  return normToRaw[clamped];
+}
+
+export function renderedTextFromRange(range: Range): string {
+  const container = range.cloneContents();
+  const div = document.createElement('div');
+  div.appendChild(container);
+  const { text } = buildRenderedText(div);
+  return text;
+}
+
+export function renderedTextWithOffsets(root: Element): {
+  text: string;
+  rawToNorm: number[];
+  normToRaw: number[];
+  toNorm: (rawOffset: number) => number;
+  toRaw: (normOffset: number) => number;
+} {
+  const rendered = buildRenderedText(root);
+  return {
+    ...rendered,
+    toNorm: rawOffset => translateRawToNorm(rendered.rawToNorm, rawOffset),
+    toRaw: normOffset => translateNormToRaw(rendered.normToRaw, normOffset),
+  };
+}

src/annotator/anchoring/test/html-baselines/minimal.json

@@ -15,8 +15,8 @@
         "end": 7
       },{
         "type": "TextQuoteSelector",
-        "prefix": "One ",
-        "suffix": " Three\n",
+        "prefix": "One",
+        "suffix": "Three",
         "exact": "Two"
       }]
     }]

src/annotator/anchoring/test/html-baselines/wikipedia-regression-testing.json

@@ -25,10 +25,10 @@
                      "end" : 11897
                   },
                   {
-                     "suffix" : ", at 05:56.\n\t\t\t\t\t\t\t\t\t\t\tText is a",
+                     "suffix" : ", at 05:56. Text is available un",
                      "exact" : "11 November 2016",
                      "type" : "TextQuoteSelector",
-                     "prefix" : " This page was last modified on "
+                     "prefix" : "This page was last modified on"
                   }
                ]
             }
@@ -164,7 +164,7 @@
                      "end" : 7667
                   },
                   {
-                     "prefix" : " execute the test suite cases.\n\n",
+                     "prefix" : "d execute the test suite cases.\n\n",
                      "suffix" : ". Theoretically, after each fix ",
                      "exact" : "Also as a consequence of the introduction of new bugs, program maintenance requires far more system testing per statement written than any other programming",
                      "type" : "TextQuoteSelector"
@@ -254,7 +254,7 @@
                   {
                      "prefix" : "\t\n\t\t\n\t\t\n\t\t\t\n\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\n\t\t\t",
                      "exact" : "Regression testing",
-                     "suffix" : "\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tFrom Wikiped",
+                     "suffix" : "\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tFrom Wikipedia, the free encycl",
                      "type" : "TextQuoteSelector"
                   }
                ],