From 2badef407ba03baf782d15c9c2a6ff52635d59aa Mon Sep 17 00:00:00 2001 From: feruzm Date: Fri, 21 Aug 2026 09:40:35 +0000 Subject: [PATCH] Detect a feed card's language on the server instead of in every browser The Translate chip on a card needs the content language. Every visitor's browser detected it: one markdown render per card plus the franc-min detector chunk (about 47 KB gzipped), on idle after every feed and post view, for the majority of readers whose language matches the content. The slim step already derives each card's summary on the server, so the server now detects the language from that same summary once per fetch and ships it as slim.lang (ISO-639-1, or null when the text is too short or the detector is unsure). The sample is rendered to plain text with the same bounds the client applied, so an author description that is only an image link or markup counts as no text rather than as a language. The client gate reads the hint, caches it under the summary key as before and skips both the render and the chunk. Rows the browser fetches itself (later pages of an infinite feed) carry no hint and keep the on-idle path, and the feed poll's merge preserves the hint on rows it refreshes; the post page keeps detecting on the full body. The detector is imported lazily inside the server branch only, so it never enters a client bundle, and a failing detector leaves the rows without a hint rather than failing the query. Closes #1597 --- .../queries/get-account-posts-feed-query.ts | 3 +- .../feed/_components/feed-layout.tsx | 5 +- apps/web/src/core/entries/language-hint.ts | 116 ++++++++++++++++ apps/web/src/core/entries/slim-entry.ts | 7 +- apps/web/src/entities/entries.ts | 7 +- .../use-content-language-gate.ts | 13 +- .../core/entries/language-hint-client.spec.ts | 22 ++++ .../specs/core/entries/language-hint.spec.ts | 124 ++++++++++++++++++ .../entry-translate-language-gate.spec.tsx | 53 ++++++++ 9 files changed, 343 insertions(+), 7 deletions(-) create mode 100644 apps/web/src/core/entries/language-hint.ts create mode 100644 apps/web/src/specs/core/entries/language-hint-client.spec.ts create mode 100644 apps/web/src/specs/core/entries/language-hint.spec.ts diff --git a/apps/web/src/api/queries/get-account-posts-feed-query.ts b/apps/web/src/api/queries/get-account-posts-feed-query.ts index 258cbbea98..058a7c3b77 100644 --- a/apps/web/src/api/queries/get-account-posts-feed-query.ts +++ b/apps/web/src/api/queries/get-account-posts-feed-query.ts @@ -6,6 +6,7 @@ import { appAxios } from "@/api/axios"; import { apiBase } from "@/api/helper"; import { DEFAULT_OBSERVER } from "@/consts/observer"; import { slimEntryPage, withSlimEntries } from "@/core/entries/slim-entry"; +import { annotateLanguageHints } from "@/core/entries/language-hint"; // Unify all branches on a single page type type Page = Entry[] | SearchResponse; @@ -27,7 +28,7 @@ function getPromotedEntriesInfiniteQuery() { const response = await appAxios.get( apiBase(`/private-api/promoted-entries`) ); - return slimEntryPage(response.data); + return annotateLanguageHints(slimEntryPage(response.data)); }, getNextPageParam: ( _lastPage: PromotedPage, diff --git a/apps/web/src/app/(dynamicPages)/feed/_components/feed-layout.tsx b/apps/web/src/app/(dynamicPages)/feed/_components/feed-layout.tsx index e1cfdb23d9..f8f135931b 100644 --- a/apps/web/src/app/(dynamicPages)/feed/_components/feed-layout.tsx +++ b/apps/web/src/app/(dynamicPages)/feed/_components/feed-layout.tsx @@ -11,6 +11,7 @@ import { UserAvatar } from "@/features/shared/user-avatar"; import { getPostsRankedQueryOptions, QueryKeys } from "@ecency/sdk"; import { getQueryClient } from "@/core/react-query"; import { withSlimPageEntries } from "@/core/entries/slim-entry"; +import { mergePreservingHint } from "@/core/entries/language-hint"; import type { InfiniteData } from "@tanstack/react-query"; const MAX_PENDING = 20; @@ -109,7 +110,7 @@ export function FeedLayout(props: PropsWithChildren) { if (Array.isArray(page)) { return (page as Entry[]).map((item) => { const updated = map.get(`${item.author}-${item.permlink}`); - return updated ? { ...item, ...updated } : item; + return updated ? mergePreservingHint(item, updated) : item; }); } return page; // SearchResponse: leave as-is @@ -123,7 +124,7 @@ export function FeedLayout(props: PropsWithChildren) { const updated = resp.find( (e) => e.author === item.author && e.permlink === item.permlink ); - return updated ? { ...item, ...updated } : item; + return updated ? mergePreservingHint(item, updated) : item; }) ); diff --git a/apps/web/src/core/entries/language-hint.ts b/apps/web/src/core/entries/language-hint.ts new file mode 100644 index 0000000000..ce7829e3df --- /dev/null +++ b/apps/web/src/core/entries/language-hint.ts @@ -0,0 +1,116 @@ +import { postBodySummary } from "@ecency/render-helper"; +import type { Entry } from "@/entities"; +import { MIN_DETECT_CHARS, francToIso1 } from "@/features/shared/entry-translate/iso639"; + +/** + * Server-side content-language hint for slim feed rows (#1597). + * + * The Translate chip on a card needs the content language. Until now every + * visitor's browser detected it: one markdown render per card plus the + * franc-min detector chunk (~47 KB gzipped), all on idle after every feed and + * post view, for the majority of readers whose language matches the content. + * + * The slim step already derives each card's summary on the server, so the + * server detects the language from that same summary once per fetch and ships + * the answer as `slim.lang`: an ISO-639-1 code, or `null` when the text is too + * short or the detector is unsure (which the chip treats as "nothing to + * offer"). The client gate reads the hint and skips both the render and the + * chunk. Rows fetched by the browser itself (later pages of an infinite feed) + * carry no hint and keep the on-idle path; `undefined` means "not checked". + * + * Server only: the detector is imported lazily inside the server branch, so it + * never reaches a client bundle. The summary used is the one the client gate + * would have used, so the decision is the same one, made earlier and once. + */ +export type LangHint = string | null; + +let detector: Promise<((text: string) => string) | null> | null = null; + +function loadDetector(): Promise<((text: string) => string) | null> { + if (!detector) { + detector = import("franc-min") + .then((m) => m.franc) + .catch(() => { + // Let the next fetch try again rather than pinning "no detector" for + // the life of the process. + detector = null; + return null; + }); + } + return detector; +} + +// The same bounds the client gate applies before detecting: render the sample +// to plain text (an author-written description can be a bare image link or +// markup, which must count as no text, not as a language) and cap the input. +const RAW_SAMPLE_CHARS = 2000; +const SAMPLE_CHARS = 600; + +export function detectionSample(text: string): string { + return postBodySummary(text.slice(0, RAW_SAMPLE_CHARS), 0).slice(0, SAMPLE_CHARS).trim(); +} + +export function isServerRuntime(): boolean { + return typeof window === "undefined"; +} + +export function hintFor( + franc: (text: string) => string, + entry: Pick +): LangHint { + try { + // json_metadata is author-written: description may be missing or not a string. + const description = entry.json_metadata?.description; + const raw = entry.body || typeof description !== "string" ? "" : description.trim(); + if (raw.length < MIN_DETECT_CHARS) return null; + const text = detectionSample(raw); + if (text.length < MIN_DETECT_CHARS) return null; + return francToIso1(franc(text)); + } catch { + return null; + } +} + +/** + * Merge a freshly polled row over a cached one without losing the hint: rows + * the browser polls itself carry no `slim.lang`, and a plain spread would + * replace the server's answer with "not checked". + */ +export function mergePreservingHint(item: T, updated: T): T { + const merged: T = { ...item, ...updated }; + const lang = item.slim?.lang; + if (lang !== undefined && updated.slim && updated.slim.lang === undefined) { + merged.slim = { ...updated.slim, lang }; + } + // A cross-post card reads from its nested original, which the poll replaces + // wholesale; give it the same treatment. + if (item.original_entry && updated.original_entry) { + merged.original_entry = mergePreservingHint(item.original_entry, updated.original_entry); + } + return merged; +} + +/** + * Annotate a slim page in place with `slim.lang`. Never throws: a missing or + * failing detector leaves the rows without a hint and the client detects as + * before. Not applied on the client (see module comment). + */ +export async function annotateLanguageHints(page: T): Promise { + if (!isServerRuntime() || !Array.isArray(page)) return page; + const franc = await loadDetector(); + if (!franc) return page; + for (const item of page as Entry[]) { + annotate(franc, item); + } + return page; +} + +function annotate(franc: (text: string) => string, entry: Entry | null | undefined): void { + try { + if (!entry || typeof entry !== "object" || !entry.slim || entry.slim.lang !== undefined) return; + entry.slim = { ...entry.slim, lang: hintFor(franc, entry) }; + if (entry.original_entry) annotate(franc, entry.original_entry); + } catch { + // One malformed row must not cost the page; it simply carries no hint. + } +} diff --git a/apps/web/src/core/entries/slim-entry.ts b/apps/web/src/core/entries/slim-entry.ts index 3fc73f17cf..0ef8b4027c 100644 --- a/apps/web/src/core/entries/slim-entry.ts +++ b/apps/web/src/core/entries/slim-entry.ts @@ -2,6 +2,7 @@ import { catchPostImage, getEntryImageRawUrl, postBodySummary } from "@ecency/re import { hasExternalLink } from "@ecency/sdk"; import { Entry } from "@/entities"; import { parseEntryLocationFromBody } from "./entry-location"; +import { annotateLanguageHints } from "./language-hint"; /** * Feed cards render a ~200 character summary and a thumbnail, but the bridge @@ -177,8 +178,12 @@ function wrapQueryFn( return { ...options, queryKey, + // The language hint rides on the slim rows the server produces; on the + // client it is a no-op (see core/entries/language-hint.ts). queryFn: async (...args: unknown[]) => - transform(await (queryFn as (...a: unknown[]) => Promise)(...args)) + annotateLanguageHints( + transform(await (queryFn as (...a: unknown[]) => Promise)(...args)) + ) } as T; } diff --git a/apps/web/src/entities/entries.ts b/apps/web/src/entities/entries.ts index 4f2a5c790a..bb818759fd 100644 --- a/apps/web/src/entities/entries.ts +++ b/apps/web/src/entities/entries.ts @@ -107,9 +107,12 @@ export interface Entry { * Present only on feed rows that went through the slim step * (`core/entries/slim-entry.ts`): the body is `""` and everything a card needs * has been derived into `json_metadata`. `ext_link` carries the one body fact - * the SDK's moderation rules still need. Absent on full entries. + * the SDK's moderation rules still need. Absent on full entries. `lang` is + * the server-detected content language of the card summary (ISO-639-1, or + * null when too short / undetermined); absent when the row was fetched by + * the browser, which then detects on its own (core/entries/language-hint.ts). */ - slim?: { ext_link: boolean }; + slim?: { ext_link: boolean; lang?: string | null }; } export interface EntryHeader { diff --git a/apps/web/src/features/shared/entry-translate/use-content-language-gate.ts b/apps/web/src/features/shared/entry-translate/use-content-language-gate.ts index 0d944bad78..8d88a2fa16 100644 --- a/apps/web/src/features/shared/entry-translate/use-content-language-gate.ts +++ b/apps/web/src/features/shared/entry-translate/use-content-language-gate.ts @@ -79,6 +79,8 @@ interface GateEntry { permlink: string; body?: string; json_metadata?: { description?: string | null } | null; + // Server-detected language of a slim row's summary (core/entries/language-hint.ts). + slim?: { lang?: string | null } | null; } interface GateOptions { @@ -124,6 +126,9 @@ export function useContentLanguageGate( const body = entry?.body || ""; const summary = body ? "" : entry?.json_metadata?.description || ""; const sample = body || summary; + // A slim row the server already detected: the same summary, decided once + // there, so the browser neither renders markdown nor loads the detector. + const hint = summary ? entry?.slim?.lang : undefined; useEffect(() => { setDecision(null); @@ -155,6 +160,12 @@ export function useContentLanguageGate( return; } + if (hint !== undefined) { + cacheDetection(key, { lang: hint, confirmed: false }); + setDecision(resolveTranslateCta({ detected: hint, reader, textLength: MIN_DETECT_CHARS })); + return; + } + scheduleIdle(async () => { if (cancelled) { return; @@ -221,7 +232,7 @@ export function useContentLanguageGate( return () => { cancelled = true; }; - }, [author, permlink, sample, summary, canServerConfirm, disabled]); + }, [author, permlink, sample, summary, hint, canServerConfirm, disabled]); return decision; } diff --git a/apps/web/src/specs/core/entries/language-hint-client.spec.ts b/apps/web/src/specs/core/entries/language-hint-client.spec.ts new file mode 100644 index 0000000000..ff1cbd0c11 --- /dev/null +++ b/apps/web/src/specs/core/entries/language-hint-client.spec.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { annotateLanguageHints, isServerRuntime } from "@/core/entries/language-hint"; +import { mockEntry } from "@/specs/test-utils"; +import type { Entry } from "@/entities"; + +// jsdom: the browser path. Rows the browser fetches itself must stay without a +// hint so the gate keeps detecting on idle, and the detector must not load. +describe("language hint in the browser (#1597)", () => { + it("is a no-op on the client", async () => { + expect(isServerRuntime()).toBe(false); + const slim = { + ...mockEntry({ permlink: "client-row", body: "" }), + slim: { ext_link: false } + } as Entry; + slim.json_metadata = { + description: + "Today I want to share with you a story about the community and the work we do together every week." + }; + const [out] = await annotateLanguageHints([slim]); + expect(out.slim?.lang).toBeUndefined(); + }); +}); diff --git a/apps/web/src/specs/core/entries/language-hint.spec.ts b/apps/web/src/specs/core/entries/language-hint.spec.ts new file mode 100644 index 0000000000..f9d35e5851 --- /dev/null +++ b/apps/web/src/specs/core/entries/language-hint.spec.ts @@ -0,0 +1,124 @@ +// @vitest-environment node +import { describe, expect, it, vi } from "vitest"; +import { annotateLanguageHints, detectionSample, hintFor, isServerRuntime, mergePreservingHint } from "@/core/entries/language-hint"; +import { withSlimEntries } from "@/core/entries/slim-entry"; +import { mockEntry } from "@/specs/test-utils"; +import type { Entry } from "@/entities"; + +const SPANISH = + "Hoy quiero compartir con ustedes una historia sobre la comunidad y el trabajo que hacemos juntos cada semana en la ciudad."; +const ENGLISH = + "Today I want to share with you a story about the community and the work we do together every week in the city."; + +function row(body: string, overrides: Partial = {}): Entry { + return mockEntry({ permlink: `hint-${Math.random().toString(36).slice(2)}`, body, ...overrides }); +} + +describe("server-side language hint (#1597)", () => { + it("runs on the server in this environment", () => { + expect(isServerRuntime()).toBe(true); + }); + + it("detects the summary language of slim rows and ships it as slim.lang", async () => { + const options = withSlimEntries({ + queryKey: ["hint"], + queryFn: async () => [row(SPANISH), row(ENGLISH)] + }); + const page = (await (options.queryFn as () => Promise)()) as Entry[]; + expect(page[0].body).toBe(""); + expect(page[0].slim?.lang).toBe("es"); + expect(page[1].slim?.lang).toBe("en"); + }); + + it("marks a row whose summary is too short as checked with nothing to offer (null)", async () => { + const options = withSlimEntries({ queryKey: ["hint"], queryFn: async () => [row("Hola.")] }); + const page = (await (options.queryFn as () => Promise)()) as Entry[]; + expect(page[0].slim?.lang).toBeNull(); + }); + + it("reaches the nested original of a cross-post", async () => { + const original = row(SPANISH); + const options = withSlimEntries({ + queryKey: ["hint"], + queryFn: async () => [row(ENGLISH, { original_entry: original } as Partial)] + }); + const page = (await (options.queryFn as () => Promise)()) as Entry[]; + expect(page[0].original_entry?.slim?.lang).toBe("es"); + }); + + it("leaves rows without the slim marker and non-array pages alone", async () => { + const full = row(ENGLISH); + full.body = ""; // looks slim but never went through slimEntry + expect((await annotateLanguageHints([full]))[0].slim).toBeUndefined(); + expect(await annotateLanguageHints({ items: [] })).toEqual({ items: [] }); + expect(await annotateLanguageHints(undefined)).toBeUndefined(); + }); + + it("treats a missing or non-string description as no text", () => { + const franc = vi.fn(() => "eng"); + for (const description of [undefined, null, 42, { text: ENGLISH }, [ENGLISH]]) { + expect(hintFor(franc, { body: "", json_metadata: { description } as never })).toBeNull(); + } + expect(franc).not.toHaveBeenCalled(); + }); + + it("never throws out of the queryFn when the detector fails", () => { + const boom = () => { + throw new Error("boom"); + }; + expect(hintFor(boom, { body: "", json_metadata: { description: ENGLISH } })).toBeNull(); + }); + + it("does not second-guess a row that already carries a hint", async () => { + const slim = { ...row(ENGLISH), body: "", slim: { ext_link: false, lang: "de" as string | null } }; + slim.json_metadata = { description: ENGLISH }; + const [out] = await annotateLanguageHints([slim]); + expect(out.slim?.lang).toBe("de"); + }); + + it("treats an author description that is only a link or markup as no text (#1597 review)", () => { + const franc = vi.fn(() => "por"); + for (const description of [ + "![photo](https://images.ecency.com/DQmabcdefghijklmnopqrstuvwxyz/photo_2026_08_21.jpg)", + "https://3speak.tv/watch?v=someone/abcdefgh https://images.ecency.com/p/abcdefghijklmnop.png", + "
", + "[Source](https://www.example.com/some/long/path/that/keeps/going/and/going/x)" + ]) { + expect(hintFor(franc, { body: "", json_metadata: { description } })).toBeNull(); + } + expect(franc).not.toHaveBeenCalled(); + // Plain summaries are passed through unchanged, so the server and the + // client decide on the same text. + expect(detectionSample(ENGLISH)).toBe(ENGLISH); + }); + + it("keeps the server hint when a browser-polled row is merged over a cached one", () => { + const cached = { ...row(ENGLISH), body: "", slim: { ext_link: false, lang: "en" as string | null } }; + const polled = { ...cached, slim: { ext_link: true }, stats: { total_votes: 9 } } as Entry; + const merged = mergePreservingHint(cached, polled); + expect(merged.slim).toEqual({ ext_link: true, lang: "en" }); + expect(merged.stats?.total_votes).toBe(9); + // A row the server did check wins over the cached answer. + const rechecked = { ...polled, slim: { ext_link: true, lang: "de" as string | null } }; + expect(mergePreservingHint(cached, rechecked).slim?.lang).toBe("de"); + // Nothing to preserve: plain merge. + const unhinted = { ...cached, slim: { ext_link: false } } as Entry; + expect(mergePreservingHint(unhinted, polled).slim).toEqual({ ext_link: true }); + // A cross-post's nested original keeps its hint too. + const original = { ...row(SPANISH), body: "", slim: { ext_link: false, lang: "es" as string | null } }; + const crossCached = { ...cached, original_entry: original } as Entry; + const crossPolled = { + ...polled, + original_entry: { ...original, slim: { ext_link: false }, stats: { total_votes: 3 } } + } as Entry; + const crossMerged = mergePreservingHint(crossCached, crossPolled); + expect(crossMerged.original_entry?.slim).toEqual({ ext_link: false, lang: "es" }); + expect(crossMerged.original_entry?.stats?.total_votes).toBe(3); + }); + + it("leaves the post page's full bodies to the client (hints are for slim rows)", () => { + const franc = vi.fn(() => "spa"); + expect(hintFor(franc, { body: SPANISH, json_metadata: { description: SPANISH } })).toBeNull(); + expect(franc).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/specs/features/entry-translate-language-gate.spec.tsx b/apps/web/src/specs/features/entry-translate-language-gate.spec.tsx index 01b7f0cb0d..9b3f1837d7 100644 --- a/apps/web/src/specs/features/entry-translate-language-gate.spec.tsx +++ b/apps/web/src/specs/features/entry-translate-language-gate.spec.tsx @@ -91,6 +91,59 @@ describe("useContentLanguageGate server /detect gating", () => { expect(detectLanguage).not.toHaveBeenCalled(); }); + it("uses a server hint on a slim row without loading the detector (#1597)", async () => { + const { franc } = await import("franc-min"); + vi.mocked(franc).mockClear(); + const { result } = renderHook(() => + useContentLanguageGate({ + author: "author-hint", + permlink: "gate-hint-es", + body: "", + json_metadata: { description: SPANISH_BODY }, + slim: { lang: "es" } + }) + ); + await waitFor(() => expect(result.current).not.toBeNull()); + expect(result.current?.show).toBe(true); + expect(result.current?.source).toBe("es"); + expect(franc).not.toHaveBeenCalled(); + expect(detectLanguage).not.toHaveBeenCalled(); + }); + + it("offers nothing for a slim row the server marked undetermined (#1597)", async () => { + const { franc } = await import("franc-min"); + vi.mocked(franc).mockClear(); + const { result } = renderHook(() => + useContentLanguageGate({ + author: "author-hint", + permlink: "gate-hint-null", + body: "", + json_metadata: { description: SPANISH_BODY }, + slim: { lang: null } + }) + ); + await waitFor(() => expect(result.current).not.toBeNull()); + expect(result.current?.show).toBe(false); + expect(franc).not.toHaveBeenCalled(); + }); + + it("ignores a stale hint once the full body is present (post page detects itself)", async () => { + const { franc } = await import("franc-min"); + vi.mocked(franc).mockClear(); + const { result } = renderHook(() => + useContentLanguageGate({ + author: "author-hint", + permlink: "gate-hint-full", + body: SPANISH_BODY, + json_metadata: { description: SPANISH_BODY }, + slim: { lang: "de" } + }) + ); + await waitFor(() => expect(result.current).not.toBeNull()); + expect(franc).toHaveBeenCalled(); + expect(result.current?.source).toBe("es"); + }); + it("does not let a summary-derived detection stand in for the full post", async () => { // Feed card first: detection comes from the card summary only. const feed = renderHook(() =>