`). When present and -// non-trivial, this is canonical — every modern CMS that ships -// semantic HTML uses it. Highest confidence. -// -// 2. Highest text-density block. Score every container element by -// `text-length × text-density` (text density = text-length / -// outer-html-length). The winner is usually the actual content -// region: lots of text, low markup overhead. Falls through when -// every container is tiny or markup-heavy. -// -// 3. Body minus chrome. Clone , remove -// header/nav/aside/footer/script/style/noscript and a list of -// common chrome class/id patterns (`.site-header`, `#footer`, -// etc.), keep the remainder. Lowest confidence; ships everything -// we couldn't classify, which on platform-rendered pages is -// still a lot. -// -// Design notes: -// - Pure transformation; no I/O, no agent. Same input → same output. -// - Returns `{ html, source, byteReduction }` so callers can log -// which rule fired and report compression to the watch log. -// - Validates the extracted region isn't catastrophically empty -// (must have at least 100 chars of text); falls through to the -// next rule if it would be. -// - Minimum-text threshold prevents the text-density rule from -// picking a 50-char widget when the real content is in a giant -// positional div with mostly spans. -// - -import * as cheerio from 'cheerio'; - -export type ContentRegionSource = 'main' | 'text-density' | 'body-minus-chrome' | 'whole-body'; - -export interface ContentRegionResult { - /** The extracted HTML content region (inner HTML — no wrapping element). */ - html: string; - /** Which rule produced the result. */ - source: ContentRegionSource; - /** Bytes of the input HTML. */ - inputBytes: number; - /** Bytes of the extracted region. */ - outputBytes: number; - /** Notes for diagnostics — e.g. text density score, removed chrome elements. */ - notes: string[]; -} - -/** Minimum text length for a candidate region to be considered "non-trivial". */ -const MIN_TEXT_LEN = 100; - -/** Class/id patterns commonly used for chrome in real-world sites. Mirrored to a CSS selector list. */ -const CHROME_PATTERNS = [ - // Direct semantic tags handled separately. - // Class-based: - '.site-header', '.site-footer', '.site-navigation', - '.global-header', '.global-footer', - '.navbar', '.nav-bar', '.menu-bar', '.top-bar', '.bottom-bar', - '.breadcrumb', '.breadcrumbs', - '.cookie-banner', '.cookie-notice', '.gdpr-banner', - '.skip-link', '.skip-to-content', - '.search-overlay', '.modal-overlay', - '.cart-drawer', '.cart-sidebar', - '.announcement-bar', - // ID-based: - '#header', '#footer', '#nav', '#navigation', '#site-header', '#site-footer', - '#cart', '#search', '#breadcrumb', '#breadcrumbs', -]; - -export function extractContentRegion(sanitizedHtml: string): ContentRegionResult { - const inputBytes = sanitizedHtml.length; - const $ = cheerio.load(sanitizedHtml); - const notes: string[] = []; - - // Rule 1 — explicit
- const $main = $('main').first(); - if ($main.length > 0) { - const text = $main.text().trim(); - if (text.length >= MIN_TEXT_LEN) { - const html = ($main.html() ?? '').trim(); - notes.push(`
found, ${text.length} chars of text`); - return { - html, - source: 'main', - inputBytes, - outputBytes: html.length, - notes, - }; - } - notes.push(`
found but has only ${text.length} chars text — falling through`); - } - - // Rule 2 — highest text-density container - // Consider article/section/div elements that contain meaningful text. - // Score = text length × density. Density punishes containers that are - // mostly markup (e.g. navigation, sidebar widgets) and rewards prose- - // heavy regions. We require minimum text and minimum density to avoid - // picking a tiny container. - let best: { el: cheerio.Cheerio; score: number; textLen: number; density: number } | null = null; - $('article, section, div').each((_, el) => { - const $el = $(el); - const text = $el.text().trim(); - if (text.length < MIN_TEXT_LEN * 4) return; // be more demanding here - const html = $.html($el); - const density = text.length / Math.max(html.length, 1); - if (density < 0.05) return; // markup-heavy, probably navigation - const score = text.length * density; - if (!best || score > best.score) { - best = { el: $el, score, textLen: text.length, density }; - } - }); - if (best) { - // Cast through unknown — cheerio's generic parameter has tightened - // since v1.0; we don't depend on the inner node type here, only on - // .html() being available, which is on the base Cheerio. - const winner = (best as { el: { html: () => string | null } }).el; - const html = (winner.html() ?? '').trim(); - notes.push( - `text-density winner: ${(best as { textLen: number }).textLen} chars text, density=${(best as { density: number }).density.toFixed(3)}, score=${(best as { score: number }).score.toFixed(0)}`, - ); - return { - html, - source: 'text-density', - inputBytes, - outputBytes: html.length, - notes, - }; - } - notes.push('no text-density winner — falling through to body-minus-chrome'); - - // Rule 3 — body minus chrome elements - const $body = $('body').first(); - if ($body.length > 0) { - const $clone = cheerio.load(`${$body.html() ?? ''}`); - // Strip semantic chrome. - $clone('header, nav, aside, footer, script, style, noscript').remove(); - // Strip pattern-matched chrome (best-effort; selectors that fail - // silently do nothing). - let removedPatterns = 0; - for (const pat of CHROME_PATTERNS) { - try { - const matched = $clone(pat); - if (matched.length > 0) { - removedPatterns += matched.length; - matched.remove(); - } - } catch { - // Invalid selector — skip silently. - } - } - const html = ($clone('body').html() ?? '').trim(); - if (html.length > 0) { - notes.push(`body-minus-chrome: stripped ${removedPatterns} chrome-pattern matches`); - return { - html, - source: 'body-minus-chrome', - inputBytes, - outputBytes: html.length, - notes, - }; - } - } - - // Last resort — return the whole body if we have one, else the whole - // sanitized input. Better to ship something than nothing. - const $bodyFallback = $('body').first(); - const fallback = ($bodyFallback.length > 0 ? $bodyFallback.html() : sanitizedHtml) ?? sanitizedHtml; - notes.push('all rules failed — returning whole body unchanged'); - return { - html: fallback.trim(), - source: 'whole-body', - inputBytes, - outputBytes: fallback.length, - notes, - }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/design-fragment-install.test.ts b/packages/data-liberation-agent/src/lib/streaming/design-fragment-install.test.ts deleted file mode 100644 index 0e5f5455c4..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/design-fragment-install.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Design-fragment sidecar → contentOverride → media-URL rewrite - * ============================================================== - * Integration test for Task 10: when `/design/.fragment.html` - * exists, its contents become the post's contentOverride, flowing through the - * existing prepareInstallContentWithMediaUrls so source URLs are - * swapped to local upload URLs. - * - * This test exercises the exact sequence that processOne() in - * watch-runner.ts executes: - * 1. designSidecarPath() → resolve sidecar path - * 2. readFileSync(sidecar) → contentOverride = fragment - * 3. prepareInstallContentWithMediaUrls({ sourceContent, contentOverride, mediaUrlMap }) - * → rewrites source CDN URLs to local upload URLs in the fragment - */ - -import { describe, expect, it } from 'vitest'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { designSidecarPath } from '../screenshot/design-capture-runner.js'; -import { prepareInstallContentWithMediaUrls } from './post-content-media-rewrite.js'; - -const TMP_ROOT = join(process.cwd(), '.tmp-test', 'design-fragment-install'); -mkdirSync(TMP_ROOT, { recursive: true }); - -describe('design-fragment sidecar as contentOverride with media-URL rewrite', () => { - it('loads the design sidecar as contentOverride and rewrites source img URLs to local upload URLs', () => { - const outDir = mkdtempSync(join(TMP_ROOT, 'out-')); - try { - const slug = 'about'; - const sourceImgUrl = 'https://src.test/a.png'; - const localUploadUrl = 'http://localhost:8881/wp-content/uploads/a.png'; - - // Write the design fragment sidecar (mirrors what captureDesignForUrl produces) - const sidecar = designSidecarPath(outDir, slug); - mkdirSync(join(outDir, 'design'), { recursive: true }); - const fragmentHtml = `
hero

About us

`; - writeFileSync(sidecar, fragmentHtml, 'utf8'); - - // Step 1: processOne reads the sidecar (mirrors the watch-runner.ts logic) - const fragment = readFileSync(sidecar, 'utf8'); - expect(fragment.trim().length).toBeGreaterThan(0); - - // Step 2: use it as contentOverride exactly as processOne does - const contentOverride = fragment; - - // Step 3: pass through prepareInstallContentWithMediaUrls (the existing media-rewrite) - const mediaUrlMap = new Map([[sourceImgUrl, localUploadUrl]]); - const result = prepareInstallContentWithMediaUrls({ - sourceContent: '

raw extracted content

', - contentOverride, - mediaUrlMap, - }); - - // The design fragment's img src must be rewritten to the local upload URL - expect(result.contentOverride).toContain(`src="${localUploadUrl}"`); - // The source CDN URL must NOT appear in the installed content - expect(result.contentOverride).not.toContain(sourceImgUrl); - expect(result.rewritten).toBe(true); - // contentOverride was provided — sourceContent was NOT promoted - expect(result.usedSourceContent).toBe(false); - expect(result.missing).toEqual([]); - } finally { - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('designSidecarPath returns /design/.fragment.html', () => { - expect(designSidecarPath('/tmp/mysite', 'contact')).toBe('/tmp/mysite/design/contact.fragment.html'); - }); - - it('falls back to raw source content when no design sidecar exists', () => { - const outDir = mkdtempSync(join(TMP_ROOT, 'out-nosidecar-')); - try { - const slug = 'services'; - const sourceImgUrl = 'https://src.test/banner.jpg'; - const localUploadUrl = 'http://localhost:8881/wp-content/uploads/banner.jpg'; - - // No sidecar written — contentOverride stays undefined - const contentOverride = undefined; - - const mediaUrlMap = new Map([[sourceImgUrl, localUploadUrl]]); - const result = prepareInstallContentWithMediaUrls({ - sourceContent: `

`, - contentOverride, - mediaUrlMap, - }); - - // Falls back to sourceContent, still rewrites the URL - expect(result.contentOverride).toContain(`src="${localUploadUrl}"`); - expect(result.usedSourceContent).toBe(true); - } finally { - rmSync(outDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/foundation-drift.test.ts b/packages/data-liberation-agent/src/lib/streaming/foundation-drift.test.ts deleted file mode 100644 index 62b61234cc..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/foundation-drift.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { computeInputsDigest, driftScore } from './foundation-drift.js'; - -const baselinePalette = { - version: 1, - sampledUrls: 4, - colors: [ - { hex: '#111111', count: 10, urls: 4 }, - { hex: '#fefefe', count: 9, urls: 4 }, - ], -}; -const baselineTypography = { - version: 1, - sampledUrls: 4, - bySelector: { - body: [{ fontFamily: 'Inter', fontSize: '16px', fontWeight: '400', lineHeight: '24px', urls: 4 }], - }, -}; -const baselineBreakpoints = { version: 1, sampledUrls: 4, minWidth: [768, 1024], maxWidth: [] }; - -describe('computeInputsDigest', () => { - it('returns a sha256: digest', () => { - const d = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - expect(d).toMatch(/^sha256:[a-f0-9]{64}$/); - }); - - it('is stable across key reorderings', () => { - const a = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - // Reorder top-level keys of palette - const reordered = { sampledUrls: 4, colors: baselinePalette.colors, version: 1 as const }; - const b = computeInputsDigest(reordered, baselineTypography, baselineBreakpoints); - expect(a).toBe(b); - }); - - it('changes when palette content changes', () => { - const a = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - const shifted = { - ...baselinePalette, - colors: [...baselinePalette.colors, { hex: '#ff0000', count: 5, urls: 4 }], - }; - const b = computeInputsDigest(shifted, baselineTypography, baselineBreakpoints); - expect(a).not.toBe(b); - }); - - it('changes when typography changes', () => { - const a = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - const shifted = { - ...baselineTypography, - bySelector: { - body: [ - { fontFamily: 'Roboto', fontSize: '16px', fontWeight: '400', lineHeight: '24px', urls: 4 }, - ], - }, - }; - const b = computeInputsDigest(baselinePalette, shifted, baselineBreakpoints); - expect(a).not.toBe(b); - }); - - it('changes when breakpoints change', () => { - const a = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - const shifted = { ...baselineBreakpoints, minWidth: [1024, 1280] }; - const b = computeInputsDigest(baselinePalette, baselineTypography, shifted); - expect(a).not.toBe(b); - }); -}); - -describe('driftScore', () => { - it('returns 0 when current inputs hash matches prevDigest', () => { - const digest = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - const score = driftScore(digest, { - palette: baselinePalette, - typography: baselineTypography, - breakpoints: baselineBreakpoints, - }); - expect(score).toBe(0); - }); - - it('returns > 1 when palette has shifted (re-rev needed)', () => { - const digest = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - const shifted = { - ...baselinePalette, - colors: [...baselinePalette.colors, { hex: '#00ffaa', count: 7, urls: 3 }], - }; - const score = driftScore(digest, { - palette: shifted, - typography: baselineTypography, - breakpoints: baselineBreakpoints, - }); - expect(score).toBeGreaterThan(1); - }); - - it('returns > 1 when typography has a font-family change', () => { - const digest = computeInputsDigest(baselinePalette, baselineTypography, baselineBreakpoints); - const shifted = { - ...baselineTypography, - bySelector: { - body: [ - { fontFamily: 'Comic Sans', fontSize: '16px', fontWeight: '400', lineHeight: '24px', urls: 4 }, - ], - }, - }; - const score = driftScore(digest, { - palette: baselinePalette, - typography: shifted, - breakpoints: baselineBreakpoints, - }); - expect(score).toBeGreaterThan(1); - }); - - it('returns > 1 when prevDigest is empty (first run)', () => { - const score = driftScore('', { - palette: baselinePalette, - typography: baselineTypography, - breakpoints: baselineBreakpoints, - }); - expect(score).toBeGreaterThan(1); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/foundation-drift.ts b/packages/data-liberation-agent/src/lib/streaming/foundation-drift.ts deleted file mode 100644 index 3b99fbb98e..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/foundation-drift.ts +++ /dev/null @@ -1,90 +0,0 @@ -// -// Foundation drift -// ================ -// Helpers the tick-scheduler uses to decide whether to re-rev the design -// foundation. Wraps the existing `sha256` utility from -// `src/lib/design-foundation/scaffold.ts` (re-exported here so the streaming -// pipeline doesn't depend on a deep import path) and adds a `driftScore` -// estimate. -// -// Drift threshold contract: a returned score `> 1` means the foundation -// should be re-revved. The tick-scheduler reads `state.lastFoundationInputsDigest` -// and feeds it here alongside the current input objects. -// -import { sha256 } from '../design-foundation/scaffold.js'; - -/** - * Compute a single sha256 digest over palette + typography + breakpoints. - * The inputs are first JSON-stringified with stable key ordering (via - * `JSON.stringify` of canonical-keyed values) so two semantically-equal inputs - * always produce the same digest. - * - * Reuses `sha256` from scaffold.ts to keep the digest convention identical. - */ -export function computeInputsDigest( - palette: unknown, - typography: unknown, - breakpoints: unknown, - computedStyles?: unknown, -): string { - const canonical = JSON.stringify({ - palette: canonicalize(palette), - typography: canonicalize(typography), - breakpoints: canonicalize(breakpoints), - ...(computedStyles === undefined ? {} : { computedStyles: canonicalize(computedStyles) }), - }); - return sha256(canonical); -} - -/** - * Estimate how much the foundation inputs have drifted since the previous - * digest was recorded. - * - * Returns: - * 0 — current inputs hash to `prevDigest` (no change). - * 2 — current inputs hash differs (above the re-rev threshold). - * - * The "count changed top-8 palette entries + font-family changes" part of the - * contract requires the previous inputs to reconstruct a per-entry diff; - * because the caller only retains the prior digest string, we collapse the - * decision to a binary same / different signal at a value (2) that exceeds - * the documented `> 1` threshold. - * - * If the prevDigest is empty (first run), we treat that as "first foundation - * — please run a tick" and return 2. - */ -export function driftScore( - prevDigest: string, - currentInputs: { palette: unknown; typography: unknown; breakpoints: unknown; computedStyles?: unknown }, -): number { - const current = computeInputsDigest( - currentInputs.palette, - currentInputs.typography, - currentInputs.breakpoints, - currentInputs.computedStyles, - ); - if (!prevDigest) return 2; - if (prevDigest === current) return 0; - return 2; -} - -// --------------------------------------------------------------------------- -// Internals -// --------------------------------------------------------------------------- - -/** - * Recursively sort object keys so two structurally-equal inputs produce the - * same JSON string. Arrays preserve order — caller is responsible for any - * domain-level normalization (e.g. ranking palette entries). - */ -function canonicalize(value: unknown): unknown { - if (Array.isArray(value)) return value.map(canonicalize); - if (value && typeof value === 'object') { - const obj = value as Record; - const sortedKeys = Object.keys(obj).sort(); - const out: Record = {}; - for (const k of sortedKeys) out[k] = canonicalize(obj[k]); - return out; - } - return value; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/foundation-run-state.test.ts b/packages/data-liberation-agent/src/lib/streaming/foundation-run-state.test.ts deleted file mode 100644 index 9fcbd6ba3a..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/foundation-run-state.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { emptyState, loadReplicateState, saveReplicateState } from './replicate-state.js'; -import { computeInputsDigest } from './foundation-drift.js'; -import { - foundationRevDecision, - recordFoundationInputsDigest, - selectFoundationSample, -} from './foundation-run-state.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function tmp(): string { - return mkdtempSync(join(FIXTURE_TMP, 'frs-')); -} - -function seedFoundationInputs(dir: string): { digest: string } { - const palette = { - version: 1, - sampledUrls: 3, - colors: [{ hex: '#000000', count: 10, urls: 3 }], - }; - const typography = { - version: 1, - sampledUrls: 3, - bySelector: { body: [{ fontFamily: 'Inter', fontSize: '16px', fontWeight: '400', lineHeight: '24px', urls: 3 }] }, - }; - const breakpoints = { version: 1, sampledUrls: 3, minWidth: [768], maxWidth: [] }; - writeFileSync(join(dir, 'palette.json'), JSON.stringify(palette)); - writeFileSync(join(dir, 'typography.json'), JSON.stringify(typography)); - writeFileSync(join(dir, 'breakpoints.json'), JSON.stringify(breakpoints)); - return { digest: computeInputsDigest(palette, typography, breakpoints) }; -} - -describe('foundation run state', () => { - it('skips a foundation-rev when the current aggregate digest is already recorded', () => { - const dir = tmp(); - const { digest } = seedFoundationInputs(dir); - saveReplicateState(dir, { ...emptyState(), lastFoundationInputsDigest: digest }); - - expect(foundationRevDecision(dir)).toEqual({ - shouldRun: false, - digest, - reason: 'foundation inputs unchanged', - }); - }); - - it('runs a foundation-rev when the recorded digest is stale', () => { - const dir = tmp(); - const { digest } = seedFoundationInputs(dir); - saveReplicateState(dir, { - ...emptyState(), - lastFoundationInputsDigest: 'sha256:0000000000000000000000000000000000000000000000000000000000000000', - }); - - expect(foundationRevDecision(dir)).toEqual({ - shouldRun: true, - digest, - reason: 'foundation inputs changed', - }); - }); - - it('records the current foundation aggregate digest after a successful run', () => { - const dir = tmp(); - const { digest } = seedFoundationInputs(dir); - - const recorded = recordFoundationInputsDigest(dir); - - expect(recorded).toBe(digest); - expect(loadReplicateState(dir).lastFoundationInputsDigest).toBe(digest); - }); - - it('uses one representative sample for the foundation fast path and prefers homepage', () => { - const sample = selectFoundationSample({ - page: [ - { url: 'a', html: 'html/a.html', screenshot: 'screenshots/desktop/a.png' }, - { url: 'b', html: 'html/b.html', screenshot: 'screenshots/desktop/b.png' }, - { url: 'c', html: 'html/c.html', screenshot: 'screenshots/desktop/c.png' }, - { url: 'd', html: 'html/d.html', screenshot: 'screenshots/desktop/d.png' }, - ], - homepage: [ - { url: 'home', html: 'html/home.html', screenshot: 'screenshots/desktop/home.png' }, - ], - product: [ - { url: 'p1', html: 'html/p1.html', screenshot: 'screenshots/desktop/p1.png' }, - { url: 'p2', html: 'html/p2.html', screenshot: 'screenshots/desktop/p2.png' }, - ], - }); - - expect(sample).toEqual({ - homepage: [{ url: 'home', html: 'html/home.html', screenshot: 'screenshots/desktop/home.png' }], - }); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/foundation-run-state.ts b/packages/data-liberation-agent/src/lib/streaming/foundation-run-state.ts deleted file mode 100644 index dfa128cb22..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/foundation-run-state.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { existsSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { classifyUrl, type UrlType } from '../extraction/sitemap.js'; -import { computeInputsDigest } from './foundation-drift.js'; -import { loadReplicateState, saveReplicateState } from './replicate-state.js'; - -export interface FoundationRevDecision { - shouldRun: boolean; - digest: string | null; - reason: string; -} - -export interface FoundationSampleEntry { - url: string; - html?: string | null; - screenshot?: string | null; - scrolledScreenshot?: string | null; -} - -export type FoundationSample = Partial>; - -interface ManifestEntry { - html?: string; - desktop?: string; - desktopScrolled?: string; -} - -interface Manifest { - entries?: Record; -} - -const FOUNDATION_INPUT_FILES = ['palette.json', 'typography.json', 'breakpoints.json'] as const; -const OPTIONAL_FOUNDATION_INPUT_FILES = ['computed-styles.json'] as const; -const DEFAULT_MAX_FOUNDATION_SAMPLES = 1; -const FOUNDATION_ARCHETYPE_PRIORITY: UrlType[] = ['homepage', 'page', 'product', 'post', 'gallery', 'event']; - -export function readCurrentFoundationInputsDigest(outputDir: string): string | null { - try { - const [palette, typography, breakpoints] = FOUNDATION_INPUT_FILES.map((file) => - JSON.parse(readFileSync(join(outputDir, file), 'utf8')) as unknown, - ); - const computedStyles = readOptionalJson(outputDir, OPTIONAL_FOUNDATION_INPUT_FILES[0]); - return computeInputsDigest(palette, typography, breakpoints, computedStyles); - } catch { - return null; - } -} - -function readOptionalJson(outputDir: string, file: string): unknown { - const path = join(outputDir, file); - if (!existsSync(path)) return undefined; - try { - return JSON.parse(readFileSync(path, 'utf8')) as unknown; - } catch { - return undefined; - } -} - -export function foundationRevDecision(outputDir: string): FoundationRevDecision { - const digest = readCurrentFoundationInputsDigest(outputDir); - if (!digest) { - return { shouldRun: true, digest: null, reason: 'foundation inputs unavailable' }; - } - - const state = loadReplicateState(outputDir); - if (state.lastFoundationInputsDigest === digest) { - return { shouldRun: false, digest, reason: 'foundation inputs unchanged' }; - } - - return { - shouldRun: true, - digest, - reason: state.lastFoundationInputsDigest ? 'foundation inputs changed' : 'foundation inputs not recorded', - }; -} - -export function recordFoundationInputsDigest(outputDir: string): string | null { - const digest = readCurrentFoundationInputsDigest(outputDir); - if (!digest) return null; - - const state = loadReplicateState(outputDir); - saveReplicateState(outputDir, { - ...state, - lastFoundationInputsDigest: digest, - }); - return digest; -} - -export function selectFoundationSample( - representatives: Partial>, - maxSamples = DEFAULT_MAX_FOUNDATION_SAMPLES, -): FoundationSample { - const out: FoundationSample = {}; - if (maxSamples <= 0) return out; - - let selected = 0; - for (const archetype of FOUNDATION_ARCHETYPE_PRIORITY) { - const entries = representatives[archetype]; - if (!Array.isArray(entries) || entries.length === 0) continue; - - const remaining = maxSamples - selected; - if (remaining <= 0) break; - - const picked = entries.slice(0, remaining); - out[archetype] = picked; - selected += picked.length; - } - return out; -} - -export function buildFoundationSampleFromManifest( - outputDir: string, - maxSamples = DEFAULT_MAX_FOUNDATION_SAMPLES, -): FoundationSample { - const manifestPath = join(outputDir, 'screenshots', 'manifest.json'); - if (!existsSync(manifestPath)) return {}; - - let manifest: Manifest; - try { - manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as Manifest; - } catch { - return {}; - } - - const buckets: Partial> = {}; - const entries = manifest.entries ?? {}; - for (const [url, entry] of Object.entries(entries)) { - const archetype = classifyUrl(url); - const bucket = buckets[archetype] ?? []; - bucket.push({ - url, - html: entry.html ?? null, - }); - buckets[archetype] = bucket; - } - - return selectFoundationSample(buckets, maxSamples); -} diff --git a/packages/data-liberation-agent/src/lib/streaming/heuristic-blocks.test.ts b/packages/data-liberation-agent/src/lib/streaming/heuristic-blocks.test.ts deleted file mode 100644 index 3298eb345e..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/heuristic-blocks.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { heuristicBlocks } from './heuristic-blocks.js'; - -describe('heuristicBlocks', () => { - it('handles pure paragraphs', () => { - const html = '

First paragraph.

Second paragraph.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(true); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain('First paragraph.'); - expect(result.blocks).toContain('Second paragraph.'); - }); - - it('handles paragraphs interleaved with h2/h3 headings', () => { - const html = '

Section

Some prose.

Subsection

More prose.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(true); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain('Section'); - expect(result.blocks).toContain('Subsection'); - }); - - it('handles a single image followed by paragraphs', () => { - const html = 'Hero

Caption-like text.

More body.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(true); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain('src="https://example.com/hero.jpg"'); - expect(result.blocks).toContain('alt="Hero"'); - expect(result.blocks).toContain(''); - }); - - it('handles a
followed by paragraphs', () => { - const html = '
H
Cap

Body.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(true); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain('src="https://example.com/h.jpg"'); - }); - - it('handles a single
with heading + paragraphs as a wp:group', () => { - const html = '

About

We make things.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(true); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain('About'); - expect(result.blocks).toContain('We make things.'); - }); - - it('refuses complex page with multiple
blocks', () => { - const html = '

One

Two

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(false); - }); - - it('refuses pages with lists, tables, or unfamiliar elements', () => { - expect(heuristicBlocks('
  • a
  • b
').handled).toBe(false); - expect(heuristicBlocks('
x
').handled).toBe(false); - expect(heuristicBlocks('
stuff
').handled).toBe(false); - }); - - it('refuses an empty or whitespace-only input', () => { - expect(heuristicBlocks('').handled).toBe(false); - expect(heuristicBlocks(' \n ').handled).toBe(false); - }); - - it('refuses pages where a paragraph is followed by an image (out-of-order)', () => { - // Image-then-paragraphs is fine; paragraph-then-image is not in our - // shape set — fall through to the AI path. - const html = '

Lead.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(false); - }); - - it('refuses h1 (since post_content should not duplicate post title)', () => { - const html = '

Title

Body.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(false); - }); - - it('refuses a section that mixes images with text', () => { - const html = '

Hi

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(false); - }); - - it('preserves inline markup inside paragraphs (e.g. , )', () => { - const html = '

Click here now.

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(true); - expect(result.blocks).toContain(''); - expect(result.blocks).toContain('here'); - }); - - it('rejects pages with stray top-level text (not inside any element)', () => { - const html = 'stray text

then a paragraph

'; - const result = heuristicBlocks(html); - expect(result.handled).toBe(false); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/heuristic-blocks.ts b/packages/data-liberation-agent/src/lib/streaming/heuristic-blocks.ts deleted file mode 100644 index 936fa4edaa..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/heuristic-blocks.ts +++ /dev/null @@ -1,205 +0,0 @@ -// -// Heuristic block transformer -// =========================== -// Pure function that recognises trivially-structured pages and emits valid -// WP block markup directly, sidestepping the AI compose path. Returning -// `{handled: false}` means "I'm not sure" — the caller falls through to the -// AI skill. -// -// Confidence floor: heuristic only claims `handled: true` when EVERY visible -// element fits one of the recognised shapes. Any unexpected element type -// (lists, tables, sections, divs with classes, custom elements, etc.) flips -// to `handled: false`. -// -// Recognised shapes (calibrated for the first eval pass): -// 1. Pure text page — only `

` and `

`/`

` elements. -// Heading levels stay 2-3; we don't synthesise `

` here because -// `post_content` shouldn't repeat the post title. -// 2. Single image followed by paragraphs — leading `` (or -// `
`) then 1+ paragraphs. -// 3. Single section with a heading + text — one `
` containing -// one `

`/`

` and 1+ paragraphs. -// -// Any other shape returns `{handled: false}`. -// - -import * as cheerio from 'cheerio'; - -export interface HeuristicResult { - handled: boolean; - blocks?: string; - /** Internal — surfaced for debugging / audit logs. */ - reason?: string; -} - -const ALLOWED_TEXTISH = new Set(['p', 'h2', 'h3']); - -import { escapeHtmlText as escapeHtml } from '../html-escape.js'; - -function paragraphBlock(html: string): string { - return `\n

${html}

\n`; -} - -function headingBlock(level: 2 | 3, html: string): string { - const attrs = level === 2 ? '' : ` {"level":${level}}`; - return `\n${html}\n`; -} - -function imageBlock(src: string, alt: string): string { - const escapedSrc = escapeHtml(src); - const escapedAlt = escapeHtml(alt); - return `\n
${escapedAlt}
\n`; -} - -function groupBlock(inner: string): string { - return `\n
\n${inner}\n
\n`; -} - -interface SimpleEl { - tag: string; - innerHtml: string; - attrs: Record; - childTags: string[]; -} - -/** - * Wrap input in a synthetic body so cheerio's `*` traversal sees the input - * as siblings even when the user passed a fragment without a wrapping element. - */ -function topLevelChildren(html: string): SimpleEl[] { - const $ = cheerio.load(`${html}`); - const body = $('body').first(); - const elements: SimpleEl[] = []; - body.contents().each((_, node) => { - if (node.type === 'tag') { - const $node = $(node); - const attrs: Record = {}; - const tagAttrs = (node as { attribs?: Record }).attribs ?? {}; - for (const [k, v] of Object.entries(tagAttrs)) attrs[k] = v; - const childTags: string[] = []; - $node.children().each((__, c) => { - if (c.type === 'tag') childTags.push((c as { tagName: string }).tagName.toLowerCase()); - }); - elements.push({ - tag: (node as { tagName: string }).tagName.toLowerCase(), - innerHtml: $node.html() ?? '', - attrs, - childTags, - }); - } else if (node.type === 'text') { - const text = (node as { data: string }).data ?? ''; - if (text.trim()) { - elements.push({ tag: '#textnode', innerHtml: text, attrs: {}, childTags: [] }); - } - } - }); - return elements; -} - -interface ImageInfo { - src: string; - alt: string; -} - -/** Parse a `
` element to recognize a `
` (with optional
). */ -function pickFigureImage(figureInnerHtml: string): ImageInfo | null { - const $ = cheerio.load(`${figureInnerHtml}`); - const body = $('body').first(); - const childEls: Array<{ tag: string; src: string; alt: string }> = []; - body.contents().each((_, node) => { - if (node.type === 'tag') { - const tagName = (node as { tagName: string }).tagName.toLowerCase(); - if (tagName === 'img' || tagName === 'figcaption') { - const $n = $(node); - childEls.push({ - tag: tagName, - src: $n.attr('src') ?? '', - alt: $n.attr('alt') ?? '', - }); - } else { - childEls.push({ tag: tagName, src: '', alt: '' }); - } - } - }); - const hasOnlyAllowed = childEls.every((c) => c.tag === 'img' || c.tag === 'figcaption'); - const img = childEls.find((c) => c.tag === 'img'); - if (!hasOnlyAllowed || !img) return null; - return { src: img.src, alt: img.alt }; -} - -function pickLeadingImage(el: SimpleEl): ImageInfo | null { - if (el.tag === 'img') { - return { src: el.attrs.src ?? '', alt: el.attrs.alt ?? '' }; - } - if (el.tag === 'figure') { - return pickFigureImage(el.innerHtml); - } - return null; -} - -function textishToBlock(el: SimpleEl): string { - const inner = el.innerHtml.trim(); - if (el.tag === 'p') return paragraphBlock(inner); - if (el.tag === 'h2') return headingBlock(2, inner); - if (el.tag === 'h3') return headingBlock(3, inner); - return paragraphBlock(escapeHtml(inner)); -} - -/** - * Try to compose blocks from the input HTML using the trivial-shape rules - * above. Returns `{handled: false}` whenever the structure isn't a perfect - * match — the AI path will run instead. - */ -export function heuristicBlocks(html: string): HeuristicResult { - if (!html || !html.trim()) { - return { handled: false, reason: 'empty input' }; - } - - const children = topLevelChildren(html); - if (children.length === 0) { - return { handled: false, reason: 'no structured children' }; - } - - // Stray text directly between top-level blocks is unusual and risky to - // synthesize — bail. - if (children.some((c) => c.tag === '#textnode')) { - return { handled: false, reason: 'top-level stray text' }; - } - - // Shape 3: single
with heading + paragraphs → wrap in wp:group - if (children.length === 1 && children[0].tag === 'section') { - const inner = topLevelChildren(children[0].innerHtml); - const allTextish = inner.every((c) => ALLOWED_TEXTISH.has(c.tag)); - const hasHeading = inner.some((c) => c.tag === 'h2' || c.tag === 'h3'); - if (allTextish && hasHeading && inner.length > 0) { - const innerBlocks = inner.map((c) => textishToBlock(c)).join('\n\n'); - return { handled: true, blocks: groupBlock(innerBlocks), reason: 'section-with-heading' }; - } - return { handled: false, reason: 'section is not pure heading+paragraphs' }; - } - - // Shape 2: leading image (raw or
) followed by paragraphs - const leadingImage = pickLeadingImage(children[0]); - if (leadingImage) { - const rest = children.slice(1); - const restAllParagraphs = rest.every((c) => c.tag === 'p'); - if (restAllParagraphs && rest.length > 0) { - const blocks = [imageBlock(leadingImage.src, leadingImage.alt)]; - for (const p of rest) blocks.push(paragraphBlock(p.innerHtml.trim())); - return { handled: true, blocks: blocks.join('\n\n'), reason: 'image+paragraphs' }; - } - return { handled: false, reason: 'leading image not followed by paragraphs only' }; - } - - // Shape 1: pure paragraphs / h2 / h3 - const allTextish = children.every((c) => ALLOWED_TEXTISH.has(c.tag)); - if (allTextish) { - return { - handled: true, - blocks: children.map((c) => textishToBlock(c)).join('\n\n'), - reason: 'paragraphs+headings', - }; - } - - return { handled: false, reason: 'mixed structure outside heuristic shapes' }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/internal-link-rewrite.test.ts b/packages/data-liberation-agent/src/lib/streaming/internal-link-rewrite.test.ts deleted file mode 100644 index 3d8017499a..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/internal-link-rewrite.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { buildInternalLinkMap, rewriteInternalLinks } from './internal-link-rewrite.js'; - -// Fictional source site — no real source-site URLs/slugs (project convention). -// `redirect-map.json` is the canonical source-path -> local-permalink map (the -// same map the nav/footer rewrite in theme-scaffold consumes). -const redirectMap = [ - { from: '/about-the-shop', to: '/about-the-shop/' }, - { from: '/contact', to: '/contact/' }, -]; -const origins = ['craftwood-fixture.test']; - -describe('buildInternalLinkMap', () => { - it('always maps the site root to "/" under both path and host+path keys', () => { - const map = buildInternalLinkMap(redirectMap, { siteOrigins: origins }); - expect(map.get('/')).toBe('/'); - expect(map.get('craftwood-fixture.test/')).toBe('/'); - }); - - it('maps a redirect entry under both path and host+path keys', () => { - const map = buildInternalLinkMap(redirectMap, { siteOrigins: origins }); - expect(map.get('/about-the-shop')).toBe('/about-the-shop/'); - expect(map.get('craftwood-fixture.test/about-the-shop')).toBe('/about-the-shop/'); - }); - - it('builds path-only keys when no origins are supplied', () => { - const map = buildInternalLinkMap(redirectMap); - expect(map.get('/contact')).toBe('/contact/'); - expect(map.get('craftwood-fixture.test/contact')).toBeUndefined(); - }); -}); - -describe('rewriteInternalLinks', () => { - const map = buildInternalLinkMap(redirectMap, { siteOrigins: origins }); - - it('rewrites an absolute internal href to the root-relative permalink', () => { - const out = rewriteInternalLinks('About', map); - expect(out).toBe('About'); - }); - - it('rewrites a root-relative href', () => { - const out = rewriteInternalLinks('Contact', map); - expect(out).toBe('Contact'); - }); - - it('rewrites a bare relative href', () => { - const out = rewriteInternalLinks('About', map); - expect(out).toBe('About'); - }); - - it('rewrites a .html form', () => { - const out = rewriteInternalLinks('About', map); - expect(out).toBe('About'); - }); - - it('rewrites a trailing-slash form', () => { - const out = rewriteInternalLinks('Contact', map); - expect(out).toBe('Contact'); - }); - - it('matches the non-www host variant', () => { - const out = rewriteInternalLinks('Contact', map); - expect(out).toBe('Contact'); - }); - - it('preserves a #fragment when rewriting', () => { - const out = rewriteInternalLinks('Team', map); - expect(out).toBe('Team'); - }); - - it('leaves an external host untouched and does not warn', () => { - const onMissing = vi.fn(); - const out = rewriteInternalLinks('x', map, { onMissing }); - expect(out).toBe('x'); - expect(onMissing).not.toHaveBeenCalled(); - }); - - it('leaves mailto:/tel: and in-page anchors untouched', () => { - const input = 'mts'; - const out = rewriteInternalLinks(input, map); - expect(out).toBe(input); - }); - - it('leaves an unmapped internal relative href as-is and reports it via onMissing', () => { - const onMissing = vi.fn(); - const out = rewriteInternalLinks('x', map, { onMissing }); - expect(out).toBe('x'); - expect(onMissing).toHaveBeenCalledWith('/never-extracted'); - }); - - it('returns input unchanged for an empty map', () => { - const input = 'Contact'; - expect(rewriteInternalLinks(input, new Map())).toBe(input); - }); - - it('rewrites single-quoted href attributes too', () => { - const out = rewriteInternalLinks("Contact", map); - expect(out).toBe("Contact"); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/internal-link-rewrite.ts b/packages/data-liberation-agent/src/lib/streaming/internal-link-rewrite.ts deleted file mode 100644 index 263a34d839..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/internal-link-rewrite.ts +++ /dev/null @@ -1,170 +0,0 @@ -// -// Internal link rewriting (source href -> imported permalink) -// =========================================================== -// Reconstructed pages and generated nav template parts carry source hrefs -// verbatim. After import, a link to the source site's `/about` should point at -// the imported WordPress page's permalink instead. -// -// `output//screenshots/manifest.json` is the authoritative -// `sourceUrl -> slug` map. Imported pages get `post_name = slug`, so the target -// is the root-relative pretty permalink `/{slug}/` (homepage -> `/`). -// -// This module is pure (no I/O), mirroring `media-url-rewrite.ts`: the caller -// builds the map from the manifest and hands us a string. The same function is -// used for page-body block markup and for generated nav template parts — both -// expose links as `href="..."` attribute surfaces. -// -/** A `redirect-map.json` entry: source path -> local WP permalink. */ -export interface RedirectMapEntry { - from: string; - to: string; -} - -/** Normalized link key -> root-relative target permalink (e.g. "/about/"). */ -export type InternalLinkMap = Map; - -export interface BuildInternalLinkMapOpts { - /** - * Source-site hostnames (e.g. ["example.test"]). When supplied, each redirect - * entry also registers a `host+path` key so ABSOLUTE same-site hrefs match. - * Absolute hrefs to any other host are left untouched (no false rewrites). - */ - siteOrigins?: string[]; -} - -export interface InternalLinkRewriteOpts { - /** - * Fired once per unique candidate href that looked internal (root-relative or - * bare-relative) but had no mapping — e.g. a page we didn't extract. Mirrors - * `rewriteMediaUrls`' missing-warning contract. - */ - onMissing?: (href: string) => void; -} - -/** - * Collapse a URL pathname into the canonical key form used for both map keys - * and candidate lookups: percent-decoded, `.html`/`.htm` stripped, trailing - * slash removed (except root), lowercased, leading-slash guaranteed. - */ -function normalizePath(pathname: string): string { - let p = pathname; - try { - p = decodeURIComponent(p); - } catch { - // Leave malformed percent-sequences as-is. - } - p = p.replace(/\.html?$/i, ''); - if (!p.startsWith('/')) p = '/' + p; - if (p !== '/') p = p.replace(/\/+$/, ''); - if (p === '') p = '/'; - return p.toLowerCase(); -} - -/** Lowercase host with a leading `www.` stripped. */ -function normalizeHost(host: string): string { - return host.toLowerCase().replace(/^www\./, ''); -} - -/** - * Build the rewrite map from `redirect-map.json` entries — the canonical - * source-path -> local-permalink map the nav/footer rewrite also consumes. - * - * Each entry registers two keys pointing at the same target so both absolute - * and relative source hrefs match: - * - path-only `/about` (root-relative + bare hrefs) - * - host + path `example.test/about` (absolute hrefs; requires origins) - * - * The site root (`/`) is always seeded to `/` so homepage links pass through - * without a spurious "unmapped" warning. - */ -export function buildInternalLinkMap( - redirectMap: RedirectMapEntry[], - opts: BuildInternalLinkMapOpts = {}, -): InternalLinkMap { - const map: InternalLinkMap = new Map(); - const hosts = (opts.siteOrigins ?? []).map(normalizeHost).filter(Boolean); - - const register = (from: string, to: string) => { - const path = normalizePath(from); - map.set(path, to); - for (const host of hosts) map.set(`${host}${path}`, to); - }; - - register('/', '/'); - for (const entry of redirectMap ?? []) { - if (!entry?.from || !entry?.to) continue; - register(entry.from, entry.to); - } - return map; -} - -const SKIP_SCHEME = /^(?:mailto:|tel:|javascript:|data:|sms:|geo:|callto:)/i; - -interface Candidate { - /** Map lookup key, or null when the href should be skipped entirely. */ - key: string | null; - /** `#fragment` (including the leading `#`) to re-append after rewrite, or ''. */ - fragment: string; - /** True when the href is root-relative or bare-relative (clearly internal). */ - internalRelative: boolean; -} - -/** Derive the lookup key + fragment for a single href value. */ -function analyzeHref(rawHref: string): Candidate { - const href = rawHref.trim(); - const none: Candidate = { key: null, fragment: '', internalRelative: false }; - if (!href || SKIP_SCHEME.test(href)) return none; - // Pure in-page anchor: no path component. - if (href.startsWith('#')) return none; - - // Absolute (or protocol-relative) URL. - if (/^https?:\/\//i.test(href) || href.startsWith('//')) { - let url: URL; - try { - url = new URL(href.startsWith('//') ? `https:${href}` : href); - } catch { - return none; - } - const key = `${normalizeHost(url.hostname)}${normalizePath(url.pathname)}`; - return { key, fragment: url.hash, internalRelative: false }; - } - - // Relative (root-relative `/x` or bare `x` / `./x` / `../x`). - const hashIdx = href.indexOf('#'); - const fragment = hashIdx >= 0 ? href.slice(hashIdx) : ''; - let pathPart = hashIdx >= 0 ? href.slice(0, hashIdx) : href; - const queryIdx = pathPart.indexOf('?'); - if (queryIdx >= 0) pathPart = pathPart.slice(0, queryIdx); - pathPart = pathPart.replace(/^(?:\.\.?\/)+/, ''); - return { key: normalizePath(pathPart), fragment, internalRelative: true }; -} - -/** - * Rewrite internal href surfaces in an HTML / block-markup string. Pure. - * - * Only `href` attribute values are touched; unmatched/external/scheme links are - * left as-is. Internal-looking misses are reported via `opts.onMissing`. - */ -export function rewriteInternalLinks( - input: string, - map: InternalLinkMap, - opts: InternalLinkRewriteOpts = {}, -): string { - if (!input || map.size === 0) return input; - - const warned = new Set(); - // Capture the quote char so we re-emit the same style (group 1 = quote). - return input.replace(/\bhref\s*=\s*(["'])([^"']*)\1/gi, (whole, quote: string, value: string) => { - const { key, fragment, internalRelative } = analyzeHref(value); - if (key === null) return whole; - const target = map.get(key); - if (target) { - return `href=${quote}${target}${fragment}${quote}`; - } - if (internalRelative && opts.onMissing && !warned.has(value)) { - warned.add(value); - opts.onMissing(value); - } - return whole; - }); -} diff --git a/packages/data-liberation-agent/src/lib/streaming/media-install.test.ts b/packages/data-liberation-agent/src/lib/streaming/media-install.test.ts deleted file mode 100644 index 8be99a50a1..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/media-install.test.ts +++ /dev/null @@ -1,805 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync, utimesSync } from 'node:fs'; -import { join } from 'node:path'; -import { installMediaFiles, installMediaForUrl } from './media-install.js'; -import { MediaStubStore } from '../resume-state/index.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -interface SetupOpts { - /** Stubs to seed; key is sourceUrl, value defines status + localPath relative to outputDir/media. */ - stubs: Array<{ - url: string; - filename: string; - bytes?: Buffer; - alreadyInstalled?: number; - status?: 'success' | 'error' | 'awaiting'; - /** Record `svgRisky` on the stub (SVG survival routing). */ - svgRisky?: boolean; - /** Write this PNG into media/ AND record it as the stub's rasterPath. */ - rasterFilename?: string; - /** Write this PNG into media/ WITHOUT recording rasterPath (dedup-guard scenario). */ - sidecarPng?: string; - }>; -} - -function setup(opts: SetupOpts) { - const outputDir = mkdtempSync(join(FIXTURE_TMP, 'mi-')); - const wpRoot = join(outputDir, 'site', 'wordpress'); - mkdirSync(wpRoot, { recursive: true }); - mkdirSync(join(outputDir, 'media'), { recursive: true }); - - const store = MediaStubStore.load(outputDir); - for (const s of opts.stubs) { - const status = s.status ?? 'success'; - const filePath = join(outputDir, 'media', s.filename); - if (status === 'success') { - writeFileSync(filePath, s.bytes ?? Buffer.from('fake')); - let extra: { rasterPath?: string; svgRisky?: boolean } | undefined; - if (s.rasterFilename) { - const rasterPath = join(outputDir, 'media', s.rasterFilename); - writeFileSync(rasterPath, Buffer.from('fake-png')); - extra = { rasterPath, svgRisky: s.svgRisky }; - } else if (s.svgRisky !== undefined) { - extra = { svgRisky: s.svgRisky }; - } - if (s.sidecarPng) { - writeFileSync(join(outputDir, 'media', s.sidecarPng), Buffer.from('fake-png')); - } - store.markSuccess(s.url, filePath, extra); - if (s.alreadyInstalled !== undefined) { - store.recordWpPostId(s.url, s.alreadyInstalled); - } - } else if (status === 'error') { - store.markFailure(s.url, 'test-error'); - } - // 'awaiting' is the default-no-mutation state - } - store.flush(); - return { outputDir, wpRoot }; -} - -/** Read back the JSON payload staged for a given eval-file exec call. */ -function readStagedPayload(outputDir: string, args: string[]): Array<{ filename: string; sourceUrl: string }> { - const vfsPath = args[args.indexOf('eval-file') + 2] as string; - const name = vfsPath.split('/').pop()!; - return JSON.parse(readFileSync(join(outputDir, 'site', '.dla-scripts', 'payloads', name), 'utf8')); -} - -/** All exec calls that are wp-cli `plugin …` invocations (ensurePlugin traffic). */ -function pluginCalls(exec: ReturnType): string[][] { - return exec.mock.calls.filter(([, args]) => (args as string[]).includes('plugin')).map(([, args]) => args as string[]); -} - -const SUCCESS_RESPONSE = (entries: Array<{ sourceUrl: string; filename: string; postId: number; localUrl: string; reused?: boolean }>) => - `Some other PHP output...\nDLA_INSTALL_MEDIA_JSON_BEGIN\n${JSON.stringify({ - results: entries.map((e) => ({ ...e, reused: e.reused ?? false })), - errors: [], - })}\nDLA_INSTALL_MEDIA_JSON_END\nMore noise after\n`; - -describe('installMediaFiles', () => { - it('copies caller-supplied files into uploads and returns parsed installs', async () => { - const root = mkdtempSync(join(FIXTURE_TMP, 'mi-files-')); - const sourceDir = join(root, 'source', 'assets', 'media'); - const wpRoot = join(root, 'site', 'wordpress'); - mkdirSync(sourceDir, { recursive: true }); - mkdirSync(wpRoot, { recursive: true }); - const absPath = join(sourceDir, 'card-aurora.png'); - writeFileSync(absPath, Buffer.from('fictional image')); - const stamp = new Date(2026, 5, 9, 12, 0, 0); - utimesSync(absPath, stamp, stamp); - - try { - const exec = vi.fn().mockResolvedValue({ - stdout: SUCCESS_RESPONSE([ - { - sourceUrl: 'assets/media/card-aurora.png', - filename: 'card-aurora.png', - postId: 17, - localUrl: 'https://studio.test/wp-content/uploads/2026/06/card-aurora.png', - }, - ]), - stderr: '', - }); - - const result = await installMediaFiles({ - files: [{ absPath, sourceUrl: 'assets/media/card-aurora.png' }], - wpRoot, - _execFile: exec, - }); - - expect(result).toEqual({ - installed: [ - { - sourceUrl: 'assets/media/card-aurora.png', - postId: 17, - localUrl: 'https://studio.test/wp-content/uploads/2026/06/card-aurora.png', - }, - ], - errors: [], - }); - expect(existsSync(join(wpRoot, 'wp-content', 'uploads', '2026', '06', 'card-aurora.png'))).toBe(true); - expect(exec).toHaveBeenCalledTimes(1); - } finally { - rmSync(root, { recursive: true, force: true }); - } - }); -}); - -describe('installMediaForUrl', () => { - it('copies media into wpRoot uploads, runs PHP, and records wpPostId', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg' }], - }); - try { - const exec = vi.fn().mockResolvedValue({ - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/a.jpg', filename: 'a.jpg', postId: 42, localUrl: 'http://wp/uploads/2024/01/a.jpg' }, - ]), - stderr: '', - }); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - expect(result.errors).toEqual([]); - expect(result.installed).toHaveLength(1); - expect(result.installed[0]).toMatchObject({ sourceUrl: 'https://cdn/a.jpg', postId: 42 }); - - // File was copied into the wpRoot under the year/month derived from mtime. - // The exact year/month varies with the run, but we know the file should - // exist under wp-content/uploads somewhere. - const uploadsDir = join(wpRoot, 'wp-content', 'uploads'); - expect(existsSync(uploadsDir)).toBe(true); - - // Stub store now records the post ID. - const store = MediaStubStore.load(outputDir); - expect(store.get('https://cdn/a.jpg')?.wpPostId).toBe(42); - - // The PHP script + payload were staged to the parent of wpRoot (the site path). - const sitePath = join(outputDir, 'site'); - expect(existsSync(join(sitePath, '.dla-scripts', 'install-media.php'))).toBe(true); - - // exec was called with studio + wp + eval-file + script + payload. - expect(exec).toHaveBeenCalledTimes(1); - const [bin, args] = exec.mock.calls[0]; - expect(bin).toBe('studio'); - expect(args).toContain('wp'); - expect(args).toContain('eval-file'); - expect(args).toContain('--path'); - expect(args[args.indexOf('--path') + 1]).toBe(sitePath); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('uses wpRoot itself as the Studio site path for flat Studio installs', async () => { - const outputDir = mkdtempSync(join(FIXTURE_TMP, 'mi-flat-studio-')); - const wpRoot = join(outputDir, 'flat-site'); - mkdirSync(join(wpRoot, 'wp-content'), { recursive: true }); - mkdirSync(join(outputDir, 'media'), { recursive: true }); - - const filePath = join(outputDir, 'media', 'a.jpg'); - writeFileSync(filePath, Buffer.from('fake')); - const store = MediaStubStore.load(outputDir); - store.markSuccess('https://cdn/a.jpg', filePath); - store.flush(); - - try { - const exec = vi.fn().mockResolvedValue({ - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/a.jpg', filename: 'a.jpg', postId: 42, localUrl: 'http://wp/uploads/2024/01/a.jpg' }, - ]), - stderr: '', - }); - - await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - const [, args] = exec.mock.calls[0]; - expect(args[args.indexOf('--path') + 1]).toBe(wpRoot); - expect(existsSync(join(wpRoot, '.dla-scripts', 'install-media.php'))).toBe(true); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('skips entries already installed without persisted localUrl (legacy stub)', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg', alreadyInstalled: 99 }], - }); - try { - const exec = vi.fn(); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - // Nothing pending → no exec call. - expect(exec).not.toHaveBeenCalled(); - expect(result.installed).toHaveLength(0); - expect(result.skipped.some((s) => s.reason === 'already-installed')).toBe(true); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('returns already-installed stubs in result.installed when localUrl is persisted', async () => { - // Regression for the streaming-mode bug where mediaUrlMap stayed empty - // on resume runs: with localUrl persisted to MediaStub, idempotent - // re-calls surface the mapping so flushPendingImports can rebuild - // its rewrite map without re-running the PHP installer. - const outputDir = mkdtempSync(join(FIXTURE_TMP, 'mi-resume-')); - const wpRoot = join(outputDir, 'site', 'wordpress'); - mkdirSync(wpRoot, { recursive: true }); - mkdirSync(join(outputDir, 'media'), { recursive: true }); - const filePath = join(outputDir, 'media', 'a.jpg'); - writeFileSync(filePath, Buffer.from('fake')); - - const store = MediaStubStore.load(outputDir); - store.markSuccess('https://cdn/a.jpg', filePath); - store.recordWpPostId('https://cdn/a.jpg', 42); - store.recordLocalUrl('https://cdn/a.jpg', 'http://localhost:8882/wp-content/uploads/2024/01/a.jpg'); - store.flush(); - - try { - const exec = vi.fn(); - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - expect(exec).not.toHaveBeenCalled(); - expect(result.skipped).toHaveLength(0); - expect(result.installed).toEqual([ - { - sourceUrl: 'https://cdn/a.jpg', - postId: 42, - // Stored + surfaced root-relative (port-independent) by the stub store. - localUrl: '/wp-content/uploads/2024/01/a.jpg', - localPath: filePath, - }, - ]); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('persists localUrl to the stub on fresh install (so resume runs can rebuild the map)', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/b.jpg', filename: 'b.jpg' }], - }); - try { - const exec = vi.fn().mockResolvedValue({ - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/b.jpg', filename: 'b.jpg', postId: 7, localUrl: 'http://localhost:8882/wp-content/uploads/2024/01/b.jpg' }, - ]), - stderr: '', - }); - - await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - const store = MediaStubStore.load(outputDir); - // PHP returns an absolute URL; the stub store persists it root-relative - // so the mapping survives a Studio site/port change. - expect(store.get('https://cdn/b.jpg')?.localUrl).toBe('/wp-content/uploads/2024/01/b.jpg'); - expect(store.get('https://cdn/b.jpg')?.wpPostId).toBe(7); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('skips stubs whose local file is missing', async () => { - const outputDir = mkdtempSync(join(FIXTURE_TMP, 'mi-missing-')); - const wpRoot = join(outputDir, 'site', 'wordpress'); - mkdirSync(wpRoot, { recursive: true }); - mkdirSync(join(outputDir, 'media'), { recursive: true }); - - // Stub recorded as success but the file isn't actually on disk. - const store = MediaStubStore.load(outputDir); - store.markSuccess('https://cdn/ghost.jpg', join(outputDir, 'media', 'ghost.jpg')); - store.flush(); - - try { - const exec = vi.fn(); - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - expect(exec).not.toHaveBeenCalled(); - expect(result.skipped).toEqual([{ sourceUrl: 'https://cdn/ghost.jpg', reason: 'no-local-file' }]); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('returns errors when the studio exec fails', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg' }], - }); - try { - const exec = vi.fn().mockRejectedValue(new Error('studio not found')); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - expect(result.installed).toEqual([]); - expect(result.errors).toHaveLength(1); - expect(result.errors[0].error).toMatch(/studio not found/); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('includes stderr/stdout details when the studio exec fails', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg' }], - }); - try { - const err = Object.assign(new Error('Command failed: studio wp eval-file'), { - stderr: 'Fatal error: database is locked', - stdout: 'wp-cli bootstrap output', - }); - const exec = vi.fn().mockRejectedValue(err); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - expect(result.errors).toHaveLength(1); - expect(result.errors[0].error).toContain('Fatal error: database is locked'); - expect(result.errors[0].error).toContain('wp-cli bootstrap output'); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('returns errors when the PHP response has no parseable JSON', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg' }], - }); - try { - const exec = vi.fn().mockResolvedValue({ stdout: 'unrelated wp-cli output without sentinels', stderr: '' }); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors[0].error).toMatch(/no parseable JSON/); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('reads the response from the sidecar result file (bypasses Studio 64KB stdout cap)', async () => { - // The script writes its full JSON to `.result.json` and emits only - // a tiny `{resultFile}` pointer to stdout. Simulate that: the mock locates - // the staged payload, writes the sidecar, and returns the pointer block. - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/big.jpg', filename: 'big.jpg' }], - }); - try { - const sitePath = join(outputDir, 'site'); - const payloadsDir = join(sitePath, '.dla-scripts', 'payloads'); - const exec = vi.fn().mockImplementation(async () => { - // Find the payload the real code just staged. - const { readdirSync } = await import('node:fs'); - const payloadFile = readdirSync(payloadsDir).find((f) => f.endsWith('.json') && !f.endsWith('.result.json')); - const payloadHostPath = join(payloadsDir, payloadFile!); - const fullResponse = JSON.stringify({ - results: [{ sourceUrl: 'https://cdn/big.jpg', filename: 'big.jpg', postId: 99, reused: false, localUrl: 'http://wp/uploads/2026/05/big.jpg' }], - errors: [], - }); - writeFileSync(`${payloadHostPath}.result.json`, fullResponse); - // stdout carries ONLY the small pointer between the sentinels. - return { - stdout: `noise\nDLA_INSTALL_MEDIA_JSON_BEGIN\n${JSON.stringify({ resultFile: `/wordpress/.dla-scripts/payloads/${payloadFile}.result.json` })}\nDLA_INSTALL_MEDIA_JSON_END\nmore noise\n`, - stderr: '', - }; - }); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - - expect(result.errors).toEqual([]); - expect(result.installed).toHaveLength(1); - expect(result.installed[0]).toMatchObject({ sourceUrl: 'https://cdn/big.jpg', postId: 99 }); - const store = MediaStubStore.load(outputDir); - expect(store.get('https://cdn/big.jpg')?.wpPostId).toBe(99); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('returns errors when the sidecar result file is missing/unreadable', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg' }], - }); - try { - // Pointer references a sidecar that was never written. - const exec = vi.fn().mockResolvedValue({ - stdout: `DLA_INSTALL_MEDIA_JSON_BEGIN\n${JSON.stringify({ resultFile: '/wordpress/.dla-scripts/payloads/nonexistent.json.result.json' })}\nDLA_INSTALL_MEDIA_JSON_END\n`, - stderr: '', - }); - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors[0].error).toMatch(/no parseable JSON/); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('reports per-stub errors that came back from PHP', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [ - { url: 'https://cdn/a.jpg', filename: 'a.jpg' }, - { url: 'https://cdn/b.jpg', filename: 'b.jpg' }, - ], - }); - try { - const exec = vi.fn().mockResolvedValue({ - stdout: 'noise\nDLA_INSTALL_MEDIA_JSON_BEGIN\n' + JSON.stringify({ - results: [{ sourceUrl: 'https://cdn/a.jpg', filename: 'a.jpg', postId: 1, reused: false, localUrl: 'http://l/a.jpg' }], - errors: [{ sourceUrl: 'https://cdn/b.jpg', filename: 'b.jpg', error: 'wp_insert_attachment returned 0' }], - }) + '\nDLA_INSTALL_MEDIA_JSON_END\n', - stderr: '', - }); - - const result = await installMediaForUrl({ - outputDir, - url: 'https://example.com/page', - wpRoot, - _execFile: exec, - }); - expect(result.installed).toHaveLength(1); - expect(result.errors).toHaveLength(1); - expect(result.errors[0].sourceUrl).toBe('https://cdn/b.jpg'); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); -}); - -describe('installMediaForUrl — SVG routing (svg survival)', () => { - it('substitutes the PNG sibling for risky SVGs without touching safe-svg', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/logo.svg', filename: 'logo.svg', svgRisky: true, rasterFilename: 'logo.png' }], - }); - try { - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - return { - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/logo.svg', filename: 'logo.png', postId: 5, localUrl: 'http://wp/uploads/2026/06/logo.png' }, - ]), - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - // No SVG left in the batch → no ensurePlugin traffic at all. - expect(pluginCalls(exec)).toHaveLength(0); - expect(exec).toHaveBeenCalledTimes(1); - const payload = readStagedPayload(outputDir, exec.mock.calls[0][1] as string[]); - expect(payload).toHaveLength(1); - expect(payload[0].filename).toBe('logo.png'); - expect(payload[0].sourceUrl).toBe('https://cdn/logo.svg'); - expect(result.errors).toEqual([]); - expect(result.installed).toHaveLength(1); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 1, svgFailed: 0, safeSvgEnsured: false }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('keeps clean SVGs as SVG and ensures safe-svg exactly once before the batch', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [ - { url: 'https://cdn/a.svg', filename: 'a.svg' }, - { url: 'https://cdn/b.svg', filename: 'b.svg' }, - ], - }); - try { - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - return { - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/a.svg', filename: 'a.svg', postId: 1, localUrl: 'http://wp/uploads/2026/06/a.svg' }, - { sourceUrl: 'https://cdn/b.svg', filename: 'b.svg', postId: 2, localUrl: 'http://wp/uploads/2026/06/b.svg' }, - ]), - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - // ensurePlugin ran once (one is-installed probe) and BEFORE the eval-file batch. - const isInstalledCalls = exec.mock.calls.filter(([, args]) => (args as string[]).includes('is-installed')); - expect(isInstalledCalls).toHaveLength(1); - expect((isInstalledCalls[0][1] as string[])).toContain('safe-svg'); - const firstPluginIdx = exec.mock.calls.findIndex(([, args]) => (args as string[]).includes('plugin')); - const evalIdx = exec.mock.calls.findIndex(([, args]) => (args as string[]).includes('eval-file')); - expect(firstPluginIdx).toBeGreaterThanOrEqual(0); - expect(firstPluginIdx).toBeLessThan(evalIdx); - - const payload = readStagedPayload(outputDir, exec.mock.calls[evalIdx][1] as string[]); - expect(payload.map((p) => p.filename).sort()).toEqual(['a.svg', 'b.svg']); - expect(result.errors).toEqual([]); - expect(result.svg).toEqual({ svgUploaded: 2, svgSubstituted: 0, svgFailed: 0, safeSvgEnsured: true }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('risky deduped SVG (no rasterPath) substitutes the on-disk PNG sibling via the dedup guard', async () => { - // Byte-duplicate SVG URLs dedupe at fetch: the stub points at the - // ORIGINAL's localPath but carries no rasterPath of its own. The - // original's sibling lives at exactly localPath with .svg → .png. - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/dup.svg', filename: 'shared.svg', svgRisky: true, sidecarPng: 'shared.png' }], - }); - try { - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - return { - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/dup.svg', filename: 'shared.png', postId: 8, localUrl: 'http://wp/uploads/2026/06/shared.png' }, - ]), - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - const payload = readStagedPayload(outputDir, exec.mock.calls[0][1] as string[]); - expect(payload[0].filename).toBe('shared.png'); - expect(pluginCalls(exec)).toHaveLength(0); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 1, svgFailed: 0, safeSvgEnsured: false }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('clean deduped SVG stays SVG even when a PNG sibling exists on disk', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/dup2.svg', filename: 'icon.svg', sidecarPng: 'icon.png' }], - }); - try { - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - return { - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/dup2.svg', filename: 'icon.svg', postId: 3, localUrl: 'http://wp/uploads/2026/06/icon.svg' }, - ]), - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - const evalCall = exec.mock.calls.find(([, args]) => (args as string[]).includes('eval-file'))!; - const payload = readStagedPayload(outputDir, evalCall[1] as string[]); - expect(payload[0].filename).toBe('icon.svg'); - expect(result.svg).toEqual({ svgUploaded: 1, svgSubstituted: 0, svgFailed: 0, safeSvgEnsured: true }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('ensurePlugin failure → mass PNG substitution + error stub for SVGs without raster', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [ - { url: 'https://cdn/c.svg', filename: 'c.svg', rasterFilename: 'c.png' }, - { url: 'https://cdn/d.svg', filename: 'd.svg' }, - ], - }); - try { - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('plugin')) throw new Error('no network'); - if (args.includes('eval-file')) { - return { - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/c.svg', filename: 'c.png', postId: 6, localUrl: 'http://wp/uploads/2026/06/c.png' }, - ]), - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - const evalCall = exec.mock.calls.find(([, args]) => (args as string[]).includes('eval-file'))!; - const payload = readStagedPayload(outputDir, evalCall[1] as string[]); - expect(payload.map((p) => p.filename)).toEqual(['c.png']); - expect(result.installed).toHaveLength(1); - expect(result.errors).toHaveLength(1); - expect(result.errors[0].sourceUrl).toBe('https://cdn/d.svg'); - expect(result.errors[0].error).toMatch(/safe-svg unavailable and no raster fallback/); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 1, svgFailed: 1, safeSvgEnsured: false }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('retries a per-file SVG insert failure once with the PNG sibling', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/e.svg', filename: 'e.svg', rasterFilename: 'e.png' }], - }); - try { - let evalCalls = 0; - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - evalCalls += 1; - if (evalCalls === 1) { - return { - stdout: 'DLA_INSTALL_MEDIA_JSON_BEGIN\n' + JSON.stringify({ - results: [], - errors: [{ sourceUrl: 'https://cdn/e.svg', filename: 'e.svg', error: 'svg_mime_rejected: image/svg+xml is not allowed on this site (Safe SVG inactive)' }], - }) + '\nDLA_INSTALL_MEDIA_JSON_END\n', - stderr: '', - }; - } - return { - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/e.svg', filename: 'e.png', postId: 9, localUrl: 'http://wp/uploads/2026/06/e.png' }, - ]), - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - expect(evalCalls).toBe(2); - const evalArgList = exec.mock.calls.filter(([, args]) => (args as string[]).includes('eval-file')); - const retryPayload = readStagedPayload(outputDir, evalArgList[1][1] as string[]); - expect(retryPayload.map((p) => p.filename)).toEqual(['e.png']); - expect(result.errors).toEqual([]); - expect(result.installed).toHaveLength(1); - expect(result.installed[0].postId).toBe(9); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 1, svgFailed: 0, safeSvgEnsured: true }); - // The PNG was copied into uploads for the retry batch. - const store = MediaStubStore.load(outputDir); - expect(store.get('https://cdn/e.svg')?.wpPostId).toBe(9); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('a failed PNG retry surfaces as svgFailed with a retry-tagged error', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/f.svg', filename: 'f.svg', rasterFilename: 'f.png' }], - }); - try { - let evalCalls = 0; - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - evalCalls += 1; - const failure = evalCalls === 1 - ? { sourceUrl: 'https://cdn/f.svg', filename: 'f.svg', error: 'svg_mime_rejected: nope' } - : { sourceUrl: 'https://cdn/f.svg', filename: 'f.png', error: 'wp_insert_attachment returned 0' }; - return { - stdout: 'DLA_INSTALL_MEDIA_JSON_BEGIN\n' + JSON.stringify({ results: [], errors: [failure] }) + '\nDLA_INSTALL_MEDIA_JSON_END\n', - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - expect(evalCalls).toBe(2); - expect(result.installed).toEqual([]); - expect(result.errors).toHaveLength(1); - expect(result.errors[0].error).toMatch(/svg png retry/); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 0, svgFailed: 1, safeSvgEnsured: true }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('a per-file SVG failure with no raster fallback stays an error (no retry batch)', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/g.svg', filename: 'g.svg' }], - }); - try { - let evalCalls = 0; - const exec = vi.fn().mockImplementation(async (_bin: string, args: string[]) => { - if (args.includes('eval-file')) { - evalCalls += 1; - return { - stdout: 'DLA_INSTALL_MEDIA_JSON_BEGIN\n' + JSON.stringify({ - results: [], - errors: [{ sourceUrl: 'https://cdn/g.svg', filename: 'g.svg', error: 'svg_mime_rejected: nope' }], - }) + '\nDLA_INSTALL_MEDIA_JSON_END\n', - stderr: '', - }; - } - return { stdout: '', stderr: '' }; - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - expect(evalCalls).toBe(1); - expect(result.errors).toHaveLength(1); - expect(result.errors[0].error).toMatch(/svg_mime_rejected/); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 0, svgFailed: 1, safeSvgEnsured: true }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); - - it('non-SVG batches never touch the plugin CLI and report a zero tally', async () => { - const { outputDir, wpRoot } = setup({ - stubs: [{ url: 'https://cdn/a.jpg', filename: 'a.jpg' }], - }); - try { - const exec = vi.fn().mockResolvedValue({ - stdout: SUCCESS_RESPONSE([ - { sourceUrl: 'https://cdn/a.jpg', filename: 'a.jpg', postId: 42, localUrl: 'http://wp/uploads/2024/01/a.jpg' }, - ]), - stderr: '', - }); - - const result = await installMediaForUrl({ outputDir, url: 'https://example.com/page', wpRoot, _execFile: exec }); - - expect(exec).toHaveBeenCalledTimes(1); - expect(pluginCalls(exec)).toHaveLength(0); - expect(result.svg).toEqual({ svgUploaded: 0, svgSubstituted: 0, svgFailed: 0, safeSvgEnsured: false }); - } finally { - rmSync(outputDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/media-install.ts b/packages/data-liberation-agent/src/lib/streaming/media-install.ts deleted file mode 100644 index 6d297b209b..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/media-install.ts +++ /dev/null @@ -1,650 +0,0 @@ -// -// Per-URL media install -// ===================== -// Phase 1.5 of the streaming/incremental replicate pipeline. For each URL -// processed by the streaming loop, install pending media into the running -// Studio replica WP site so pages render with real images while streaming. -// -// Behavior: -// - Reads MediaStubStore.list() for all stubs in `success` state with -// `localPath` set and no `wpPostId` (i.e., not yet installed). -// - Copies each file from /media/ into -// /wp-content/uploads/// based on the -// local file's mtime (matches WP's default uploads layout). -// - Invokes a vendored PHP script (install-media.php) via `studio wp -// eval-file` that runs `wp_insert_attachment` for each entry. -// The script is idempotent: it checks `_wp_attached_file` first and -// re-uses an existing attachment ID when present. -// - Records the resulting post ID back into MediaStubStore via -// `recordWpPostId(url, postId)` so subsequent calls skip the URL. -// -// Scope: -// - Per the contract, this installs ALL pending media each call. The -// existing MediaStubStore doesn't track URL→media membership, so -// scoping to one URL's media isn't possible without a schema change. -// Idempotency keeps duplicate calls cheap. -// -import { copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; -import { dirname, join, resolve } from 'node:path'; -import { execFile } from 'node:child_process'; -import { promisify } from 'node:util'; -import { fileURLToPath } from 'node:url'; -import { MediaStubStore, type MediaStub } from '../resume-state/index.js'; -import { ensurePlugin, type ExecFn } from '../preview/ensure-plugin.js'; -import { studioExecFileAsync } from '../studio-cli.js'; - -const execFileAsync = promisify(execFile); - -/** Vendored PHP installer that runs inside the running WP site via wp-cli. */ -const INSTALL_MEDIA_SCRIPT = resolve( - dirname(fileURLToPath(import.meta.url)), - '..', - 'preview', - 'scripts', - 'install-media.php', -); - -/** - * Studio mounts the host site directory at VFS path `/wordpress` (mirrors - * studio.ts's STUDIO_VFS_ROOT constant). Re-declared here to avoid a - * cross-module dependency for a path constant. - */ -const STUDIO_VFS_ROOT = '/wordpress'; - -const SCRIPTS_SUBDIR = '.dla-scripts'; -const PAYLOADS_SUBDIR = '.dla-scripts/payloads'; - -/** Monotonic per-process payload counter — see payloadFilename below. */ -let payloadSeq = 0; - -export interface MediaInstallOpts { - /** Liberation output directory containing media/ and media-stubs.json. */ - outputDir: string; - /** Source URL whose media we're installing — kept for trace logging. */ - url: string; - /** Running Studio WP install root (e.g. /wordpress or for flat sites). */ - wpRoot: string; - /** Override the studio binary location (for tests). */ - _studioBin?: string; - /** Inject an exec-file impl (for tests). */ - _execFile?: (file: string, args: readonly string[]) => Promise<{ stdout: string; stderr: string }>; -} - -/** Install-time SVG routing tally (svg survival, F1). */ -export interface SvgInstallTally { - /** SVG-origin assets that landed in the library as SVG. */ - svgUploaded: number; - /** SVG-origin assets that landed as their rasterized PNG sibling. */ - svgSubstituted: number; - /** SVG-origin assets that failed to land at all. */ - svgFailed: number; - /** True when ensurePlugin('safe-svg') ran and succeeded for this batch. */ - safeSvgEnsured: boolean; -} - -export interface MediaInstallResult { - installed: Array<{ sourceUrl: string; postId: number; localUrl: string; localPath: string }>; - skipped: Array<{ sourceUrl: string; reason: 'already-installed' | 'no-local-file' | 'no-stub' }>; - errors: Array<{ sourceUrl: string; error: string }>; - svg: SvgInstallTally; -} - -export interface MediaFile { - absPath: string; - sourceUrl: string; -} - -export interface MediaFilesResult { - installed: Array<{ sourceUrl: string; postId: number; localUrl: string }>; - errors: Array<{ sourceUrl: string; error: string }>; -} - -export interface MediaFilesInstallOpts { - files: MediaFile[]; - wpRoot: string; - _studioBin?: string; - _execFile?: (file: string, args: readonly string[]) => Promise<{ stdout: string; stderr: string }>; -} - -interface PayloadEntry { - filename: string; - year: string; - month: string; - sourceUrl: string; -} - -interface PendingItem { - url: string; - stub: MediaStub; - entry: PayloadEntry; - absPath: string; - /** Set when the stub's local file is an SVG — drives install-time routing. */ - svgOrigin?: boolean; - /** Resolved absolute path of the PNG raster sibling (stub field or dedup-guard derivation). */ - rasterAbs?: string | null; - /** True once the entry was rerouted to upload the PNG instead of the SVG. */ - substituted?: boolean; - /** Dropped from the batch entirely (safe-svg unavailable + no raster fallback). */ - dropped?: boolean; -} - -interface PhpResultEntry { - sourceUrl: string; - filename: string; - postId: number; - reused: boolean; - localUrl: string; -} - -interface PhpErrorEntry { - sourceUrl: string; - filename: string; - error: string; -} - -interface PhpResponse { - results: PhpResultEntry[]; - errors: PhpErrorEntry[]; -} - -export async function installMediaFiles(opts: MediaFilesInstallOpts): Promise { - const result: MediaFilesResult = { installed: [], errors: [] }; - const pending: Array<{ file: MediaFile; entry: PayloadEntry }> = []; - - for (const file of opts.files) { - let mtime: Date; - try { - mtime = statSync(file.absPath).mtime; - } catch { - result.errors.push({ sourceUrl: file.sourceUrl, error: 'source file is missing or unstattable' }); - continue; - } - - const filename = basenameOf(file.absPath); - const year = String(mtime.getFullYear()).padStart(4, '0'); - const month = String(mtime.getMonth() + 1).padStart(2, '0'); - pending.push({ file, entry: { filename, year, month, sourceUrl: file.sourceUrl } }); - } - - if (pending.length === 0) { - return result; - } - - // Copy each file into the running site's uploads dir before wp_insert_attachment. - const uploadsRoot = join(resolve(opts.wpRoot), 'wp-content', 'uploads'); - for (const item of pending) { - const destDir = join(uploadsRoot, item.entry.year, item.entry.month); - const destPath = join(destDir, item.entry.filename); - try { - mkdirSync(destDir, { recursive: true }); - if (!existsSync(destPath)) { - copyFileSync(item.file.absPath, destPath); - } - } catch (err) { - result.errors.push({ sourceUrl: item.file.sourceUrl, error: `copy: ${(err as Error).message}` }); - } - } - - const copied = pending.filter( - (p) => !result.errors.find((e) => e.sourceUrl === p.file.sourceUrl), - ); - if (copied.length === 0) { - return result; - } - - let scriptOut: { stdout: string; resultHostPath: string }; - try { - scriptOut = await installViaStudio(opts, copied.map((p) => p.entry)); - } catch (err) { - for (const item of copied) { - result.errors.push({ - sourceUrl: item.file.sourceUrl, - error: `wp eval-file install-media.php failed: ${formatExecError(err)}`, - }); - } - return result; - } - - const parsed = parsePhpResponse(scriptOut.stdout, scriptOut.resultHostPath); - if (!parsed) { - for (const item of copied) { - result.errors.push({ - sourceUrl: item.file.sourceUrl, - error: 'install-media.php produced no parseable JSON response', - }); - } - return result; - } - - for (const ok of parsed.results) { - result.installed.push({ - sourceUrl: ok.sourceUrl, - postId: ok.postId, - localUrl: ok.localUrl, - }); - } - for (const fail of parsed.errors) { - result.errors.push({ sourceUrl: fail.sourceUrl, error: fail.error }); - } - - return result; -} - -/** Single entry-point. Always opens MediaStubStore in-place. */ -export async function installMediaForUrl(opts: MediaInstallOpts): Promise { - const result: MediaInstallResult = { - installed: [], - skipped: [], - errors: [], - svg: { svgUploaded: 0, svgSubstituted: 0, svgFailed: 0, safeSvgEnsured: false }, - }; - const stubs = MediaStubStore.load(opts.outputDir); - const mediaDir = join(resolve(opts.outputDir), 'media'); - - // 1. Walk every stub and bucket it: ready-to-install, already-done, - // not-locally-downloaded, etc. - const pending: PendingItem[] = []; - for (const [url, stub] of stubs.list()) { - if (stub.status !== 'success' || !stub.localPath) { - result.skipped.push({ sourceUrl: url, reason: 'no-local-file' }); - continue; - } - if (typeof stub.wpPostId === 'number') { - // Already-installed: surface the persisted localUrl in `installed` - // so the run-wide rewrite map can be (re-)built from this call's - // result alone, even on resume runs where the PHP script wouldn't - // re-run for these entries. Falls back to `skipped` when the stub - // pre-dates the localUrl persistence change. - if (stub.localUrl) { - result.installed.push({ - sourceUrl: url, - postId: stub.wpPostId, - localUrl: stub.localUrl, - localPath: stub.localPath ?? '', - }); - } else { - result.skipped.push({ sourceUrl: url, reason: 'already-installed' }); - } - continue; - } - - // Resolve the canonical filename. localPath may be absolute (older - // adapters) or just a basename — handle both. Source-of-truth is - // /media/. - const filename = basenameOf(stub.localPath); - const absPath = join(mediaDir, filename); - if (!existsSync(absPath)) { - result.skipped.push({ sourceUrl: url, reason: 'no-local-file' }); - continue; - } - let mtime: Date; - try { - mtime = statSync(absPath).mtime; - } catch { - // Treat unstattable files as missing — should be rare; surfaces as - // a skip rather than a hard failure. - result.skipped.push({ sourceUrl: url, reason: 'no-local-file' }); - continue; - } - const year = String(mtime.getFullYear()).padStart(4, '0'); - const month = String(mtime.getMonth() + 1).padStart(2, '0'); - - pending.push({ url, stub, entry: { filename, year, month, sourceUrl: url }, absPath }); - } - - if (pending.length === 0) { - return result; - } - - // 1.5 SVG routing (svg survival, F1). Default WP rejects image/svg+xml, so - // before the PHP batch each SVG-origin item is routed: - // - risky SVG (Safe SVG's sanitizer would mangle its / graph) - // with a PNG raster sibling → upload the PNG instead; - // - any SVG still in the batch → ensurePlugin('safe-svg') ONCE first; - // when that fails, fall back to PNG for every SVG that has a raster and - // error-stub the rest. - const svgItems = pending.filter((p) => /\.svg$/i.test(p.entry.filename)); - for (const item of svgItems) { - item.svgOrigin = true; - item.rasterAbs = resolveRasterAbs(item.stub, mediaDir); - if (item.stub.svgRisky === true && item.rasterAbs) { - substituteRaster(item); - } - } - const svgStillInBatch = svgItems.filter((p) => !p.substituted); - if (svgStillInBatch.length > 0) { - const ensured = await ensurePlugin( - studioSitePathForWpRoot(opts.wpRoot), - 'safe-svg', - wpExecFor(opts), - ); - if (ensured.ok) { - result.svg.safeSvgEnsured = true; - } else { - for (const item of svgStillInBatch) { - if (item.rasterAbs) { - substituteRaster(item); - } else { - item.dropped = true; - result.errors.push({ - sourceUrl: item.url, - error: `safe-svg unavailable and no raster fallback (${ensured.error})`, - }); - } - } - } - } - const batch = pending.filter((p) => !p.dropped); - if (batch.length === 0) { - finalizeSvgTally(result, svgItems); - return result; - } - - // 2. Copy each pending file into the running site's uploads dir. - // This must happen BEFORE the PHP script runs — wp_insert_attachment - // requires the file to exist on disk to compute metadata. - const uploadsRoot = join(resolve(opts.wpRoot), 'wp-content', 'uploads'); - for (const item of batch) { - const destDir = join(uploadsRoot, item.entry.year, item.entry.month); - const destPath = join(destDir, item.entry.filename); - try { - mkdirSync(destDir, { recursive: true }); - // Idempotent: if the file is already in place, skip the copy. - if (!existsSync(destPath)) { - copyFileSync(item.absPath, destPath); - } - } catch (err) { - result.errors.push({ sourceUrl: item.url, error: `copy: ${(err as Error).message}` }); - } - } - - // Drop any items whose copy failed before invoking PHP. - const installedFiles = batch.filter( - (p) => !result.errors.find((e) => e.sourceUrl === p.url), - ); - if (installedFiles.length === 0) { - finalizeSvgTally(result, svgItems); - return result; - } - - // 3. Stage payload + invoke wp eval-file. - let scriptOut: { stdout: string; resultHostPath: string }; - try { - scriptOut = await installViaStudio(opts, installedFiles.map((p) => p.entry)); - } catch (err) { - // The shell-level failure means none of the entries got registered. - // Each pending entry surfaces as an error so the caller can retry. - for (const item of installedFiles) { - result.errors.push({ - sourceUrl: item.url, - error: `wp eval-file install-media.php failed: ${formatExecError(err)}`, - }); - } - finalizeSvgTally(result, svgItems); - return result; - } - - // 4. Parse the script's response and reconcile with the stub store. - const parsed = parsePhpResponse(scriptOut.stdout, scriptOut.resultHostPath); - if (!parsed) { - for (const item of installedFiles) { - result.errors.push({ - sourceUrl: item.url, - error: 'install-media.php produced no parseable JSON response', - }); - } - finalizeSvgTally(result, svgItems); - return result; - } - - // 4.5 Per-file SVG retry: an SVG that PHP rejected per-file (e.g. the - // svg_mime_rejected marker when Safe SVG didn't take) gets ONE retry as its - // PNG sibling in a second mini-batch. Everything else flows straight through - // as an error. - const phpResults: PhpResultEntry[] = [...parsed.results]; - const retryable: PendingItem[] = []; - for (const fail of parsed.errors) { - const item = installedFiles.find((p) => p.url === fail.sourceUrl); - if (item?.svgOrigin && !item.substituted && item.rasterAbs) { - retryable.push(item); - } else { - result.errors.push({ sourceUrl: fail.sourceUrl, error: fail.error }); - } - } - if (retryable.length > 0) { - const copied: PendingItem[] = []; - for (const item of retryable) { - substituteRaster(item); - try { - const destDir = join(uploadsRoot, item.entry.year, item.entry.month); - mkdirSync(destDir, { recursive: true }); - const destPath = join(destDir, item.entry.filename); - if (!existsSync(destPath)) { - copyFileSync(item.absPath, destPath); - } - copied.push(item); - } catch (err) { - result.errors.push({ sourceUrl: item.url, error: `svg png retry copy: ${(err as Error).message}` }); - } - } - if (copied.length > 0) { - try { - const retryOut = await installViaStudio(opts, copied.map((p) => p.entry)); - const parsedRetry = parsePhpResponse(retryOut.stdout, retryOut.resultHostPath); - if (parsedRetry) { - phpResults.push(...parsedRetry.results); - for (const fail of parsedRetry.errors) { - result.errors.push({ sourceUrl: fail.sourceUrl, error: `svg png retry: ${fail.error}` }); - } - } else { - for (const item of copied) { - result.errors.push({ - sourceUrl: item.url, - error: 'svg png retry: install-media.php produced no parseable JSON response', - }); - } - } - } catch (err) { - for (const item of copied) { - result.errors.push({ sourceUrl: item.url, error: `svg png retry failed: ${formatExecError(err)}` }); - } - } - } - } - - for (const ok of phpResults) { - if (typeof ok.postId === 'number' && ok.postId > 0) { - stubs.recordWpPostId(ok.sourceUrl, ok.postId); - } - if (ok.localUrl) { - // Persist the localUrl to the stub so resume runs can rebuild the - // source→local rewrite map without re-running the PHP script. - stubs.recordLocalUrl(ok.sourceUrl, ok.localUrl); - } - const stub = stubs.get(ok.sourceUrl); - result.installed.push({ - sourceUrl: ok.sourceUrl, - postId: ok.postId, - // Prefer the store's normalized (root-relative) localUrl so the run-wide - // rewrite map is port-independent; fall back to the raw upload URL. - localUrl: stub?.localUrl ?? ok.localUrl, - localPath: stub?.localPath ?? '', - }); - } - finalizeSvgTally(result, svgItems); - return result; -} - -/** - * Resolve the absolute on-disk path of an SVG stub's PNG raster sibling. - * Primary source is the `rasterPath` recorded at fetch time. Dedup guard: - * byte-duplicate SVG URLs dedupe at fetch, so a deduped URL's stub points at - * the ORIGINAL's localPath but carries no rasterPath of its own — the - * original's sibling lives at exactly localPath with `.svg` → `.png` (modulo - * the rare `-N` collision suffix, in which case we miss and the SVG continues - * alone). - */ -function resolveRasterAbs(stub: MediaStub, mediaDir: string): string | null { - if (stub.rasterPath) { - const abs = join(mediaDir, basenameOf(stub.rasterPath)); - if (existsSync(abs)) return abs; - return existsSync(stub.rasterPath) ? stub.rasterPath : null; - } - if (stub.localPath && /\.svg$/i.test(stub.localPath)) { - const abs = join(mediaDir, basenameOf(stub.localPath).replace(/\.svg$/i, '.png')); - return existsSync(abs) ? abs : null; - } - return null; -} - -/** Reroute a pending SVG item to upload its PNG raster sibling instead. */ -function substituteRaster(item: PendingItem): void { - item.entry.filename = basenameOf(item.rasterAbs!); - item.absPath = item.rasterAbs!; - item.substituted = true; -} - -/** - * Count each SVG-origin item exactly once: installed-as-SVG, installed-as-PNG, - * or failed. Called on every post-routing exit path so the tally is accurate - * even when the batch aborts early. - */ -function finalizeSvgTally(result: MediaInstallResult, svgItems: PendingItem[]): void { - for (const item of svgItems) { - const ok = result.installed.some((i) => i.sourceUrl === item.url); - if (!ok) result.svg.svgFailed += 1; - else if (item.substituted) result.svg.svgSubstituted += 1; - else result.svg.svgUploaded += 1; - } -} - -/** - * Adapt this module's injected exec into ensurePlugin's StudioWpRunner shape - * (`studio wp --path <...args>` → stdout). - */ -function wpExecFor(opts: MediaInstallOpts): ExecFn { - const studioBin = opts._studioBin ?? 'studio'; - const exec = opts._execFile ?? defaultExec; - return (sitePath, args) => exec(studioBin, ['wp', '--path', sitePath, ...args]).then((o) => o.stdout); -} - -async function installViaStudio(opts: Pick, entries: PayloadEntry[]): Promise<{ stdout: string; resultHostPath: string }> { - // The PHP script must be readable inside Studio's VFS. Studio mounts the - // *site* directory at /wordpress. Studio sites exist in two layouts: - // - flat: /wp-content - // - nested: /wordpress/wp-content - // The watch runner passes the WP root, so resolve it back to the Studio - // site path before invoking `studio wp --path`. - const sitePath = studioSitePathForWpRoot(opts.wpRoot); - const scriptsDir = join(sitePath, SCRIPTS_SUBDIR); - const payloadsDir = join(sitePath, PAYLOADS_SUBDIR); - mkdirSync(scriptsDir, { recursive: true }); - mkdirSync(payloadsDir, { recursive: true }); - - const scriptHostPath = join(scriptsDir, 'install-media.php'); - copyFileSync(INSTALL_MEDIA_SCRIPT, scriptHostPath); - - // Sequence suffix: the SVG retry mini-batch can fire within the same - // millisecond as the main batch — Date.now()+pid alone would collide and - // overwrite the first payload + sidecar result file. - const payloadFilename = `install-media-${Date.now()}-${process.pid}-${++payloadSeq}.json`; - const payloadHostPath = join(payloadsDir, payloadFilename); - writeFileSync(payloadHostPath, JSON.stringify(entries), 'utf8'); - - const scriptVfsPath = `${STUDIO_VFS_ROOT}/${SCRIPTS_SUBDIR}/install-media.php`; - const payloadVfsPath = `${STUDIO_VFS_ROOT}/${PAYLOADS_SUBDIR}/${payloadFilename}`; - - const studioBin = opts._studioBin ?? 'studio'; - const exec = opts._execFile ?? defaultExec; - const out = await exec(studioBin, [ - 'wp', '--path', sitePath, - 'eval-file', scriptVfsPath, payloadVfsPath, - ]); - // The script writes its full response to `.result.json` on the host - // FS (Studio mounts the site dir), so we can read it directly and bypass the - // 64KB stdout cap. - return { stdout: out.stdout, resultHostPath: `${payloadHostPath}.result.json` }; -} - -function studioSitePathForWpRoot(wpRoot: string): string { - const resolved = resolve(wpRoot); - if (basenameOf(resolved) === 'wordpress') { - return dirname(resolved); - } - return resolved; -} - -function defaultExec(file: string, args: readonly string[]): Promise<{ stdout: string; stderr: string }> { - const opts = { timeout: 300_000, maxBuffer: 50 * 1024 * 1024 }; - // 'studio' resolves via studio-cli (Windows .cmd shims can't be spawned - // directly — STU-2020); an overridden _studioBin path spawns as-is. - const run = file === 'studio' - ? studioExecFileAsync(args as string[], opts) - : execFileAsync(file, args as string[], opts); - return run.then(({ stdout, stderr }) => ({ stdout: String(stdout), stderr: String(stderr) })); -} - -function formatExecError(err: unknown): string { - const e = err as Error & { stderr?: string; stdout?: string }; - const parts = [e?.message ? e.message.trim() : String(err)]; - if (e?.stderr?.trim()) parts.push(`stderr: ${e.stderr.trim().slice(-1000)}`); - if (e?.stdout?.trim()) parts.push(`stdout: ${e.stdout.trim().slice(-1000)}`); - return parts.join(' | '); -} - -/** - * Extract the JSON payload between the script's BEGIN/END sentinels. Returns - * null when the sentinels are missing or the body fails to parse — the caller - * surfaces a generic error in either case. - * - * Two body shapes are supported: - * 1. A `{ resultFile: "" }` pointer — the script wrote the full - * response to a sidecar file (default; bypasses Studio's 64KB stdout cap). - * We read that file off the host FS. `resultHostPath` is the host path the - * caller knows; we prefer it over the (VFS) path the script reports so the - * read works regardless of mount mapping. - * 2. Inline JSON (backward-compatible fallback for small payloads or when the - * sidecar write failed). - */ -function parsePhpResponse(stdout: string, resultHostPath?: string): PhpResponse | null { - const begin = 'DLA_INSTALL_MEDIA_JSON_BEGIN'; - const end = 'DLA_INSTALL_MEDIA_JSON_END'; - const start = stdout.indexOf(begin); - const stop = stdout.indexOf(end); - if (start < 0 || stop < 0 || stop <= start) return null; - const body = stdout.slice(start + begin.length, stop).trim(); - - let raw = body; - try { - const maybePointer = JSON.parse(body) as { resultFile?: string }; - if (maybePointer && typeof maybePointer.resultFile === 'string') { - // Prefer the host path the caller computed; fall back to the path the - // script reported (only valid when host FS === reported path). - const path = resultHostPath ?? maybePointer.resultFile; - try { - raw = readFileSync(path, 'utf8'); - } catch { - return null; - } - } - } catch { - // Not JSON at all → fall through; the parse below will fail and return null. - } - - try { - const parsed = JSON.parse(raw) as PhpResponse; - if (!parsed || !Array.isArray(parsed.results) || !Array.isArray(parsed.errors)) { - return null; - } - return parsed; - } catch { - return null; - } -} - -/** Cross-platform basename — avoids path.basename pitfalls on mixed separators. */ -function basenameOf(p: string): string { - const trimmed = p.replace(/[\\/]+$/, ''); - const idx = Math.max(trimmed.lastIndexOf('/'), trimmed.lastIndexOf('\\')); - return idx >= 0 ? trimmed.slice(idx + 1) : trimmed; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.test.ts b/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.test.ts index 7a7f775fe5..48d1ffe3d0 100644 --- a/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.test.ts +++ b/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.test.ts @@ -50,6 +50,13 @@ describe('rewriteMediaUrls', () => { expect(out).toContain('https://cdn/unknown.jpg'); }); + it('ignores a root-path mapping that would rewrite every slash', () => { + const html = 'About'; + const map = new Map([['/', 'https://example.com/']]); + + expect(rewriteMediaUrls(html, map)).toBe(html); + }); + it('reports unmapped URLs via onMissing callback', () => { const html = ''; const onMissing = vi.fn(); @@ -138,7 +145,7 @@ describe('rewriteMediaUrls', () => { it('rewrites a Wix srcset whose display filename contains parentheses (no `).png` mangle)', () => { // The Wix logo srcset ends each variant with the display name `… (1).png`. - // URL_LIKE must not truncate at the `)`, or the rewrite leaves `).png`. + // URL extraction must not truncate at the `)`, or the rewrite leaves `).png`. const hash = '670df9_dc553b632f22456e8f3e591105cdc3da'; const base = `https://static.wixstatic.com/media/${hash}~mv2.png`; const local = `http://localhost:8884/wp-content/uploads/2026/05/Cornelius-Holmes-1.png`; @@ -153,6 +160,21 @@ describe('rewriteMediaUrls', () => { expect(out).not.toContain('static.wixstatic.com'); expect(out).not.toContain(').png'); // the mangle signature }); + + it("rewrites Wix display filenames containing apostrophes without suffix corruption", () => { + const hash = '670df9_dc553b632f22456e8f3e591105cdc3da'; + const base = `https://static.wixstatic.com/media/${hash}~mv2.jpg`; + const local = 'http://localhost:8884/wp-content/uploads/2026/05/womens-day.jpg'; + const variant = `${base}/v1/fill/w_640,h_480,q_85,enc_avif,quality_auto/Happy%20Women's%20Day.jpg`; + const html = ``; + + const out = rewriteMediaUrls(html, new Map([[base, local]])); + + expect(out).toBe(``); + expect(out).not.toContain("Women's%20Day.jpg"); + expect(out).not.toContain('data:image/gif;base64,'); + expect(out).not.toContain(`${local}'s%20Day.jpg`); + }); }); describe('toLocalUrlMapping', () => { diff --git a/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.ts b/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.ts index adc5834dc8..e271fe3995 100644 --- a/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.ts +++ b/packages/data-liberation-agent/src/lib/streaming/media-url-rewrite.ts @@ -88,7 +88,9 @@ export function rewriteMediaUrls( // (a 404). Longest-first guarantees the most-specific (full) url is replaced // before any shorter substring of it. const ordered = [...replacements.entries()] - .filter(([source]) => source) + // A same-origin media URL can produce `/` as an alias. Replacing that + // substring would corrupt every path, closing tag, and MIME type in the document. + .filter(([source]) => source && source !== '/') .sort((a, b) => b[0].length - a[0].length); for (const [source, local] of ordered) { // Escape the source URL for safe inclusion in a RegExp. This handles @@ -125,8 +127,9 @@ export function toLocalUrlMapping( // truncate the URL at `(1`, and the rewrite would then swap only the prefix — // leaving `).png` (a 404). Candidates are extracted from quoted attribute // surfaces / srcset (whitespace- and comma-delimited), so a literal `)` is part -// of the URL, never a delimiter. -const URL_LIKE = /https?:\/\/[^\s"'<>\\]+/g; +// of the URL, never a delimiter. Attribute values are bounded before this +// matcher runs, so apostrophes remain valid inside double-quoted URLs. +const URL_LIKE = /https?:\/\/[^\s"<>\\]+/g; /** * Collect plausible media URLs from common attribute surfaces. We don't try @@ -137,16 +140,16 @@ function collectMediaCandidates(input: string): string[] { const candidates: string[] = []; // Direct attribute-style matches first — high signal. const attrPatterns: RegExp[] = [ - /]*\bsrc\s*=\s*["']([^"']+)["']/gi, - /]*\bhref\s*=\s*["']([^"']+\.(?:jpe?g|png|gif|webp|svg|avif|mp4|webm|pdf))["']/gi, - /\bsrcset\s*=\s*["']([^"']+)["']/gi, + /]*\bsrc\s*=\s*(["'])([\s\S]*?)\1/gi, + /]*\bhref\s*=\s*(["'])([\s\S]*?\.(?:jpe?g|png|gif|webp|svg|avif|mp4|webm|pdf))\1/gi, + /\bsrcset\s*=\s*(["'])([\s\S]*?)\1/gi, /"src"\s*:\s*"([^"]+)"/g, /"url"\s*:\s*"([^"]+)"/g, ]; for (const re of attrPatterns) { let m: RegExpExecArray | null; while ((m = re.exec(input)) !== null) { - const value = m[1]; + const value = m[2] ?? m[1]; // srcset can contain multiple URLs — extract via URL_LIKE so that Wix // transform URLs (which embed commas in their parameter segments, e.g. // `/v1/fill/w_680,h_510,q_90,enc_avif,quality_auto/`) are captured diff --git a/packages/data-liberation-agent/src/lib/streaming/output-verify.test.ts b/packages/data-liberation-agent/src/lib/streaming/output-verify.test.ts deleted file mode 100644 index 5e0cb07ca5..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/output-verify.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { verifyComposedOutput } from './output-verify.js'; - -describe('verifyComposedOutput', () => { - it('passes when every text node appears in source plain text', () => { - const source = '

Welcome to Foo Industries

We make widgets for the modern era.

'; - const blocks = ` -

Welcome to Foo Industries

- - - -

We make widgets for the modern era.

-`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - expect(result.hallucinated).toEqual([]); - }); - - it('fails when output substitutes a different brand name', () => { - const source = '

Foo Industries

About us.

'; - const blocks = `

Bar Inc

About us.

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(false); - expect(result.hallucinated).toContain('Bar Inc'); - }); - - it('treats wp: block comments as metadata, not text', () => { - // Block names like `wp:cover` or attribute slugs like `accent-primary` - // are NOT user-facing copy — they should be ignored even if they don't - // appear in source plain text. - const source = '

Hello

'; - const blocks = `

Hello

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - }); - - it('is case-insensitive for substring matching', () => { - const source = '

welcome to foo industries

'; - const blocks = `

Welcome to Foo Industries

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - }); - - it('normalizes whitespace before comparison', () => { - const source = '

We make\n\nwidgets.

'; - const blocks = `

We make widgets.

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - }); - - it('decodes HTML entities in both source and output before comparison', () => { - const source = '

Tom & Jerry

'; - const blocks = `

Tom & Jerry

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - }); - - it('skips trivial text nodes (very short / pure punctuation)', () => { - // The dash and emoji-like glyph aren't in source, but they're stylistic - // and well below the alnum threshold, so they should not trip the check. - const source = '

Real content here

'; - const blocks = `

Real content here

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - }); - - it('reports multiple hallucinations independently', () => { - const source = '

The original text only.

'; - const blocks = `

Hallucinated heading

-

Made-up paragraph copy.

`; - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(false); - expect(result.hallucinated.length).toBeGreaterThanOrEqual(2); - }); - - it('accepts source as raw HTML — extracts plain text internally', () => { - const source = '

Hello

World

'; - const blocks = `

Hello World

`; - // "Hello World" appears in the plain-text extraction even though - //

Hello

World

separates the words by a tag. - const result = verifyComposedOutput(blocks, source); - expect(result.valid).toBe(true); - }); - - it('handles empty markup gracefully', () => { - const result = verifyComposedOutput('', '

anything

'); - expect(result.valid).toBe(true); - expect(result.hallucinated).toEqual([]); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/output-verify.ts b/packages/data-liberation-agent/src/lib/streaming/output-verify.ts deleted file mode 100644 index 97fcd9dd5f..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/output-verify.ts +++ /dev/null @@ -1,121 +0,0 @@ -// -// Block-output verifier (post-skill defense layer 2) -// ================================================== -// After the `compose-page-blocks` skill emits block markup, confirm every -// text node in that markup is also present in the source HTML's plain text. -// This is an anti-hallucination check: an LLM that "helpfully" rewrites -// "Foo Industries" to "Bar Inc" gets caught here, and the apply pipeline -// preserves the raw post_content instead of overwriting with hallucinated -// copy. -// -// Strategy: -// 1. Strip block-comment delimiters (`` / ``) -// from the markup. The block-attribute JSON blob inside the open -// comment is metadata, not user-facing copy — verifying its slugs -// against source text would be wrong. -// 2. Tokenize the remaining HTML into text nodes (between tags). Treat any -// whitespace-collapsed chunk as a "candidate text node." -// 3. For each non-trivial text node, check it appears as a substring of -// the whitespace-normalized, lowercased plain text of the source HTML. -// 4. Return the list of nodes that didn't match (`hallucinated`). -// -// Trivial nodes (single punctuation, numbers shorter than 3 chars, etc.) -// are skipped because LLM outputs commonly include emoji-like glyphs or -// stylistic markers (—, ›) that aren't always present in source. -// -// Comparison: -// - Case-insensitive (lowercase both) -// - Whitespace-normalized (collapse runs of whitespace to single spaces) -// - Substring match (allows source "Welcome to Foo Industries, Inc." to -// accept output text "Foo Industries, Inc.") -// - -export interface VerifyResult { - valid: boolean; - /** Text nodes from the block markup that are NOT present in the source. */ - hallucinated: string[]; -} - -/** Strip the open and close `` comments from block markup. */ -function stripBlockComments(markup: string): string { - // Block opens: `` or `` - // Block closes: `` - // We deliberately only strip wp: comments — generic HTML comments would - // already have been removed by the pre-skill sanitizer, but if any survive - // they should NOT be treated as text content (they're metadata). - return markup - .replace(//g, ' ') - .replace(//g, ' '); -} - -/** Strip all HTML tags, returning whitespace-collapsed plain text. */ -function htmlToPlainText(html: string): string { - if (!html) return ''; - return html - .replace(/<[^>]+>/g, ' ') - .replace(/ /g, ' ') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/'/g, "'") - .replace(/\s+/g, ' ') - .trim(); -} - -/** Pull each text node (chunk between tags / block comments) out of the markup. */ -function extractTextNodes(markup: string): string[] { - const stripped = stripBlockComments(markup); - const nodes: string[] = []; - // Split on tag boundaries; each piece between tags is a candidate text node. - // We keep punctuation-and-whitespace nodes out of the result because they - // would always trivially match (and obscure real hallucinations). - const parts = stripped.split(/<[^>]+>/); - for (const raw of parts) { - const text = htmlToPlainText(raw); - if (!text) continue; - // Skip trivial chunks (single short tokens like dates, numbers, punctuation). - // We require at least 3 alphanumeric characters to avoid matching against - // common stylistic adornments. - const alnum = text.replace(/[^a-zA-Z0-9]/g, ''); - if (alnum.length < 3) continue; - nodes.push(text); - } - return nodes; -} - -/** Normalize for substring comparison: lowercase + collapse internal whitespace. */ -function normalize(text: string): string { - return text.toLowerCase().replace(/\s+/g, ' ').trim(); -} - -/** - * Verify every textual chunk emitted by the skill is grounded in the source - * HTML's plain text. Returns `valid: false` plus the offending chunks if any - * text was hallucinated (i.e. doesn't appear as a substring of the source). - */ -export function verifyComposedOutput( - blocksMarkup: string, - sourceHtmlPlainText: string, -): VerifyResult { - // Allow callers to pass either raw HTML or pre-extracted plain text — we - // run the same plain-text extraction either way so tags don't trip the - // substring search. - const sourceText = normalize(htmlToPlainText(sourceHtmlPlainText)); - const nodes = extractTextNodes(blocksMarkup); - const hallucinated: string[] = []; - - for (const node of nodes) { - const needle = normalize(node); - if (!needle) continue; - if (!sourceText.includes(needle)) { - hallucinated.push(node); - } - } - - return { - valid: hallucinated.length === 0, - hallucinated, - }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/pending-imports.test.ts b/packages/data-liberation-agent/src/lib/streaming/pending-imports.test.ts deleted file mode 100644 index 6fc44d856d..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/pending-imports.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdirSync, mkdtempSync, readFileSync, appendFileSync, existsSync } from 'node:fs'; -import { join } from 'node:path'; -import { PendingImportsBuffer } from './pending-imports.js'; -import type { PageItem } from '../wxr/index.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function tmp(): string { - return mkdtempSync(join(FIXTURE_TMP, 'pi-')); -} - -function makePage(overrides: Partial = {}): PageItem { - return { - id: 1, - type: 'page', - title: 'About', - slug: 'about', - content: '

About us

', - excerpt: '', - date: '2026-04-29T12:00:00.000Z', - parent: 0, - menuOrder: 0, - author: 'admin', - seoTitle: '', - seoDescription: '', - sourceUrl: 'https://example.com/about', - ...overrides, - }; -} - -describe('PendingImportsBuffer', () => { - it('writes a header on first enqueue', () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage() }); - const lines = readFileSync(join(dir, 'pending-imports.jsonl'), 'utf8').trim().split('\n'); - expect(lines.length).toBe(2); - const header = JSON.parse(lines[0]); - expect(header.version).toBe(1); - expect(typeof header.createdAt).toBe('string'); - }); - - it('listPending returns nothing when the file does not exist', () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - expect(buf.listPending()).toEqual([]); - expect(existsSync(join(dir, 'pending-imports.jsonl'))).toBe(false); - }); - - it('returns queued URLs in queued-at order', async () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage({ slug: 'a' }) }); - // Wait a millisecond so timestamps differ. - await new Promise((r) => setTimeout(r, 2)); - buf.enqueue({ url: 'https://example.com/b', archetype: 'page', slug: 'b', payload: makePage({ slug: 'b' }) }); - const pending = buf.listPending(); - expect(pending.map((p) => p.url)).toEqual(['https://example.com/a', 'https://example.com/b']); - }); - - it('markImported removes a URL from listPending', () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage() }); - expect(buf.size()).toBe(1); - buf.markImported({ url: 'https://example.com/a', postId: 42, action: 'inserted', composedAs: 'raw-html' }); - expect(buf.size()).toBe(0); - expect(buf.listPending()).toEqual([]); - }); - - it('re-enqueueing after an imported entry makes the URL pending again', () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage({ content: '

v1

' }) }); - buf.markImported({ url: 'https://example.com/a', postId: 1, action: 'inserted', composedAs: 'raw-html' }); - expect(buf.size()).toBe(0); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage({ content: '

v2

' }) }); - const pending = buf.listPending(); - expect(pending).toHaveLength(1); - expect((pending[0].payload as PageItem).content).toBe('

v2

'); - }); - - it('latest queued payload wins when a URL is re-enqueued without imported in between', () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage({ content: '

v1

' }) }); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage({ content: '

v2

' }) }); - const pending = buf.listPending(); - expect(pending).toHaveLength(1); - expect((pending[0].payload as PageItem).content).toBe('

v2

'); - }); - - it('tolerates corrupt / partial lines mid-file', () => { - const dir = tmp(); - const buf = new PendingImportsBuffer(dir); - buf.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage() }); - appendFileSync(join(dir, 'pending-imports.jsonl'), '{"event":"queued","url":"https://e'); - appendFileSync(join(dir, 'pending-imports.jsonl'), '\n'); - buf.enqueue({ url: 'https://example.com/b', archetype: 'page', slug: 'b', payload: makePage({ slug: 'b' }) }); - const pending = buf.listPending(); - expect(pending.map((p) => p.url).sort()).toEqual([ - 'https://example.com/a', - 'https://example.com/b', - ]); - }); - - it('persists queued payloads across new buffer instances (resume)', () => { - const dir = tmp(); - const buf1 = new PendingImportsBuffer(dir); - buf1.enqueue({ url: 'https://example.com/a', archetype: 'page', slug: 'a', payload: makePage() }); - buf1.enqueue({ url: 'https://example.com/b', archetype: 'page', slug: 'b', payload: makePage({ slug: 'b' }) }); - - const buf2 = new PendingImportsBuffer(dir); - expect(buf2.size()).toBe(2); - expect(buf2.listPending().map((p) => p.url).sort()).toEqual([ - 'https://example.com/a', - 'https://example.com/b', - ]); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/pending-imports.ts b/packages/data-liberation-agent/src/lib/streaming/pending-imports.ts deleted file mode 100644 index 388e0839c1..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/pending-imports.ts +++ /dev/null @@ -1,190 +0,0 @@ -// -// Pending-imports JSONL buffer -// ============================ -// Holds extracted URLs that haven't been imported into the running site yet. -// Used by the streaming watch loop: per-URL extraction enqueues a payload, -// then a flush pass — gated on the design foundation being ready (and any -// other agent-side prerequisites) — drains the buffer by calling -// installPost. -// -// File format mirrors block-transform-log.jsonl: -// - First line is a header `{version: 1, createdAt}` written exactly once. -// - Each subsequent line is one entry: `queued` or `imported`. -// - Append-only; never rewrite. Partial / corrupt lines from interrupted -// writes are tolerated by the reader (skipped silently). -// -// "Pending" semantics: a URL is pending iff its most-recent entry is -// `queued`. listPending() walks the log keeping the latest entry per URL and -// returns those whose final state is `queued`. This means re-enqueueing the -// same URL (e.g. on resume) replaces the prior payload, and a re-import -// after `imported` is allowed if a new `queued` entry is appended. -// -// The buffer is intentionally crash-tolerant rather than transactional. A -// crash between "post installed" and "marked imported in log" leaves the URL -// pending; the next flush re-runs installPost, which is idempotent on -// `_source_url`. -// - -import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import type { WxrItem } from '../wxr/index.js'; - -const LOG_FILENAME = 'pending-imports.jsonl'; - -interface HeaderLine { - version: 1; - createdAt: string; -} - -interface QueuedEntry { - event: 'queued'; - url: string; - archetype: string; - /** Slug of the post being queued (for sidecar paths, html / screenshots). */ - slug: string; - payload: WxrItem; - queuedAt: string; -} - -interface ImportedEntry { - event: 'imported'; - url: string; - postId: number | null; - action: 'inserted' | 'updated' | 'error'; - composedAs?: 'blocks' | 'raw-html'; - importedAt: string; -} - -type LogEntry = QueuedEntry | ImportedEntry; - -export interface PendingImport { - url: string; - archetype: string; - slug: string; - payload: WxrItem; - queuedAt: string; -} - -function logPath(outputDir: string): string { - return join(outputDir, LOG_FILENAME); -} - -function ensureHeader(outputDir: string): void { - const path = logPath(outputDir); - if (existsSync(path)) return; - const header: HeaderLine = { - version: 1, - createdAt: new Date().toISOString(), - }; - writeFileSync(path, JSON.stringify(header) + '\n'); -} - -function isHeader(parsed: Record): boolean { - return parsed.version !== undefined && parsed.event === undefined; -} - -/** - * Append-only buffer of URLs awaiting import. Construct once per outputDir; - * the underlying file is opened lazily on first append. - */ -export class PendingImportsBuffer { - constructor(private readonly outputDir: string) {} - - /** - * Queue one item for later import. Multiple enqueues for the same URL are - * allowed — the latest queued entry wins until an `imported` entry follows - * it. - */ - enqueue(opts: { - url: string; - archetype: string; - slug: string; - payload: WxrItem; - }): void { - ensureHeader(this.outputDir); - const entry: QueuedEntry = { - event: 'queued', - url: opts.url, - archetype: opts.archetype, - slug: opts.slug, - payload: opts.payload, - queuedAt: new Date().toISOString(), - }; - appendFileSync(logPath(this.outputDir), JSON.stringify(entry) + '\n'); - } - - /** - * Mark a URL as imported. Subsequent listPending() calls will not return - * it unless a new `queued` entry is appended. - */ - markImported(opts: { - url: string; - postId: number | null; - action: 'inserted' | 'updated' | 'error'; - composedAs?: 'blocks' | 'raw-html'; - }): void { - ensureHeader(this.outputDir); - const entry: ImportedEntry = { - event: 'imported', - url: opts.url, - postId: opts.postId, - action: opts.action, - composedAs: opts.composedAs, - importedAt: new Date().toISOString(), - }; - appendFileSync(logPath(this.outputDir), JSON.stringify(entry) + '\n'); - } - - /** - * Walk the log, keeping the latest entry per URL. Return URLs whose final - * state is `queued`, in queued-at order (oldest first). Corrupt / partial - * lines are skipped silently. - */ - listPending(): PendingImport[] { - const path = logPath(this.outputDir); - if (!existsSync(path)) return []; - - const latest = new Map(); - for (const parsed of readEntries(path)) { - if (typeof parsed.url !== 'string') continue; - // Trust-the-writer: the file is owned by this module, so a - // shape-loose cast is safe. Corrupt lines were already filtered by - // readEntries. - latest.set(parsed.url, parsed as unknown as LogEntry); - } - - const pending: PendingImport[] = []; - for (const entry of latest.values()) { - if (entry.event !== 'queued') continue; - pending.push({ - url: entry.url, - archetype: entry.archetype, - slug: entry.slug, - payload: entry.payload, - queuedAt: entry.queuedAt, - }); - } - pending.sort((a, b) => a.queuedAt.localeCompare(b.queuedAt)); - return pending; - } - - /** Convenience — number of pending URLs. */ - size(): number { - return this.listPending().length; - } -} - -function* readEntries(path: string): Iterable> { - const content = readFileSync(path, 'utf8'); - for (const line of content.split('\n')) { - if (!line.trim()) continue; - let parsed: Record; - try { - parsed = JSON.parse(line) as Record; - } catch { - continue; - } - if (isHeader(parsed)) continue; - yield parsed; - } -} diff --git a/packages/data-liberation-agent/src/lib/streaming/per-url-pipeline.test.ts b/packages/data-liberation-agent/src/lib/streaming/per-url-pipeline.test.ts deleted file mode 100644 index c828666c1e..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/per-url-pipeline.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { processOneUrl } from './per-url-pipeline.js'; -import { ExtractionLog } from '../resume-state/index.js'; -import { WxrBuilder } from '../wxr/index.js'; -import type { ExtractedPage } from '../../adapters/shared.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function makeWxr() { - return new WxrBuilder({ - title: 'Test', - url: 'https://example.com', - description: '', - language: 'en-US', - }); -} - -function makePage(overrides: Partial = {}): ExtractedPage { - return { - title: 'About', - slug: 'about', - content: '

About us

', - excerpt: '', - date: '2026-04-29 12:00:00', - seoTitle: '', - seoDescription: '', - mediaUrls: [], - qualityScore: 'high', - ...overrides, - }; -} - -describe('processOneUrl', () => { - it('extracts a single URL through the runExtractionLoop wrapper', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'pp-')); - try { - const log = new ExtractionLog(dir); - const wxr = makeWxr(); - const extractPage = vi.fn().mockResolvedValue(makePage()); - - const result = await processOneUrl({ - url: 'https://example.com/about', - outputDir: dir, - wxr, - log, - extractPage, - }); - - expect(result.url).toBe('https://example.com/about'); - expect(result.classifyUrl).toBe('page'); - expect(result.extracted).toBe(true); - expect(result.pagesExtracted).toBe(1); - expect(result.failed).toBe(0); - expect(result.errors).toEqual([]); - expect(extractPage).toHaveBeenCalledTimes(1); - expect(extractPage).toHaveBeenCalledWith('https://example.com/about'); - expect(wxr.items).toHaveLength(1); - expect(wxr.items[0].type).toBe('page'); - } finally { - rmSync(dir, { recursive: true, force: true }); - } - }); - - it('records errors when adapter throws and reports failed=1', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'pp-err-')); - try { - const log = new ExtractionLog(dir); - const wxr = makeWxr(); - const extractPage = vi.fn().mockRejectedValue(new Error('boom')); - - const result = await processOneUrl({ - url: 'https://example.com/broken', - outputDir: dir, - wxr, - log, - extractPage, - }); - - expect(result.extracted).toBe(false); - expect(result.failed).toBe(1); - expect(result.pagesExtracted).toBe(0); - } finally { - rmSync(dir, { recursive: true, force: true }); - } - }); - - it('classifies URL types correctly', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'pp-class-')); - try { - const log = new ExtractionLog(dir); - const wxr = makeWxr(); - const extractPage = vi.fn().mockResolvedValue(makePage()); - - const post = await processOneUrl({ - url: 'https://example.com/blog/hello', - outputDir: dir, - wxr, - log, - extractPage, - }); - expect(post.classifyUrl).toBe('post'); - - const product = await processOneUrl({ - url: 'https://example.com/products/foo', - outputDir: dir, - wxr, - log, - extractPage, - }); - expect(product.classifyUrl).toBe('product'); - } finally { - rmSync(dir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/per-url-pipeline.ts b/packages/data-liberation-agent/src/lib/streaming/per-url-pipeline.ts deleted file mode 100644 index ab0cc7225d..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/per-url-pipeline.ts +++ /dev/null @@ -1,158 +0,0 @@ -// -// Per-URL pipeline -// ================ -// Single-URL wrapper around runExtractionLoop. Processes one URL through the -// adapter's extractPage closure, downloads media, appends to the WXR, and -// (optionally) captures screenshots + html. -// -// Used by both: -// - The watch CLI, which calls processOneUrl in a loop -// - The liberate_extract_one MCP tool, for agent-driven streaming -// -// The function defers to runExtractionLoop for the heavy lifting (media -// download, tuner, session updates) by passing a 1-URL inventory + resume:true -// so the existing per-URL logic runs without purging prior state. -// -import { existsSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; -import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import { runExtractionLoop, type ExtractedPage } from '../../adapters/shared.js'; -import { ExtractionLog } from '../resume-state/index.js'; -import { WxrBuilder } from '../wxr/index.js'; -import { ImportSession } from '../resume-state/index.js'; -import { classifyUrl, type UrlType } from '../extraction/sitemap.js'; -import type { WooProduct, WooProductCsvBuilder } from '../woo-csv/index.js'; - -export interface ProcessOneUrlOpts { - /** Absolute URL to extract. Must be a full URL (https://...). */ - url: string; - /** Liberation output directory. */ - outputDir: string; - /** WxrBuilder owned by the caller. processOneUrl appends one item to it. */ - wxr: WxrBuilder; - /** ExtractionLog owned by the caller. */ - log: ExtractionLog; - /** Optional ImportSession for stage + counter updates. */ - session?: ImportSession; - /** Adapter's per-URL extractor closure. */ - extractPage: (url: string) => Promise; - /** Optional platform-specific product extractor. */ - extractProduct?: (url: string, html: string) => WooProduct | null; - /** Optional CSV builder for Woo product output. */ - csvBuilder?: WooProductCsvBuilder; - /** Per-page delay floor (ms). Defaults to 0. */ - delay?: number; - /** Verbose logging during the per-URL run. */ - verbose?: boolean; - /** MCP server for log-message routing. */ - server?: Server; - /** Capture desktop+mobile screenshot + rendered HTML after extraction. Default: false. */ - screenshot?: boolean; -} - -export interface ProcessOneUrlResult { - url: string; - /** Archetype classification at the URL level. */ - classifyUrl: UrlType; - /** True when adapter.extractPage returned a non-null page. */ - extracted: boolean; - /** Counters from the underlying loop (each is 0 or 1). */ - pagesExtracted: number; - postsExtracted: number; - productsExtracted: number; - failed: number; - mediaCollected: number; - /** Wall-clock duration. */ - durationMs: number; - /** Per-URL errors surfaced from extract or screenshot. */ - errors: string[]; - /** Path relative to outputDir for the captured desktop screenshot, when screenshot:true. */ - screenshotPath: string | null; - /** Path relative to outputDir for the captured rendered HTML, when screenshot:true. */ - htmlPath: string | null; -} - -export async function processOneUrl(opts: ProcessOneUrlOpts): Promise { - const start = Date.now(); - const errors: string[] = []; - const archetype = classifyUrl(opts.url); - - // Run the existing extraction loop with an inventory of exactly one URL. - // resume:true skips the fresh-start cleanup that would erase media + log. - let loopResult = { - pagesExtracted: 0, - postsExtracted: 0, - productsExtracted: 0, - failed: 0, - mediaCollected: 0, - }; - try { - loopResult = await runExtractionLoop({ - urls: [{ url: opts.url, type: archetype }], - navigation: [], - wxr: opts.wxr, - log: opts.log, - outputDir: opts.outputDir, - delay: opts.delay ?? 0, - dryRun: false, - resume: true, - verbose: opts.verbose, - server: opts.server, - csvBuilder: opts.csvBuilder, - session: opts.session, - extractPage: opts.extractPage, - extractProduct: opts.extractProduct, - limit: 1, - }); - } catch (err) { - errors.push((err as Error).message); - } - - const extracted = loopResult.pagesExtracted + loopResult.postsExtracted + loopResult.productsExtracted > 0; - - let screenshotPath: string | null = null; - let htmlPath: string | null = null; - if (opts.screenshot && extracted) { - try { - const { captureScreenshots } = await import('../screenshot/screenshotter.js'); - await captureScreenshots({ - urls: [opts.url], - outputDir: opts.outputDir, - primaryUrl: opts.url, - server: opts.server, - }); - // The screenshotter writes manifest.json on the way out — read the - // entry for this URL to surface the captured paths in our result. - const manifestPath = join(opts.outputDir, 'screenshots', 'manifest.json'); - if (existsSync(manifestPath)) { - try { - const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { - entries?: Record; - }; - const entry = manifest.entries?.[opts.url]; - screenshotPath = entry?.desktop ?? null; - htmlPath = entry?.html ?? null; - } catch { - // manifest unreadable — leave paths null - } - } - } catch (err) { - errors.push(`screenshot: ${(err as Error).message}`); - } - } - - return { - url: opts.url, - classifyUrl: archetype, - extracted, - pagesExtracted: loopResult.pagesExtracted, - postsExtracted: loopResult.postsExtracted, - productsExtracted: loopResult.productsExtracted, - failed: loopResult.failed, - mediaCollected: loopResult.mediaCollected, - durationMs: Date.now() - start, - errors, - screenshotPath, - htmlPath, - }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/post-content-media-rewrite.test.ts b/packages/data-liberation-agent/src/lib/streaming/post-content-media-rewrite.test.ts deleted file mode 100644 index aeaf4aba94..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/post-content-media-rewrite.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { prepareInstallContentWithMediaUrls } from './post-content-media-rewrite.js'; - -describe('prepareInstallContentWithMediaUrls', () => { - it('rewrites raw extracted post content when there is no composed override', () => { - const result = prepareInstallContentWithMediaUrls({ - sourceContent: '

', - mediaUrlMap: new Map([ - ['https://cdn.example.com/hero.jpg', 'http://playground.test/wp-content/uploads/hero.jpg'], - ]), - }); - - expect(result.contentOverride).toBe('

'); - expect(result.rewritten).toBe(true); - expect(result.usedSourceContent).toBe(true); - expect(result.missing).toEqual([]); - }); - - it('rewrites an existing composed block override', () => { - const result = prepareInstallContentWithMediaUrls({ - sourceContent: '

Raw source

', - contentOverride: '
', - mediaUrlMap: new Map([ - ['https://cdn.example.com/hero.jpg', 'http://playground.test/wp-content/uploads/hero.jpg'], - ]), - }); - - expect(result.contentOverride).toContain('"url":"http://playground.test/wp-content/uploads/hero.jpg"'); - expect(result.contentOverride).toContain('src="http://playground.test/wp-content/uploads/hero.jpg"'); - expect(result.contentOverride).not.toContain('https://cdn.example.com/hero.jpg'); - expect(result.rewritten).toBe(true); - expect(result.usedSourceContent).toBe(false); - }); - - it('preserves the old no-override behavior when there is no media map', () => { - const result = prepareInstallContentWithMediaUrls({ - sourceContent: '

', - mediaUrlMap: new Map(), - }); - - expect(result.contentOverride).toBeUndefined(); - expect(result.rewritten).toBe(false); - expect(result.usedSourceContent).toBe(false); - expect(result.missing).toEqual([]); - }); - - it('reports unmapped media URLs from the content being installed', () => { - const result = prepareInstallContentWithMediaUrls({ - sourceContent: '', - mediaUrlMap: new Map([ - ['https://cdn.example.com/known.jpg', 'http://playground.test/wp-content/uploads/known.jpg'], - ]), - }); - - expect(result.contentOverride).toBe(''); - expect(result.rewritten).toBe(false); - expect(result.usedSourceContent).toBe(true); - expect(result.missing).toEqual(['https://cdn.example.com/missing.jpg']); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/post-content-media-rewrite.ts b/packages/data-liberation-agent/src/lib/streaming/post-content-media-rewrite.ts deleted file mode 100644 index 7cdcbc85d8..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/post-content-media-rewrite.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { rewriteMediaUrls } from './media-url-rewrite.js'; - -export interface PrepareInstallContentOpts { - /** Raw extracted post/page HTML from the adapter payload. */ - sourceContent: string; - /** Optional block/raw override produced by compose, cache, or heuristics. */ - contentOverride?: string; - /** Source media URL -> local Studio upload URL mapping. */ - mediaUrlMap: Map; -} - -export interface PrepareInstallContentResult { - /** - * Content to pass to installPost. Undefined preserves installPost's native - * fallback to item.content when no rewrite is needed. - */ - contentOverride?: string; - /** True when at least one URL was replaced. */ - rewritten: boolean; - /** True when sourceContent was promoted into contentOverride for rewriting. */ - usedSourceContent: boolean; - /** Source URLs found in the installed content without a local upload URL. */ - missing: string[]; -} - -/** - * Ensure the exact content sent to Studio has local media URLs. - * - * Compose/block paths already pass contentOverride; raw/no-agent paths do not. - * When media mappings exist, promote sourceContent into contentOverride so the - * same rewrite guarantee applies before installPost serializes the payload. - */ -export function prepareInstallContentWithMediaUrls( - opts: PrepareInstallContentOpts, -): PrepareInstallContentResult { - const { sourceContent, contentOverride, mediaUrlMap } = opts; - if (mediaUrlMap.size === 0) { - return { - contentOverride, - rewritten: false, - usedSourceContent: false, - missing: [], - }; - } - - const input = contentOverride ?? sourceContent; - const missing: string[] = []; - const rewrittenContent = rewriteMediaUrls(input, mediaUrlMap, { - onMissing: (url) => missing.push(url), - }); - - return { - contentOverride: rewrittenContent, - rewritten: rewrittenContent !== input, - usedSourceContent: contentOverride === undefined, - missing, - }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/post-existence-poll.test.ts b/packages/data-liberation-agent/src/lib/streaming/post-existence-poll.test.ts deleted file mode 100644 index dbe366ef59..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/post-existence-poll.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { pollForPost, type PollRunner } from './post-existence-poll.js'; - -function makeRunner(responses: Array<{ stdout: string } | Error>): PollRunner & { calls: number } { - let i = 0; - const fn: PollRunner & { calls: number } = (async (_cmd: string, _args: string[]) => { - const r = responses[Math.min(i, responses.length - 1)]; - i++; - fn.calls = i; - if (r instanceof Error) throw r; - return r; - }) as PollRunner & { calls: number }; - fn.calls = 0; - return fn; -} - -describe('pollForPost', () => { - it('returns found:true with postId on first successful attempt', async () => { - const runner = makeRunner([{ stdout: '[123]' }]); - const sleep = vi.fn(); - const result = await pollForPost({ - siteUrl: 'http://localhost:9400', - sourceUrl: 'https://example.com/about', - studioSitePath: '/tmp/site', - runner, - sleep, - }); - expect(result).toEqual({ found: true, postId: 123, attempts: 1 }); - expect(runner.calls).toBe(1); - // No sleep before the first attempt and no sleep after success. - expect(sleep).not.toHaveBeenCalled(); - }); - - it('retries when first attempt returns empty array, succeeds on attempt 2', async () => { - const runner = makeRunner([{ stdout: '[]' }, { stdout: '[42]' }]); - const sleep = vi.fn().mockResolvedValue(undefined); - const result = await pollForPost({ - siteUrl: 'http://localhost:9400', - sourceUrl: 'https://example.com/about', - studioSitePath: '/tmp/site', - runner, - sleep, - backoffMs: [10, 20, 30], - }); - expect(result.found).toBe(true); - expect(result.postId).toBe(42); - expect(result.attempts).toBe(2); - expect(sleep).toHaveBeenCalledTimes(1); - expect(sleep).toHaveBeenNthCalledWith(1, 10); - }); - - it('exhausts all 3 retries and returns found:false', async () => { - const runner = makeRunner([ - { stdout: '[]' }, - { stdout: '[]' }, - { stdout: '[]' }, - ]); - const sleep = vi.fn().mockResolvedValue(undefined); - const result = await pollForPost({ - siteUrl: 'http://localhost:9400', - sourceUrl: 'https://example.com/about', - studioSitePath: '/tmp/site', - runner, - sleep, - backoffMs: [10, 20, 30], - }); - expect(result).toEqual({ found: false, postId: null, attempts: 3 }); - expect(sleep).toHaveBeenCalledTimes(2); - expect(sleep).toHaveBeenNthCalledWith(1, 10); - expect(sleep).toHaveBeenNthCalledWith(2, 20); - }); - - it('uses the documented default 500/2000/5000 backoff when none is overridden', async () => { - const runner = makeRunner([ - { stdout: '[]' }, - { stdout: '[]' }, - { stdout: '[]' }, - ]); - const sleep = vi.fn().mockResolvedValue(undefined); - const result = await pollForPost({ - siteUrl: 'http://localhost:9400', - sourceUrl: 'https://example.com/about', - studioSitePath: '/tmp/site', - runner, - sleep, - }); - expect(result.attempts).toBe(3); - expect(sleep).toHaveBeenNthCalledWith(1, 500); - expect(sleep).toHaveBeenNthCalledWith(2, 2000); - }); - - it('treats runner errors as "not found" for that attempt and continues', async () => { - const runner = makeRunner([new Error('studio not running'), { stdout: '[7]' }]); - const sleep = vi.fn().mockResolvedValue(undefined); - const result = await pollForPost({ - siteUrl: 'http://localhost:9400', - sourceUrl: 'https://example.com/about', - studioSitePath: '/tmp/site', - runner, - sleep, - backoffMs: [1, 1, 1], - }); - expect(result.found).toBe(true); - expect(result.postId).toBe(7); - expect(result.attempts).toBe(2); - }); - - it('parses stdout that has noise before the JSON array', async () => { - const runner = makeRunner([{ stdout: 'Some warning line\n[99]\n' }]); - const result = await pollForPost({ - siteUrl: 'http://localhost:9400', - sourceUrl: 'https://example.com/foo', - studioSitePath: '/tmp/site', - runner, - sleep: vi.fn(), - }); - expect(result.found).toBe(true); - expect(result.postId).toBe(99); - }); - -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/post-existence-poll.ts b/packages/data-liberation-agent/src/lib/streaming/post-existence-poll.ts deleted file mode 100644 index 77a7cbe32a..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/post-existence-poll.ts +++ /dev/null @@ -1,140 +0,0 @@ -// -// Post-existence poller -// ===================== -// Resolves the WordPress post ID for a given source URL by querying the -// running site's `_source_url` postmeta. Used by `liberate_block_transform_apply` -// to avoid the compose-then-apply race: compose can finish before the WXR -// import finishes landing the post, so we poll a small number of times with -// backoff before giving up. -// -// Backoff schedule: 500ms, 2000ms, 5000ms (3 retries). After all 3 attempts -// fail, the apply tool skips with a warning and the page keeps its raw -// `post_content` from WXR import. -// -// Studio path is the v1 target. We invoke `studio wp post list` with a -// `--meta_key=_source_url --meta_value= --field=ID --format=json` -// filter and parse the resulting JSON array. Studio runs WP-CLI inside the -// site VFS, so we get the live post ID even though Studio uses SQLite under -// the hood. -// -// Only Studio is supported. Non-Studio callers receive a not-yet-supported -// error from liberate_block_transform_apply before reaching this path. -// - -import { execFile } from 'node:child_process'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); - -export interface PostExistencePollOpts { - /** Running replica URL e.g. http://localhost:9400. Kept for caller compatibility. */ - siteUrl: string; - /** `_source_url` meta value to match against. */ - sourceUrl: string; - /** Studio site path passed to `studio wp --path `. */ - studioSitePath?: string; - /** - * Override the underlying CLI runner. Tests inject a fake; production - * defaults to spawning `studio wp`. - */ - runner?: PollRunner; - /** Override the backoff schedule (ms). Defaults to [500, 2000, 5000]. */ - backoffMs?: number[]; - /** Sleep function — overridden in tests so we don't wait for real backoff. */ - sleep?: (ms: number) => Promise; -} - -export interface PostExistenceResult { - found: boolean; - postId: number | null; - attempts: number; -} - -/** Inject point for tests — replicates the relevant `execFile` shape. */ -export type PollRunner = ( - command: string, - args: string[], -) => Promise<{ stdout: string }>; - -const DEFAULT_BACKOFF_MS = [500, 2000, 5000]; - -const defaultSleep = (ms: number): Promise => - new Promise((resolve) => setTimeout(resolve, ms)); - -const defaultRunner: PollRunner = async (command, args) => { - const { stdout } = await execFileAsync(command, args, { - timeout: 30_000, - maxBuffer: 10 * 1024 * 1024, - }); - return { stdout }; -}; - -/** Parse the stdout of `wp post list ... --field=ID --format=json` into a number. */ -function parsePostId(stdout: string): number | null { - const trimmed = stdout.trim(); - if (!trimmed) return null; - // wp-cli with --format=json --field=ID returns `[123]` or `[]`. Some - // wrappers prepend status lines before the JSON; find the first `[`. - const start = trimmed.indexOf('['); - if (start < 0) return null; - let parsed: unknown; - try { - parsed = JSON.parse(trimmed.slice(start)); - } catch { - return null; - } - if (!Array.isArray(parsed) || parsed.length === 0) return null; - const first = parsed[0]; - if (typeof first === 'number') return first; - if (typeof first === 'string' && /^\d+$/.test(first)) return Number(first); - return null; -} - -/** - * Poll the running WP for a post matching `_source_url` meta. 3 retries with - * 500ms / 2000ms / 5000ms backoff. Returns `{found, postId, attempts}`. - */ -export async function pollForPost( - opts: PostExistencePollOpts, -): Promise { - const runner = opts.runner ?? defaultRunner; - const sleep = opts.sleep ?? defaultSleep; - const schedule = opts.backoffMs ?? DEFAULT_BACKOFF_MS; - const maxAttempts = schedule.length; - - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - let postId: number | null = null; - try { - if (opts.studioSitePath) { - const args = [ - 'wp', - '--path', - opts.studioSitePath, - 'post', - 'list', - `--meta_key=_source_url`, - `--meta_value=${opts.sourceUrl}`, - '--post_type=any', - '--post_status=any', - '--field=ID', - '--format=json', - ]; - const { stdout } = await runner('studio', args); - postId = parsePostId(stdout); - } - } catch { - // Treat runner failures as "not found this attempt" — try again. - postId = null; - } - - if (postId !== null && postId > 0) { - return { found: true, postId, attempts: attempt }; - } - - if (attempt < maxAttempts) { - await sleep(schedule[attempt - 1]); - } - } - - return { found: false, postId: null, attempts: maxAttempts }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/post-install.test.ts b/packages/data-liberation-agent/src/lib/streaming/post-install.test.ts deleted file mode 100644 index b0645d0bf3..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/post-install.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { mkdtempSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { installPost } from './post-install.js'; -import type { WxrItem, PageItem, PostItem, MediaItem } from '../wxr/index.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function makePage(overrides: Partial = {}): PageItem { - return { - id: 1, - type: 'page', - title: 'About', - slug: 'about', - content: '

About us

', - excerpt: '', - date: '2026-04-29T12:00:00.000Z', - parent: 0, - menuOrder: 0, - author: 'admin', - seoTitle: '', - seoDescription: '', - sourceUrl: 'https://example.com/about', - ...overrides, - }; -} - -describe('installPost', () => { - it('returns null for non-post items (attachment, nav menu, etc.)', async () => { - const attachment: MediaItem = { - id: 1, - type: 'attachment', - title: 'image.png', - slug: 'image-png', - url: 'https://example.com/image.png', - altText: '', - caption: '', - }; - const result = await installPost({ - item: attachment, - outputDir: FIXTURE_TMP, - studioSitePath: '/tmp/site', - }); - expect(result).toBeNull(); - }); - - it('errors when sourceUrl meta is missing', async () => { - const page = makePage({ sourceUrl: '' }); - const result = await installPost({ - item: page, - outputDir: FIXTURE_TMP, - studioSitePath: '/tmp/site', - }); - expect(result?.action).toBe('error'); - expect(result?.error).toMatch(/sourceUrl/); - }); -}); - -// Regression guard for the dla/editable-html "can't edit text" defect: the -// install scripts MUST wp_slash() the post array before wp_insert_post / -// wp_update_post, because those functions wp_unslash() internally. Without it, -// backslashes in block-attribute JSON (e.g. the `frame` attr's \n / - -// escapes) are stripped, invalidating the block in the editor. The bug is -// invisible for plain content (no backslashes), so only a static guard catches -// a silent removal. -describe('install scripts slash post_content before insert/update', () => { - const scriptsDir = join(process.cwd(), 'src', 'lib', 'preview', 'scripts'); - - it('install-post.php wraps wp_insert_post and wp_update_post in wp_slash', () => { - const php = readFileSync(join(scriptsDir, 'install-post.php'), 'utf8'); - expect(php).toMatch(/wp_insert_post\(\s*wp_slash\(/); - expect(php).toMatch(/wp_update_post\(\s*wp_slash\(/); - // and never an un-slashed call to either (the regression we are guarding) - expect(php).not.toMatch(/wp_insert_post\(\s*\$postarr\b/); - expect(php).not.toMatch(/wp_update_post\(\s*\$update\b/); - }); - - it('install-data.php wraps wp_insert_post and wp_update_post in wp_slash', () => { - const php = readFileSync(join(scriptsDir, 'install-data.php'), 'utf8'); - expect(php).toMatch(/wp_insert_post\(\s*wp_slash\(/); - expect(php).toMatch(/wp_update_post\(\s*wp_slash\(/); - }); - - it('install-data.php wp_slashes item-derived update_post_meta values', () => { - // update_post_meta() also wp_unslash()es internally, so backslash-bearing - // meta values (e.g. JSON/escaped text in custom meta) must be slashed too. - const php = readFileSync(join(scriptsDir, 'install-data.php'), 'utf8'); - expect(php).toMatch(/update_post_meta\(\s*\$post_id,\s*\$key,\s*wp_slash\(/); - expect(php).toMatch(/update_post_meta\([^)]*'_dla_item_id',\s*wp_slash\(/); - expect(php).toMatch(/update_post_meta\([^)]*'_dla_gallery',\s*wp_slash\(/); - // the regression we are guarding: never the un-slashed custom-meta write - expect(php).not.toMatch(/update_post_meta\(\s*\$post_id,\s*\$key,\s*\$meta\[/); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/post-install.ts b/packages/data-liberation-agent/src/lib/streaming/post-install.ts deleted file mode 100644 index 3728a35b26..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/post-install.ts +++ /dev/null @@ -1,138 +0,0 @@ -// -// Per-URL post install -// ==================== -// Inserts one extracted WXR item (page/post/product) into a running Studio -// site via `studio wp eval-file install-post.php `. Idempotent — -// install-post.php looks up by `_source_url` meta first. -// -// Used by the watch loop's per-URL incremental flow: as each URL is -// extracted, the resulting WxrItem is handed to installPost so the running -// site receives content URL by URL rather than via a final batch import. -// -import { execFile } from 'node:child_process'; -import { promisify } from 'node:util'; -import { mkdirSync, writeFileSync } from 'node:fs'; -import { join, resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import type { WxrItem, PageItem, PostItem } from '../wxr/index.js'; - -const execFileAsync = promisify(execFile); - -/** Vendored PHP file that performs the wp_insert_post call. */ -const INSTALL_POST_SCRIPT_HOST = resolve( - dirname(fileURLToPath(import.meta.url)), - '..', - 'preview', - 'scripts', - 'install-post.php', -); - -const SCRIPTS_SUBDIR = '.dla-scripts'; -const SCRIPTS_VFS_PREFIX = '/wordpress'; - -export interface InstallPostOpts { - /** Single WxrItem to install. Only `page` / `post` are supported by v1. */ - item: WxrItem; - /** Liberation outputDir (anchor for vendored script + JSON payload). */ - outputDir: string; - /** Studio site path on host (e.g. ~/Studio/example-com). */ - studioSitePath: string; - /** Optional content override — used after media-url-rewrite swaps source URLs for local upload URLs. */ - contentOverride?: string; -} - -export interface InstallPostResult { - sourceUrl: string; - postId: number | null; - action: 'inserted' | 'updated' | 'error'; - error?: string; -} - -/** - * Install one post into the running Studio site. Returns null when the item - * isn't a supported post type (attachment, nav menu items, terms — these - * have their own install paths). - */ -export async function installPost(opts: InstallPostOpts): Promise { - const { item, outputDir, studioSitePath, contentOverride } = opts; - if (item.type !== 'page' && item.type !== 'post') { - return null; - } - const post = item as PageItem | PostItem; - - if (!post.sourceUrl) { - return { - sourceUrl: '', - postId: null, - action: 'error', - error: 'missing sourceUrl meta — install requires _source_url for idempotency', - }; - } - - // Stage the script + JSON payload under /.dla-scripts/. - // Studio's wp-cli rejects host paths, so payloads must live inside the - // mounted site dir. - const scriptsDir = join(studioSitePath, SCRIPTS_SUBDIR); - mkdirSync(scriptsDir, { recursive: true }); - const scriptVfs = `${SCRIPTS_VFS_PREFIX}/${SCRIPTS_SUBDIR}/install-post.php`; - const payloadHost = join(scriptsDir, `install-post-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`); - const payloadVfs = `${SCRIPTS_VFS_PREFIX}/${SCRIPTS_SUBDIR}/${payloadHost.split('/').pop()}`; - - // Copy script in (overwrite-safe; matches studio.ts pattern). - const { copyFileSync } = await import('node:fs'); - const scriptHost = join(scriptsDir, 'install-post.php'); - copyFileSync(INSTALL_POST_SCRIPT_HOST, scriptHost); - - const payload = { - _source_url: post.sourceUrl, - post_type: post.type, - title: post.title, - slug: post.slug, - content: contentOverride ?? post.content, - excerpt: post.excerpt ?? '', - date: post.date ?? '', - post_status: 'publish', - meta: { - ...(post.seoTitle ? { _seo_title: post.seoTitle } : {}), - ...(post.seoDescription ? { _seo_description: post.seoDescription } : {}), - }, - }; - writeFileSync(payloadHost, JSON.stringify(payload), 'utf8'); - - try { - const { stdout } = await execFileAsync( - 'studio', - ['wp', '--path', studioSitePath, 'eval-file', scriptVfs, payloadVfs], - { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 }, - ); - const trimmed = stdout.trim(); - // Studio's wp-cli wrapper sometimes prefixes lines; pull the JSON object out. - const jsonStart = trimmed.indexOf('{'); - const jsonEnd = trimmed.lastIndexOf('}'); - if (jsonStart < 0 || jsonEnd < jsonStart) { - return { sourceUrl: post.sourceUrl, postId: null, action: 'error', error: `unexpected stdout: ${trimmed.slice(0, 200)}` }; - } - const parsed = JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)) as { post_id: number | null; action: string; error?: string }; - return { - sourceUrl: post.sourceUrl, - postId: parsed.post_id ?? null, - action: parsed.action === 'inserted' ? 'inserted' : parsed.action === 'updated' ? 'updated' : 'error', - error: parsed.error, - }; - } catch (err) { - // execFileAsync errors carry stderr/stdout on the Error object — the - // default `.message` is just "Command failed: " with no PHP - // detail. Pull stderr (and stdout when present) into the surfaced - // error so the watch.log shows what install-post.php actually said. - const e = err as Error & { stderr?: string; stdout?: string; code?: number }; - const parts: string[] = [e.message]; - if (e.stderr && e.stderr.trim()) parts.push(`stderr: ${e.stderr.trim().slice(-1000)}`); - if (e.stdout && e.stdout.trim()) parts.push(`stdout: ${e.stdout.trim().slice(-1000)}`); - return { - sourceUrl: post.sourceUrl, - postId: null, - action: 'error', - error: parts.join(' | '), - }; - } -} diff --git a/packages/data-liberation-agent/src/lib/streaming/replicate-state-cache.test.ts b/packages/data-liberation-agent/src/lib/streaming/replicate-state-cache.test.ts deleted file mode 100644 index 575147cd51..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/replicate-state-cache.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { mkdirSync, mkdtempSync } from 'node:fs'; -import { join } from 'node:path'; -import { ReplicateStateCache } from './replicate-state-cache.js'; -import * as state from './replicate-state.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function tmp(): string { - return mkdtempSync(join(FIXTURE_TMP, 'rsc-')); -} - -describe('ReplicateStateCache', () => { - it('reads from disk only on first access', () => { - const dir = tmp(); - const spy = vi.spyOn(state, 'loadReplicateState'); - const cache = new ReplicateStateCache(dir); - cache.get(); - cache.get(); - cache.get(); - expect(spy).toHaveBeenCalledTimes(1); - spy.mockRestore(); - }); - - it('persists changes through update() and returns the new state', () => { - const dir = tmp(); - const cache = new ReplicateStateCache(dir); - const result = cache.update((s) => ({ ...s, urlsSeen: 7 })); - expect(result.urlsSeen).toBe(7); - - // Reload from disk via a fresh cache to confirm the write happened. - const fresh = new ReplicateStateCache(dir); - expect(fresh.get().urlsSeen).toBe(7); - }); - - it('update() updates the in-memory cache (no extra disk read)', () => { - const dir = tmp(); - const cache = new ReplicateStateCache(dir); - cache.update((s) => ({ ...s, urlsSeen: 1 })); - - const spy = vi.spyOn(state, 'loadReplicateState'); - expect(cache.get().urlsSeen).toBe(1); - expect(spy).not.toHaveBeenCalled(); - spy.mockRestore(); - }); - - it('reload() forces a fresh read from disk', () => { - const dir = tmp(); - const cache = new ReplicateStateCache(dir); - cache.get(); - - // Simulate an out-of-band write - state.saveReplicateState(dir, { ...state.emptyState(), urlsSeen: 42 }); - expect(cache.get().urlsSeen).toBe(0); // still cached - - const reloaded = cache.reload(); - expect(reloaded.urlsSeen).toBe(42); - expect(cache.get().urlsSeen).toBe(42); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/replicate-state-cache.ts b/packages/data-liberation-agent/src/lib/streaming/replicate-state-cache.ts deleted file mode 100644 index be4d997fbe..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/replicate-state-cache.ts +++ /dev/null @@ -1,61 +0,0 @@ -// -// Replicate state cache -// ===================== -// In-memory cache wrapper around `replicate-state.json`. The streaming engine -// `observe()`s on every URL, but loading the JSON file each time is wasteful; -// this cache reads once on first access, mutates in-memory, and writes back -// only when `update()` is called. -// -// Single-process semantics: the cache assumes the calling tick-scheduler is -// the sole writer. Multi-process callers must coordinate via the streaming -// lockfile and call `reload()` after acquiring the lock. -// -import { - loadReplicateState, - saveReplicateState, - type ReplicateState, -} from './replicate-state.js'; - -export class ReplicateStateCache { - private readonly outputDir: string; - private cached: ReplicateState | null = null; - - constructor(outputDir: string) { - this.outputDir = outputDir; - } - - /** - * Read the state. First call reads from disk; subsequent calls return the - * cached value without I/O. The returned object is the cache's internal - * reference — callers should treat it as read-only and use `update()` to - * mutate. - */ - get(): ReplicateState { - if (this.cached === null) { - this.cached = loadReplicateState(this.outputDir); - } - return this.cached; - } - - /** - * Apply a transform to the state and persist it. The transform should - * return a new (or mutated) state object; the cache writes whichever value - * the transform returns and uses that as its new cached value. - */ - update(fn: (state: ReplicateState) => ReplicateState): ReplicateState { - const current = this.get(); - const next = fn(current); - this.cached = next; - saveReplicateState(this.outputDir, next); - return next; - } - - /** - * Force a reread from disk (e.g. after releasing a lock that another writer - * may have held). Drops the in-memory copy and reloads on next access. - */ - reload(): ReplicateState { - this.cached = loadReplicateState(this.outputDir); - return this.cached; - } -} diff --git a/packages/data-liberation-agent/src/lib/streaming/replicate-state.test.ts b/packages/data-liberation-agent/src/lib/streaming/replicate-state.test.ts deleted file mode 100644 index 5b438758ad..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/replicate-state.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { - computeThemeFilesDigest, - emptyState, - loadReplicateState, - saveReplicateState, - type ReplicateState, -} from './replicate-state.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function tmp(): string { - return mkdtempSync(join(FIXTURE_TMP, 'rs-')); -} - -describe('replicate-state', () => { - it('returns an empty state when the file does not exist', () => { - const dir = tmp(); - const state = loadReplicateState(dir); - expect(state.version).toBe(1); - expect(state.urlsSeen).toBe(0); - expect(state.archetypesObserved).toEqual([]); - expect(state.archetypeTemplateMap).toEqual({}); - expect(state.lastThemeFilesDigest).toBe(''); - expect(state.lastFoundationInputsDigest).toBe(''); - expect(state.lastTickAt).toBeNull(); - expect(state.lastTickReason).toBeNull(); - }); - - it('round-trips through save + load', () => { - const dir = tmp(); - const state: ReplicateState = { - ...emptyState(), - urlsSeen: 3, - archetypesObserved: ['homepage', 'page'], - archetypeTemplateMap: { page: ['templates/page.html'] }, - lastFoundationInputsDigest: 'sha256:abc', - lastTickAt: '2026-04-29T12:00:00.000Z', - lastTickReason: 'periodic', - }; - saveReplicateState(dir, state); - const loaded = loadReplicateState(dir); - expect(loaded).toEqual(state); - }); - - it('writes atomically (no .tmp left after save)', () => { - const dir = tmp(); - saveReplicateState(dir, { ...emptyState(), urlsSeen: 1 }); - const files = readdirSync(dir); - expect(files).toContain('replicate-state.json'); - expect(files.find((f) => f.endsWith('.tmp'))).toBeUndefined(); - }); - - it('quarantines a corrupt file as .corrupt. and returns empty state', () => { - const dir = tmp(); - writeFileSync(join(dir, 'replicate-state.json'), '{not valid json'); - const state = loadReplicateState(dir); - expect(state).toEqual(emptyState()); - const files = readdirSync(dir); - const corrupt = files.find((f) => f.startsWith('replicate-state.json.corrupt.')); - expect(corrupt).toBeDefined(); - }); - - it('quarantines a wrong-version file', () => { - const dir = tmp(); - writeFileSync( - join(dir, 'replicate-state.json'), - JSON.stringify({ version: 999, urlsSeen: 0 }), - ); - const state = loadReplicateState(dir); - expect(state).toEqual(emptyState()); - const files = readdirSync(dir); - expect(files.some((f) => f.startsWith('replicate-state.json.corrupt.'))).toBe(true); - }); - - it('quarantines a structurally-invalid file (wrong field types)', () => { - const dir = tmp(); - writeFileSync( - join(dir, 'replicate-state.json'), - JSON.stringify({ version: 1, urlsSeen: 'three', archetypesObserved: [], archetypeTemplateMap: {}, lastThemeFilesDigest: '', lastFoundationInputsDigest: '', lastTickAt: null, lastTickReason: null }), - ); - const state = loadReplicateState(dir); - expect(state).toEqual(emptyState()); - }); - - it('creates the parent directory on save', () => { - const dir = tmp(); - const nested = join(dir, 'nested', 'subdir'); - saveReplicateState(nested, emptyState()); - expect(existsSync(join(nested, 'replicate-state.json'))).toBe(true); - }); - - it('preserves an empty file as corrupt and returns empty state', () => { - const dir = tmp(); - writeFileSync(join(dir, 'replicate-state.json'), ''); - const state = loadReplicateState(dir); - expect(state).toEqual(emptyState()); - }); -}); - -describe('computeThemeFilesDigest', () => { - it('produces a stable sha256 digest', () => { - const digest = computeThemeFilesDigest([ - { relativePath: 'templates/page.html', content: '' }, - ]); - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }); - - it('is order-independent (sorts by relativePath)', () => { - const a = computeThemeFilesDigest([ - { relativePath: 'a.html', content: 'A' }, - { relativePath: 'b.html', content: 'B' }, - ]); - const b = computeThemeFilesDigest([ - { relativePath: 'b.html', content: 'B' }, - { relativePath: 'a.html', content: 'A' }, - ]); - expect(a).toBe(b); - }); - - it('changes when a file content changes', () => { - const a = computeThemeFilesDigest([{ relativePath: 'a.html', content: 'A' }]); - const b = computeThemeFilesDigest([{ relativePath: 'a.html', content: 'A2' }]); - expect(a).not.toBe(b); - }); - - it('changes when a new file is added', () => { - const a = computeThemeFilesDigest([{ relativePath: 'a.html', content: 'A' }]); - const b = computeThemeFilesDigest([ - { relativePath: 'a.html', content: 'A' }, - { relativePath: 'b.html', content: 'B' }, - ]); - expect(a).not.toBe(b); - }); - - it('returns a deterministic empty digest for an empty list', () => { - expect(computeThemeFilesDigest([])).toMatch(/^sha256:[a-f0-9]{64}$/); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/replicate-state.ts b/packages/data-liberation-agent/src/lib/streaming/replicate-state.ts deleted file mode 100644 index 8e89197473..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/replicate-state.ts +++ /dev/null @@ -1,161 +0,0 @@ -// -// Replicate state file -// ==================== -// Per-outputDir JSON state for the streaming replicate loop. Tracks how many -// URLs the streaming pipeline has observed, which archetypes have been seen -// (and which template/pattern files were applied to each), the digests of the -// last theme files + foundation inputs that were applied, and metadata about -// the last tick. -// -// File path: `/replicate-state.json` -// -// Persistence model: -// - Single-writer atomic rename (write `.tmp` then rename over the target). -// - Corrupt files are renamed to `replicate-state.json.corrupt.` rather -// than silently dropped — mirrors `ImportSession`. -// - `version: 1` is the current contract; consumers must pin. -// -// Lock semantics: callers are responsible for serializing writes. The streaming -// engine's lockfile (in `src/lib/preview/lockfile.ts`) is used for the full -// tick scope; this module does not acquire locks itself. -// -import { createHash } from 'node:crypto'; -import { - existsSync, - mkdirSync, - readFileSync, - renameSync, - unlinkSync, - writeFileSync, -} from 'node:fs'; -import { dirname, join } from 'node:path'; - -export interface ReplicateState { - version: 1; - /** Number of URLs the streaming pipeline has observed (monotonic). */ - urlsSeen: number; - /** Sorted unique list of archetypes (e.g. ['homepage', 'page', 'product']). */ - archetypesObserved: string[]; - /** - * Map archetype → list of theme files that were applied for that archetype - * (e.g. `{'product': ['templates/single-product.html', 'patterns/product-card.php']}`). - */ - archetypeTemplateMap: Partial>; - /** sha256 digest over the canonical JSON of the last applied theme files. */ - lastThemeFilesDigest: string; - /** Last applied design-foundation inputsDigest (palette+typography+breakpoints). */ - lastFoundationInputsDigest: string; - /** ISO timestamp of the last tick that ran (or null if none yet). */ - lastTickAt: string | null; - /** Reason of the last tick (e.g. 'periodic', 'new-archetype'). */ - lastTickReason: string | null; -} - -const STATE_FILENAME = 'replicate-state.json'; - -/** - * Construct an empty `version: 1` state. Used when no state file exists or the - * existing file is corrupt / wrong-version. - */ -export function emptyState(): ReplicateState { - return { - version: 1, - urlsSeen: 0, - archetypesObserved: [], - archetypeTemplateMap: {}, - lastThemeFilesDigest: '', - lastFoundationInputsDigest: '', - lastTickAt: null, - lastTickReason: null, - }; -} - -function statePath(outputDir: string): string { - return join(outputDir, STATE_FILENAME); -} - -/** - * Read the replicate state from disk. Missing → empty state. Corrupt or - * wrong-version → preserve original as `.corrupt.` and return empty. - */ -export function loadReplicateState(outputDir: string): ReplicateState { - const p = statePath(outputDir); - if (!existsSync(p)) return emptyState(); - - let raw: string; - try { - raw = readFileSync(p, 'utf8'); - } catch { - return emptyState(); - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - quarantine(p); - return emptyState(); - } - - if (!isReplicateState(parsed)) { - quarantine(p); - return emptyState(); - } - return parsed; -} - -/** - * Write the replicate state atomically. Caller owns serialization (lock). - */ -export function saveReplicateState(outputDir: string, state: ReplicateState): void { - const p = statePath(outputDir); - mkdirSync(dirname(p), { recursive: true }); - const tmp = p + '.tmp'; - writeFileSync(tmp, JSON.stringify(state, null, 2)); - renameSync(tmp, p); -} - -/** - * Compute sha256 over the canonical JSON of theme files. Files are sorted by - * `relativePath` so the digest is stable across reorderings. - */ -export function computeThemeFilesDigest( - files: Array<{ relativePath: string; content: string }>, -): string { - const sorted = [...files].sort((a, b) => a.relativePath.localeCompare(b.relativePath)); - const canonical = JSON.stringify( - sorted.map((f) => ({ relativePath: f.relativePath, content: f.content })), - ); - return 'sha256:' + createHash('sha256').update(canonical).digest('hex'); -} - -// --------------------------------------------------------------------------- -// Internals -// --------------------------------------------------------------------------- - -function quarantine(p: string): void { - // Preserve the corrupt file as .corrupt.; if rename fails, drop it as a - // last resort rather than leaving an unparseable file in place that future - // loads will keep failing on. - try { - const backup = `${p}.corrupt.${Date.now()}`; - renameSync(p, backup); - } catch { - try { unlinkSync(p); } catch { /* ignore */ } - } -} - -function isReplicateState(value: unknown): value is ReplicateState { - if (typeof value !== 'object' || value === null) return false; - const v = value as Record; - if (v.version !== 1) return false; - if (typeof v.urlsSeen !== 'number') return false; - if (!Array.isArray(v.archetypesObserved)) return false; - if (!v.archetypesObserved.every((x) => typeof x === 'string')) return false; - if (typeof v.archetypeTemplateMap !== 'object' || v.archetypeTemplateMap === null) return false; - if (typeof v.lastThemeFilesDigest !== 'string') return false; - if (typeof v.lastFoundationInputsDigest !== 'string') return false; - if (v.lastTickAt !== null && typeof v.lastTickAt !== 'string') return false; - if (v.lastTickReason !== null && typeof v.lastTickReason !== 'string') return false; - return true; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/site-finalize.test.ts b/packages/data-liberation-agent/src/lib/streaming/site-finalize.test.ts deleted file mode 100644 index 74842cd3e2..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/site-finalize.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -// src/lib/streaming/site-finalize.test.ts -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { mkdtempSync, mkdirSync, readFileSync, readdirSync, existsSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; - -// Mock the exec seam (same convention as convert-local-site.test.ts) so the -// wiring test can capture the eval-file argv + stage files without a real -// `studio` binary. Script copy + payload write stay REAL fs operations. -const execCalls: Array<{ cmd: string; args: string[] }> = []; -let execBehavior: (args: string[]) => { stdout: string } = () => ({ - stdout: '{"ok":true,"applied":{"options":[],"templates":[],"frontPage":false},"errors":[]}', -}); -vi.mock('node:child_process', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - execFile: vi.fn((cmd: string, args: string[], _opts: unknown, cb: (e: Error | null, r: { stdout: string; stderr: string }) => void) => { - execCalls.push({ cmd, args }); - try { - cb(null, { stdout: execBehavior(args).stdout, stderr: '' }); - } catch (e) { - cb(e as Error, { stdout: '', stderr: '' }); - } - }), - }; -}); - -import { finalizeSite, parseFinalizeStdout, type SiteFinalizePayload } from './site-finalize.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -beforeEach(() => { - execCalls.length = 0; - execBehavior = () => ({ - stdout: '{"ok":true,"applied":{"options":[],"templates":[],"frontPage":false},"errors":[]}', - }); -}); - -describe('parseFinalizeStdout', () => { - it('extracts the result JSON from prefixed wp-cli stdout', () => { - const res = parseFinalizeStdout( - 'Studio banner line\n{"ok":false,"applied":{"options":["blogname"],"templates":[12],"frontPage":true},"errors":[{"item":"template:about","error":"boom"}]}\n', - ); - expect(res.ok).toBe(false); - expect(res.applied).toEqual({ options: ['blogname'], templates: [12], frontPage: true }); - expect(res.errors).toEqual([{ item: 'template:about', error: 'boom' }]); - }); - - it('throws on stdout with no JSON object (whole-call failure)', () => { - expect(() => parseFinalizeStdout('Error: something exploded')).toThrow(/unexpected stdout/); - }); - - it('normalizes missing fields defensively', () => { - const res = parseFinalizeStdout('{"ok":true}'); - expect(res).toEqual({ ok: true, applied: { options: [], templates: [], frontPage: false }, errors: [] }); - }); -}); - -describe('finalizeSite', () => { - it('short-circuits an empty payload without staging files or shelling out', async () => { - const sitePath = mkdtempSync(join(FIXTURE_TMP, 'sf-empty-')); - try { - const res = await finalizeSite({ - payload: { options: {}, templateAssigns: [] }, - studioSitePath: sitePath, - }); - expect(res).toEqual({ ok: true, applied: { options: [], templates: [], frontPage: false }, errors: [] }); - expect(execCalls).toHaveLength(0); - expect(existsSync(join(sitePath, '.dla-scripts'))).toBe(false); - } finally { - rmSync(sitePath, { recursive: true, force: true }); - } - }); - - it('stages script + JSON payload into .dla-scripts and invokes eval-file with VFS paths', async () => { - const sitePath = mkdtempSync(join(FIXTURE_TMP, 'sf-wire-')); - try { - const payload: SiteFinalizePayload = { - options: { blogname: 'Acme' }, - templateAssigns: [{ postId: 12, slug: 'about', template: 'page-local' }], - frontPageId: 11, - }; - const res = await finalizeSite({ payload, studioSitePath: sitePath }); - expect(res.ok).toBe(true); - // One exec: studio wp --path eval-file . - expect(execCalls).toHaveLength(1); - const { cmd, args } = execCalls[0]; - expect(cmd).toBe('studio'); - expect(args.slice(0, 4)).toEqual(['wp', '--path', sitePath, 'eval-file']); - expect(args[4]).toBe('/wordpress/.dla-scripts/site-finalize.php'); - expect(args[5]).toMatch(/^\/wordpress\/\.dla-scripts\/site-finalize-.*\.json$/); - // Script copied into the site dir; payload file round-trips the input. - const scriptsDir = join(sitePath, '.dla-scripts'); - expect(existsSync(join(scriptsDir, 'site-finalize.php'))).toBe(true); - const payloadFile = readdirSync(scriptsDir).find((f) => f.startsWith('site-finalize-') && f.endsWith('.json')); - expect(payloadFile).toBeDefined(); - expect(JSON.parse(readFileSync(join(scriptsDir, payloadFile as string), 'utf8'))).toEqual(payload); - } finally { - rmSync(sitePath, { recursive: true, force: true }); - } - }); - - it('rejects on exec failure with stderr surfaced (whole-call failure)', async () => { - const sitePath = mkdtempSync(join(FIXTURE_TMP, 'sf-fail-')); - execBehavior = () => { - const e = new Error('Command failed: studio wp') as Error & { stderr?: string }; - e.stderr = 'PHP Fatal error: nope'; - throw e; - }; - try { - await expect( - finalizeSite({ - payload: { options: { blogname: 'Acme' }, templateAssigns: [] }, - studioSitePath: sitePath, - }), - ).rejects.toThrow(/Command failed.*stderr: PHP Fatal error: nope/s); - } finally { - rmSync(sitePath, { recursive: true, force: true }); - } - }); - - it('rejects on garbage stdout (parse failure is a whole-call failure)', async () => { - const sitePath = mkdtempSync(join(FIXTURE_TMP, 'sf-garbage-')); - execBehavior = () => ({ stdout: 'not json at all' }); - try { - await expect( - finalizeSite({ - payload: { options: { blogname: 'Acme' }, templateAssigns: [] }, - studioSitePath: sitePath, - }), - ).rejects.toThrow(/unexpected stdout/); - } finally { - rmSync(sitePath, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/site-finalize.ts b/packages/data-liberation-agent/src/lib/streaming/site-finalize.ts deleted file mode 100644 index 1272348b84..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/site-finalize.ts +++ /dev/null @@ -1,139 +0,0 @@ -// -// Site finalize (one-shot) -// ======================== -// Applies the post-install site finalization writes — option updates -// (blogname etc.), per-page _wp_page_template assigns, and the static -// front-page pair — in ONE `studio wp eval-file site-finalize.php ` -// call, via the same VFS bridging install-post.ts uses. -// -// Why one call: Studio's IPC layer flakes on bursts of individual argv -// commands ("Timeout waiting for response to message wp-cli-command: No -// activity for 120s") while eval-file invocations succeed reliably — one -// IPC slot, values shipped via JSON file rather than argv. On a fresh site -// a dropped blogname (the site-title block renders the wrong brand) or a -// dropped _wp_page_template assign (wrong template) is a structural parity -// failure the repair loop cannot fix, so these writes ride the reliable -// channel. -// -import { execFile } from 'node:child_process'; -import { promisify } from 'node:util'; -import { mkdirSync, writeFileSync, copyFileSync } from 'node:fs'; -import { join, resolve, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const execFileAsync = promisify(execFile); - -/** Vendored PHP file that performs the option/meta writes. */ -const SITE_FINALIZE_SCRIPT_HOST = resolve( - dirname(fileURLToPath(import.meta.url)), - '..', - 'preview', - 'scripts', - 'site-finalize.php', -); - -const SCRIPTS_SUBDIR = '.dla-scripts'; -const SCRIPTS_VFS_PREFIX = '/wordpress'; - -export interface SiteFinalizePayload { - /** wp option updates to apply (blogname etc.). */ - options: Record; - /** _wp_page_template assigns. `slug` rides along for warning text only — - * the PHP keys its error items as `template:`. */ - templateAssigns: Array<{ postId: number; slug: string; template: string }>; - /** When set: show_on_front=page + page_on_front= (applied as a pair). */ - frontPageId?: number; -} - -export interface SiteFinalizeResult { - /** False when any item errored (per-item granularity in `errors`). */ - ok: boolean; - applied: { options: string[]; templates: number[]; frontPage: boolean }; - errors: Array<{ item: string; error: string }>; -} - -export interface FinalizeSiteOpts { - payload: SiteFinalizePayload; - /** Studio site path on host (e.g. ~/Studio/example-com). */ - studioSitePath: string; -} - -/** Pure: pull the result JSON out of (possibly prefixed) wp-cli stdout. - * Studio's wp-cli wrapper sometimes prefixes lines — same extraction - * install-post.ts uses. Throws on stdout with no JSON object (the caller - * treats that as a whole-call failure). */ -export function parseFinalizeStdout(stdout: string): SiteFinalizeResult { - const trimmed = stdout.trim(); - const jsonStart = trimmed.indexOf('{'); - const jsonEnd = trimmed.lastIndexOf('}'); - if (jsonStart < 0 || jsonEnd < jsonStart) { - throw new Error(`unexpected stdout: ${trimmed.slice(0, 200)}`); - } - const parsed = JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)) as { - ok?: unknown; - applied?: { options?: string[]; templates?: number[]; frontPage?: unknown }; - errors?: Array<{ item: string; error: string }>; - }; - return { - ok: parsed.ok === true, - applied: { - options: parsed.applied?.options ?? [], - templates: parsed.applied?.templates ?? [], - frontPage: parsed.applied?.frontPage === true, - }, - errors: parsed.errors ?? [], - }; -} - -/** - * Apply the finalize payload to the running Studio site in one eval-file - * round-trip. Per-item failures come back in `result.errors` (the call - * still resolves); transport-level failures — exec error, timeout, garbage - * stdout — REJECT, so callers can map them to a single whole-call warning. - */ -export async function finalizeSite(opts: FinalizeSiteOpts): Promise { - const { payload, studioSitePath } = opts; - - // Nothing to apply → succeed without an IPC round-trip (and without - // staging any files into the site dir). - if ( - Object.keys(payload.options).length === 0 && - payload.templateAssigns.length === 0 && - payload.frontPageId === undefined - ) { - return { ok: true, applied: { options: [], templates: [], frontPage: false }, errors: [] }; - } - - // Stage the script + JSON payload under /.dla-scripts/. - // Studio's wp-cli rejects host paths, so payloads must live inside the - // mounted site dir (same bridging install-post.ts uses). - const scriptsDir = join(studioSitePath, SCRIPTS_SUBDIR); - mkdirSync(scriptsDir, { recursive: true }); - const scriptVfs = `${SCRIPTS_VFS_PREFIX}/${SCRIPTS_SUBDIR}/site-finalize.php`; - const payloadHost = join(scriptsDir, `site-finalize-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`); - const payloadVfs = `${SCRIPTS_VFS_PREFIX}/${SCRIPTS_SUBDIR}/${payloadHost.split('/').pop()}`; - - // Copy script in (overwrite-safe; matches install-post.ts pattern). - const scriptHost = join(scriptsDir, 'site-finalize.php'); - copyFileSync(SITE_FINALIZE_SCRIPT_HOST, scriptHost); - writeFileSync(payloadHost, JSON.stringify(payload), 'utf8'); - - try { - const { stdout } = await execFileAsync( - 'studio', - ['wp', '--path', studioSitePath, 'eval-file', scriptVfs, payloadVfs], - { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 }, - ); - return parseFinalizeStdout(stdout); - } catch (err) { - // execFileAsync errors carry stderr/stdout on the Error object — the - // default `.message` is just "Command failed: " with no PHP - // detail. Pull stderr (and stdout when present) into the surfaced - // error so the caller's warning shows what site-finalize.php said. - const e = err as Error & { stderr?: string; stdout?: string }; - const parts: string[] = [e.message]; - if (e.stderr && e.stderr.trim()) parts.push(`stderr: ${e.stderr.trim().slice(-1000)}`); - if (e.stdout && e.stdout.trim()) parts.push(`stdout: ${e.stdout.trim().slice(-1000)}`); - throw new Error(parts.join(' | ')); - } -} diff --git a/packages/data-liberation-agent/src/lib/streaming/tick-scheduler.test.ts b/packages/data-liberation-agent/src/lib/streaming/tick-scheduler.test.ts deleted file mode 100644 index 837fc4b037..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/tick-scheduler.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { createTickScheduler } from './tick-scheduler.js'; -import { loadReplicateState } from './replicate-state.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function tmp(): string { - return mkdtempSync(join(FIXTURE_TMP, 'ts-')); -} - -/** - * Seed a stub design-foundation.json so the scheduler treats the foundation - * as ready. Tests that don't seed this file exercise the deferred-archetype - * path. - */ -function seedFoundation(dir: string): void { - writeFileSync(join(dir, 'design-foundation.json'), '{}', 'utf8'); -} - -describe('createTickScheduler', () => { - it('emits a new-archetype judgment the first time an archetype is observed', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/p/foo', 'product'); - const judgments = await scheduler.drain(); - expect(judgments).toHaveLength(1); - expect(judgments[0].kind).toBe('archetype-template'); - expect(judgments[0].archetype).toBe('product'); - }); - - it('does not enqueue duplicate new-archetype ticks for the same archetype', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/p/a', 'product'); - scheduler.observe('https://example.com/p/b', 'product'); - scheduler.observe('https://example.com/p/c', 'product'); - const judgments = await scheduler.drain(); - expect(judgments.filter((j) => j.kind === 'archetype-template')).toHaveLength(1); - }); - - it('enqueues a periodic tick every Nth URL', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 5 }); - for (let i = 0; i < 5; i++) { - scheduler.observe(`https://example.com/p/${i}`, 'page'); - } - const judgments = await scheduler.drain(); - // 1 new-archetype + 1 periodic at the 5th observe() - expect(judgments.filter((j) => j.kind === 'archetype-template')).toHaveLength(1); - expect( - judgments.filter((j) => j.kind === 'foundation-rev' && j.inputs.tickReason === 'periodic'), - ).toHaveLength(1); - }); - - it('enqueues a periodic tick every urlsPerTick URLs (default 5)', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir }); - for (let i = 0; i < 10; i++) { - scheduler.observe(`https://example.com/p/${i}`, 'page'); - } - const judgments = await scheduler.drain(); - expect( - judgments.filter((j) => j.kind === 'foundation-rev' && j.inputs.tickReason === 'periodic'), - ).toHaveLength(2); - }); - - it('persists urlsSeen and archetypesObserved through observe()', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/a', 'page'); - scheduler.observe('https://example.com/b', 'product'); - scheduler.observe('https://example.com/c', 'page'); - const state = loadReplicateState(dir); - expect(state.urlsSeen).toBe(3); - expect(state.archetypesObserved.sort()).toEqual(['page', 'product']); - }); - - it('defers archetype-template ticks until design-foundation.json exists', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/', 'homepage'); - scheduler.observe('https://example.com/about', 'page'); - - // No foundation yet → no archetype-template judgments emerge. - const before = await scheduler.drain(); - expect(before.filter((j) => j.kind === 'archetype-template')).toHaveLength(0); - - // Foundation appears (the consumer would have run design-foundations). - seedFoundation(dir); - - // Next observe() releases the deferred ticks in observation order. - scheduler.observe('https://example.com/contact', 'page'); - const after = await scheduler.drain(); - const archetypes = after - .filter((j) => j.kind === 'archetype-template') - .map((j) => j.archetype); - expect(archetypes).toEqual(['homepage', 'page']); - }); - - it('drain() releases deferred archetype ticks once foundation exists, even without another observe', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/', 'homepage'); - seedFoundation(dir); - - const judgments = await scheduler.drain(); - const archetypes = judgments - .filter((j) => j.kind === 'archetype-template') - .map((j) => j.archetype); - expect(archetypes).toEqual(['homepage']); - }); - - it('emits foundation-rev periodic ticks even before foundation exists', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 5 }); - for (let i = 0; i < 5; i++) { - scheduler.observe(`https://example.com/p/${i}`, 'page'); - } - const judgments = await scheduler.drain(); - // archetype-template is held back… - expect(judgments.filter((j) => j.kind === 'archetype-template')).toHaveLength(0); - // …but the periodic foundation-rev that triggers foundation generation - // must fire — that's what unblocks templating in the first place. - expect( - judgments.filter((j) => j.kind === 'foundation-rev' && j.inputs.tickReason === 'periodic'), - ).toHaveLength(1); - }); - - it('archetype-template stays deferred for a still-new archetype observed after foundation appears', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/', 'homepage'); - seedFoundation(dir); - // New archetype seen with foundation present → enqueued directly. - scheduler.observe('https://example.com/p/x', 'product'); - const judgments = await scheduler.drain(); - const archetypes = judgments - .filter((j) => j.kind === 'archetype-template') - .map((j) => j.archetype); - // Released-deferred (homepage) comes before fresh-direct (product) - // because deferred ticks observed first preserve observation order. - expect(archetypes).toEqual(['homepage', 'product']); - }); - - it('enqueues a manual tick on trigger()', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.trigger('manual'); - const judgments = await scheduler.drain(); - expect(judgments).toHaveLength(1); - expect(judgments[0].kind).toBe('foundation-rev'); - expect(judgments[0].inputs.tickReason).toBe('manual'); - }); - - it('enqueues a foundation-drift tick on trigger()', async () => { - const dir = tmp(); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.trigger('foundation-drift'); - const judgments = await scheduler.drain(); - expect(judgments).toHaveLength(1); - expect(judgments[0].kind).toBe('foundation-rev'); - expect(judgments[0].inputs.tickReason).toBe('foundation-drift'); - }); - - it('drain() empties the queue (single-shot)', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 5 }); - for (let i = 0; i < 5; i++) { - scheduler.observe(`https://example.com/${i}`, 'page'); - } - const first = await scheduler.drain(); - const second = await scheduler.drain(); - expect(first.length).toBeGreaterThan(0); - expect(second).toHaveLength(0); - }); - - it('updates lastTickAt + lastTickReason on a non-empty drain()', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/p', 'product'); - await scheduler.drain(); - const state = loadReplicateState(dir); - expect(state.lastTickAt).not.toBeNull(); - expect(state.lastTickReason).toBe('new-archetype'); - }); - - it('records foundation input digest without being overwritten by later observe calls', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/a', 'page'); - scheduler.recordFoundationInputsDigest('sha256:abc'); - scheduler.observe('https://example.com/b', 'page'); - - const state = loadReplicateState(dir); - expect(state.lastFoundationInputsDigest).toBe('sha256:abc'); - }); - - it('preserves judgment for each archetype in observation order', async () => { - const dir = tmp(); - seedFoundation(dir); - const scheduler = createTickScheduler({ outputDir: dir, urlsPerTick: 100 }); - scheduler.observe('https://example.com/h', 'homepage'); - scheduler.observe('https://example.com/p', 'product'); - scheduler.observe('https://example.com/blog/x', 'post'); - const judgments = await scheduler.drain(); - const archetypes = judgments - .filter((j) => j.kind === 'archetype-template') - .map((j) => j.archetype); - expect(archetypes).toEqual(['homepage', 'product', 'post']); - }); - - it('rejects a non-positive urlsPerTick', () => { - expect(() => createTickScheduler({ outputDir: tmp(), urlsPerTick: 0 })).toThrow(); - expect(() => createTickScheduler({ outputDir: tmp(), urlsPerTick: 1.5 })).toThrow(); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/tick-scheduler.ts b/packages/data-liberation-agent/src/lib/streaming/tick-scheduler.ts deleted file mode 100644 index d2e369057d..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/tick-scheduler.ts +++ /dev/null @@ -1,227 +0,0 @@ -// -// Tick scheduler -// ============== -// Decides when the streaming replicate loop needs a "judgment moment" — i.e. -// when a skill (replicate / design-foundation) should run. The scheduler is -// a small state machine over `replicate-state.json` plus an in-memory queue -// of pending judgments. -// -// Triggers: -// - First URL of a new archetype -> 'new-archetype' tick -// - Every Nth URL (default 5) -> 'periodic' tick (caller checks drift) -// - Caller-driven -> 'foundation-drift' or 'manual' -// -// Foundation gating: `archetype-template` judgments require a design -// foundation to anchor template/pattern generation. While -// `design-foundation.json` is missing, new-archetype ticks are held in a -// deferred queue. When the foundation appears (after the first periodic -// `foundation-rev` runs and the consumer generates the file), the next -// observe() or drain() releases the deferred ticks in observation order. -// `foundation-rev` ticks (periodic / drift / manual) are never deferred. -// -// The scheduler does NOT invoke skills directly. `drain()` returns -// `JudgmentNeeded[]` markers that the calling agent (via the -// `liberate_replicate_tick` MCP handler, the watch CLI, etc.) acts on by -// running the appropriate skill. -// -// Persistence: each `observe()` writes through to `replicate-state.json` via -// `ReplicateStateCache` so a crash mid-loop doesn't lose URL counts. The -// cache batches reads but not writes; if hot-loop write cost matters in the -// future, switch the cache to deferred-write with explicit `flush()` calls. -// -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; -import { ReplicateStateCache } from './replicate-state-cache.js'; - -const FOUNDATION_FILENAME = 'design-foundation.json'; - -export type TickReason = 'new-archetype' | 'periodic' | 'foundation-drift' | 'manual'; - -export interface JudgmentNeeded { - /** What kind of skill the consumer should run. */ - kind: 'archetype-template' | 'foundation-rev' | 'theme-piece'; - /** When `kind === 'archetype-template'`, the archetype that needs templates. */ - archetype?: string; - /** Human-readable reason the agent / CLI can surface to the user. */ - rationale: string; - /** - * Free-form inputs the consumer should pass into the skill. Common keys: - * - `outputDir`: liberation output dir - * - `archetype`: same as the field above (mirrored for skill-side convenience) - * - `tickReason`: 'new-archetype' | 'periodic' | etc. - * - `urlsSeen`: snapshot of state.urlsSeen at observe-time - */ - inputs: Record; -} - -export interface TickScheduler { - /** - * Notify the scheduler a URL just finished extracting. Updates `urlsSeen` + - * `archetypesObserved`; enqueues `new-archetype` and / or `periodic` ticks - * as appropriate. - */ - observe(url: string, archetype: string): void; - /** - * Manually enqueue a tick for a given reason. Useful when the consumer has - * external evidence drift has occurred (e.g. user invoked a manual rebuild). - */ - trigger(reason: TickReason): void; - /** - * Drain all queued ticks and return the resulting JudgmentNeeded markers. - * The internal queue is cleared even if the consumer ignores the returned - * markers — `drain()` is a single-shot operation per tick window. - */ - drain(): Promise; - /** - * Persist the digest of the foundation inputs that have just been handled. - * This goes through the scheduler's cache so later observe() calls in the - * same process do not overwrite an out-of-band replicate-state write. - */ - recordFoundationInputsDigest(digest: string): void; -} - -export interface TickSchedulerOpts { - outputDir: string; - /** How many URLs between periodic ticks. Default 5. */ - urlsPerTick?: number; -} - -interface PendingTick { - reason: TickReason; - archetype?: string; - urlsSeenAtEnqueue: number; -} - -export function createTickScheduler(opts: TickSchedulerOpts): TickScheduler { - const urlsPerTick = opts.urlsPerTick ?? 5; - if (urlsPerTick < 1 || !Number.isInteger(urlsPerTick)) { - throw new Error(`urlsPerTick must be a positive integer (got ${urlsPerTick})`); - } - const cache = new ReplicateStateCache(opts.outputDir); - const queue: PendingTick[] = []; - // Held archetype-template ticks waiting for the design foundation to exist. - // Released in observation order once `design-foundation.json` appears. - const deferredArchetypeTicks: PendingTick[] = []; - // Set-based dedup so two URLs of the same new archetype don't both enqueue. - const enqueuedNewArchetypes = new Set(); - - function foundationFileExists(): boolean { - return existsSync(join(opts.outputDir, FOUNDATION_FILENAME)); - } - - function releaseDeferredIfReady(): void { - if (deferredArchetypeTicks.length === 0) return; - if (!foundationFileExists()) return; - queue.push(...deferredArchetypeTicks); - deferredArchetypeTicks.length = 0; - } - - function observe(url: string, archetype: string): void { - void url; // accepted for future use; the scheduler is archetype-driven today - - // Release any previously-deferred archetype ticks first. This preserves - // observation order: archetypes seen earlier (while the foundation was - // still missing) end up ahead of any archetype enqueued by this same - // observe() call. - releaseDeferredIfReady(); - - let wasNewArchetype = false; - const updated = cache.update((s) => { - wasNewArchetype = !s.archetypesObserved.includes(archetype); - const next = { ...s, urlsSeen: s.urlsSeen + 1 }; - if (wasNewArchetype) { - next.archetypesObserved = [...s.archetypesObserved, archetype].sort(); - } - return next; - }); - - // Set-based in-process dedup: even if `wasNewArchetype` slips (e.g. two - // observe() calls race for the same archetype before the first persists), - // we enqueue exactly one new-archetype tick per archetype per run. - if (wasNewArchetype && !enqueuedNewArchetypes.has(archetype)) { - enqueuedNewArchetypes.add(archetype); - const tick: PendingTick = { - reason: 'new-archetype', - archetype, - urlsSeenAtEnqueue: updated.urlsSeen, - }; - // Hold archetype-template ticks until the design foundation exists. - // Templates/patterns need foundation tokens (palette, typography, - // spacing) to anchor — running them first wastes an agent invocation. - if (foundationFileExists()) { - queue.push(tick); - } else { - deferredArchetypeTicks.push(tick); - } - } - - if (updated.urlsSeen > 0 && updated.urlsSeen % urlsPerTick === 0) { - queue.push({ reason: 'periodic', urlsSeenAtEnqueue: updated.urlsSeen }); - } - } - - function trigger(reason: TickReason): void { - const state = cache.get(); - queue.push({ reason, urlsSeenAtEnqueue: state.urlsSeen }); - } - - async function drain(): Promise { - // Last-chance release: foundation may have appeared between observe() and - // drain() (e.g. the consumer just finished processing a foundation-rev - // tick from this same drain cycle in a prior loop iteration). - releaseDeferredIfReady(); - const ticks = queue.splice(0, queue.length); - const judgments: JudgmentNeeded[] = []; - - for (const tick of ticks) { - if (tick.reason === 'new-archetype' && tick.archetype) { - judgments.push({ - kind: 'archetype-template', - archetype: tick.archetype, - rationale: `First ${tick.archetype} URL observed; templates and patterns for this archetype should be generated.`, - inputs: { - outputDir: opts.outputDir, - archetype: tick.archetype, - tickReason: tick.reason, - urlsSeen: tick.urlsSeenAtEnqueue, - }, - }); - } else if (tick.reason === 'periodic' || tick.reason === 'foundation-drift' || tick.reason === 'manual') { - judgments.push({ - kind: 'foundation-rev', - rationale: - tick.reason === 'periodic' - ? `Periodic check after ${tick.urlsSeenAtEnqueue} URLs; consumer should compute drift score and re-run design-foundation if needed.` - : tick.reason === 'foundation-drift' - ? 'Foundation inputs have drifted; re-run design-foundation skill.' - : 'Manual tick triggered.', - inputs: { - outputDir: opts.outputDir, - tickReason: tick.reason, - urlsSeen: tick.urlsSeenAtEnqueue, - }, - }); - } - } - - if (judgments.length > 0) { - const lastReason = ticks[ticks.length - 1].reason; - cache.update((s) => ({ - ...s, - lastTickAt: new Date().toISOString(), - lastTickReason: lastReason, - })); - } - - return judgments; - } - - function recordFoundationInputsDigest(digest: string): void { - cache.update((s) => ({ - ...s, - lastFoundationInputsDigest: digest, - })); - } - - return { observe, trigger, drain, recordFoundationInputsDigest }; -} diff --git a/packages/data-liberation-agent/src/lib/streaming/url-stream.test.ts b/packages/data-liberation-agent/src/lib/streaming/url-stream.test.ts deleted file mode 100644 index 726a6b261a..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/url-stream.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { fromArray, fromPromise, take, filter, type InventoryEntry } from './url-stream.js'; - -async function collect(source: AsyncIterable): Promise { - const out: T[] = []; - for await (const item of source) out.push(item); - return out; -} - -describe('url-stream', () => { - const items: InventoryEntry[] = [ - { url: 'https://example.com/', type: 'homepage' }, - { url: 'https://example.com/about', type: 'page' }, - { url: 'https://example.com/blog/post-1', type: 'post' }, - ]; - - it('fromArray yields each entry in order', async () => { - const out = await collect(fromArray(items)); - expect(out).toEqual(items); - }); - - it('fromPromise resolves and yields', async () => { - const out = await collect(fromPromise(Promise.resolve(items))); - expect(out).toEqual(items); - }); - - it('take caps at N', async () => { - const out = await collect(take(fromArray(items), 2)); - expect(out).toEqual(items.slice(0, 2)); - }); - - it('take(0) yields nothing', async () => { - const out = await collect(take(fromArray(items), 0)); - expect(out).toEqual([]); - }); - - it('filter applies predicate lazily', async () => { - const out = await collect(filter(fromArray(items), (i) => i.type === 'page')); - expect(out).toHaveLength(1); - expect(out[0].url).toBe('https://example.com/about'); - }); - - it('take + filter compose', async () => { - const out = await collect(take(filter(fromArray(items), (i) => i.type !== 'homepage'), 1)); - expect(out).toEqual([{ url: 'https://example.com/about', type: 'page' }]); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/streaming/url-stream.ts b/packages/data-liberation-agent/src/lib/streaming/url-stream.ts deleted file mode 100644 index 39982f3e5b..0000000000 --- a/packages/data-liberation-agent/src/lib/streaming/url-stream.ts +++ /dev/null @@ -1,55 +0,0 @@ -// -// URL stream -// ========== -// Async iterator over URLs an adapter discovers. Adapters that already -// produce a final array (most do today) wrap their result in fromArray() so -// streaming consumers can do `for await (const url of urlStream)`. Adapters -// that want true incremental discovery pass an AsyncIterable directly. -// -// Memory: arrays are not buffered into a separate structure — the iterator -// yields directly off the source. -// - -export interface InventoryEntry { - url: string; - type: string; -} - -/** Wrap a finite array of URLs into an async iterable. */ -export async function* fromArray(items: InventoryEntry[]): AsyncIterable { - for (const item of items) { - yield item; - } -} - -/** Wrap a Promise of an array (e.g. `adapter.discover(url)` result) into an async iterable. */ -export async function* fromPromise(p: Promise): AsyncIterable { - const items = await p; - yield* fromArray(items); -} - -/** - * Take the first N entries from a stream. Useful for capping streaming runs - * during tests or quick previews without burning the whole crawl. - */ -export async function* take(source: AsyncIterable, n: number): AsyncIterable { - if (n <= 0) return; - let count = 0; - for await (const item of source) { - yield item; - count += 1; - if (count >= n) return; - } -} - -/** - * Filter a stream by predicate. Same shape as Array.filter but lazy. - */ -export async function* filter( - source: AsyncIterable, - pred: (x: T) => boolean, -): AsyncIterable { - for await (const item of source) { - if (pred(item)) yield item; - } -} diff --git a/packages/data-liberation-agent/src/lib/studio-cli.ts b/packages/data-liberation-agent/src/lib/studio-cli.ts deleted file mode 100644 index 2b0345d3d0..0000000000 --- a/packages/data-liberation-agent/src/lib/studio-cli.ts +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Cross-platform Studio CLI invocation (STU-2020). - * - * Every spawn of the Studio CLI must go through this module instead of - * `execFile*('studio', ...)`. The global `studio` command is the wp-studio npm - * package's JS entry (`dist/cli/main.mjs`). On POSIX the bin is a symlink to - * that file and spawns directly. On Windows npm wraps it in a `studio.cmd` - * batch shim, which Node >= 20.12 refuses to spawn without `shell: true` - * (EINVAL, the CVE-2024-27980 hardening) — and `shell: true` is not an option - * here because call sites pass `wp eval` PHP payloads and whole post bodies - * that cmd.exe re-parsing would mangle. Instead we run the same JS entry the - * shim wraps with the current Node binary, so argv passes through byte-exact, - * no shell involved. - */ -import { - execFile, - execFileSync, - type ChildProcess, - type ExecFileOptions, - type ExecFileSyncOptions, -} from 'node:child_process'; -import { existsSync, readFileSync } from 'node:fs'; -import { delimiter, dirname, join } from 'node:path'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify( execFile ); - -let cached: { file: string; prefix: string[] } | null = null; - -/** - * Launcher layout (standalone bundle and the desktop app's `resources/bin/`): - * a bundled `node.exe` beside the shim, entry at `../cli/main.mjs` or - * `../dist/cli/main.mjs`. Mirrors `apps/studio/bin/studio-cli.bat`. - */ -function resolveLauncherLayout( dir: string ): { file: string; prefix: string[] } | null { - const bundledNode = join( dir, 'node.exe' ); - const node = existsSync( bundledNode ) ? bundledNode : process.execPath; - for ( const entry of [ - join( dir, '..', 'cli', 'main.mjs' ), - join( dir, '..', 'dist', 'cli', 'main.mjs' ), - ] ) { - if ( existsSync( entry ) ) { - return { file: node, prefix: [ '--experimental-wasm-jspi', entry ] }; - } - } - return null; -} - -/** Resolve how to spawn the Studio CLI on this platform (memoized). */ -function studioCommand(): { file: string; prefix: string[] } { - if ( cached ) return cached; - if ( process.platform !== 'win32' ) { - return ( cached = { file: 'studio', prefix: [] } ); - } - for ( const dir of ( process.env.PATH ?? '' ).split( delimiter ) ) { - if ( ! dir ) continue; - if ( existsSync( join( dir, 'studio.cmd' ) ) ) { - // npm global layout: studio.cmd + node_modules/wp-studio; the package's - // bin field names the JS entry the shim wraps. - try { - const pkgDir = join( dir, 'node_modules', 'wp-studio' ); - const { bin } = JSON.parse( readFileSync( join( pkgDir, 'package.json' ), 'utf8' ) ) as { - bin?: string | { studio?: string }; - }; - const rel = typeof bin === 'string' ? bin : bin?.studio; - if ( rel && existsSync( join( pkgDir, rel ) ) ) { - return ( cached = { file: process.execPath, prefix: [ join( pkgDir, rel ) ] } ); - } - } catch { - // No readable wp-studio package beside this shim — try the launcher layout. - } - // Standalone layout: node.exe + entry beside the shim. - const launcher = resolveLauncherLayout( dir ); - if ( launcher ) return ( cached = launcher ); - } - // Desktop app layout: a studio.bat one-liner (`"%~dp0\" %*`) forwarding - // to the versioned studio-cli.bat; see windows-installation-manager.ts. - const proxy = join( dir, 'studio.bat' ); - if ( existsSync( proxy ) ) { - try { - const target = readFileSync( proxy, 'utf8' ).match( /"%~dp0\\?([^"]+)"\s+%\*/ )?.[ 1 ]; - const launcher = target && resolveLauncherLayout( dirname( join( dir, target ) ) ); - if ( launcher ) return ( cached = launcher ); - } catch { - // Unreadable proxy — keep scanning PATH. - } - } - } - throw new Error( - 'Studio CLI not found on PATH. Enable the `studio` command from the Studio app settings, or install it with `npm i -g wp-studio`.' - ); -} - -/** - * `execFileSync('studio', args, opts)`, cross-platform. Output is utf8 text - * (empty when the caller pipes stdout away, e.g. `stdio: 'ignore'`). - */ -export function studioExecFileSync( args: string[], opts: ExecFileSyncOptions = {} ): string { - const { file, prefix } = studioCommand(); - return execFileSync( file, [ ...prefix, ...args ], { ...opts, encoding: 'utf8' } ) ?? ''; -} - -/** Promisified `execFile('studio', args, opts)`, cross-platform. */ -export function studioExecFileAsync( - args: string[], - opts: ExecFileOptions = {} -): Promise< { stdout: string; stderr: string } > { - const { file, prefix } = studioCommand(); - return execFileAsync( file, [ ...prefix, ...args ], opts ) as Promise< { - stdout: string; - stderr: string; - } >; -} - -/** - * `execFile('studio', args, opts)`, cross-platform — returns the ChildProcess, - * for callers that need stdin/stream access (e.g. answering `studio site - * delete`'s prompt). - */ -export function studioExecFile( args: string[], opts: ExecFileOptions = {} ): ChildProcess { - const { file, prefix } = studioCommand(); - return execFile( file, [ ...prefix, ...args ], opts, undefined ); -} diff --git a/packages/data-liberation-agent/src/lib/verification/verify.test.ts b/packages/data-liberation-agent/src/lib/verification/verify.test.ts deleted file mode 100644 index 21ee4a6d8e..0000000000 --- a/packages/data-liberation-agent/src/lib/verification/verify.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { mkdirSync, writeFileSync, rmSync } from 'fs'; -import { join } from 'path'; -import { verifyExtraction, type VerificationReport } from './verify.js'; - -const TMP = join(import.meta.dirname, '__test_verify__'); - -beforeEach(() => { - mkdirSync(TMP, { recursive: true }); - mkdirSync(join(TMP, 'media'), { recursive: true }); -}); - -afterEach(() => { - rmSync(TMP, { recursive: true, force: true }); -}); - -function writeMinimalWxr(content: string = '

Hello

'): void { - const wxr = ` - - - Test - 1.2 - - Page One - - - - -`; - writeFileSync(join(TMP, 'output.wxr'), wxr); -} - -function writeLog(lines: object[]): void { - writeFileSync( - join(TMP, 'extraction-log.jsonl'), - lines.map((l) => JSON.stringify(l)).join('\n') + '\n' - ); -} - -function writeRedirectMap(redirects: Array<{ from: string; to: string }>): void { - writeFileSync(join(TMP, 'redirect-map.json'), JSON.stringify(redirects, null, 2)); -} - -function writeMediaStubs(stubs: Record): void { - writeFileSync( - join(TMP, 'media-stubs.json'), - JSON.stringify({ version: 1, stubs }, null, 2) - ); -} - -/** - * Build a WXR where one CDN URL appears BOTH as an attachment_url (provenance) - * and inside content (an ) — the exact double-occurrence shape that the - * old whole-WXR scan over-counted. The host is generic Squarespace CDN infra - * (the incident platform); the path segments are synthetic, not site data. - */ -function writeWxrWithAttachmentAndContent(cdnUrl: string, includeInContent: boolean): void { - const wxr = ` - - - Test - 1.2 - - Photo - attachment - - - - Page One - page - Body${includeInContent ? ` ` : ''}

]]>
-
-
-
`; - writeFileSync(join(TMP, 'output.wxr'), wxr); -} - -describe('verifyExtraction', () => { - it('returns clean report for valid extraction', async () => { - writeMinimalWxr(); - writeLog([ - { type: 'processed', url: 'https://example.com/', slug: 'homepage', durationMs: 500, qualityScore: 'high' }, - ]); - writeRedirectMap([{ from: '/', to: '/homepage' }]); - - const report = await verifyExtraction(TMP); - - expect(report.outputDir).toBe(TMP); - expect(report.wxrFound).toBe(true); - expect(report.contentItems).toBeGreaterThan(0); - expect(report.staleCdnUrls).toEqual([]); - expect(report.failedUrls).toEqual([]); - expect(report.failedMedia).toEqual([]); - expect(report.redirectCount).toBe(1); - }); - - it('detects stale Wix CDN URLs in content', async () => { - writeMinimalWxr('

Image:

'); - writeLog([ - { type: 'processed', url: 'https://example.com/', slug: 'homepage', durationMs: 500, qualityScore: 'high' }, - ]); - - const report = await verifyExtraction(TMP); - expect(report.staleCdnUrls.length).toBe(1); - expect(report.staleCdnUrls[0]).toContain('wixstatic.com'); - }); - - it('detects stale Squarespace CDN URLs in content', async () => { - writeMinimalWxr(''); - writeLog([]); - - const report = await verifyExtraction(TMP); - expect(report.staleCdnUrls.length).toBe(1); - expect(report.staleCdnUrls[0]).toContain('squarespace-cdn.com'); - }); - - it('routes a content CDN URL WITH a success stub to downloaded-not-rewritten, NOT staleCdnUrls', async () => { - const cdn = 'https://images.squarespace-cdn.com/content/v1/test/hero.jpg'; - writeMinimalWxr(`

Image:

`); - writeLog([]); - writeMediaStubs({ [cdn]: { status: 'success', localPath: 'media/hero.jpg' } }); - - const report = await verifyExtraction(TMP); - expect(report.cdnInContentDownloadedNotRewritten).toEqual([cdn]); - expect(report.cdnInContentNoLocalCopy).toEqual([]); - expect(report.staleCdnUrls).toEqual([]); - // Wording: the downloaded-not-rewritten item must say it's NOT a breakage risk. - expect(report.manualAttentionItems.join('\n')).toContain('not a breakage risk'); - expect(report.manualAttentionItems.join('\n')).not.toMatch(/may break/); - }); - - it('routes a content CDN URL with NO stub to staleCdnUrls (genuine risk)', async () => { - const cdn = 'https://images.squarespace-cdn.com/content/v1/test/orphan.jpg'; - writeMinimalWxr(`

Image:

`); - writeLog([]); - // A success stub for a DIFFERENT url — the content url has no local copy. - writeMediaStubs({ 'https://images.squarespace-cdn.com/content/v1/test/other.jpg': { status: 'success' } }); - - const report = await verifyExtraction(TMP); - expect(report.cdnInContentNoLocalCopy).toEqual([cdn]); - expect(report.staleCdnUrls).toEqual([cdn]); - expect(report.cdnInContentDownloadedNotRewritten).toEqual([]); - expect(report.manualAttentionItems.join('\n')).toContain('may break'); - }); - - it('does NOT flag a CDN URL that appears ONLY as attachment_url (not in content)', async () => { - const cdn = 'https://images.squarespace-cdn.com/content/v1/test/provenance.jpg'; - // Same URL as attachment_url provenance, but NOT referenced in any content body. - writeWxrWithAttachmentAndContent(cdn, /* includeInContent */ false); - writeLog([]); - writeMediaStubs({ [cdn]: { status: 'success', localPath: 'media/provenance.jpg' } }); - - const report = await verifyExtraction(TMP); - expect(report.staleCdnUrls).toEqual([]); - expect(report.cdnInContentNoLocalCopy).toEqual([]); - expect(report.cdnInContentDownloadedNotRewritten).toEqual([]); - }); - - it('a downloaded image present BOTH as attachment_url and in content is rewrite-gap, not stale', async () => { - // The reported incident: 380 URLs each appeared twice (attachment_url + - // content img). The whole-WXR scan flagged all 380 as "may break". With the - // content-only scan + stub cross-ref, a downloaded copy lands in the - // not-a-risk bucket and staleCdnUrls is empty. - const cdn = 'https://images.squarespace-cdn.com/content/v1/test/double.jpg'; - writeWxrWithAttachmentAndContent(cdn, /* includeInContent */ true); - writeLog([]); - writeMediaStubs({ [cdn]: { status: 'success', localPath: 'media/double.jpg' } }); - - const report = await verifyExtraction(TMP); - expect(report.staleCdnUrls).toEqual([]); - expect(report.cdnInContentDownloadedNotRewritten).toEqual([cdn]); - }); - - it('degrades gracefully when media-stubs.json is absent (legacy run) — content CDN url treated as stale', async () => { - const cdn = 'https://images.squarespace-cdn.com/content/v1/test/legacy.jpg'; - writeMinimalWxr(`

`); - writeLog([]); - // No media-stubs.json written. - - const report = await verifyExtraction(TMP); - expect(report.staleCdnUrls).toEqual([cdn]); - expect(report.cdnInContentNoLocalCopy).toEqual([cdn]); - expect(report.cdnInContentDownloadedNotRewritten).toEqual([]); - }); - - it('reports failed URLs from extraction log', async () => { - writeMinimalWxr(); - writeLog([ - { type: 'processed', url: 'https://example.com/', slug: 'homepage', durationMs: 500, qualityScore: 'high' }, - { type: 'failed', url: 'https://example.com/broken', error: 'Timeout' }, - ]); - - const report = await verifyExtraction(TMP); - expect(report.failedUrls).toEqual([ - { url: 'https://example.com/broken', error: 'Timeout' }, - ]); - }); - - it('reports failed media downloads', async () => { - writeMinimalWxr(); - writeLog([ - { type: 'media_failed', url: 'https://cdn.example.com/img.jpg', error: '404' }, - ]); - - const report = await verifyExtraction(TMP); - expect(report.failedMedia).toEqual([ - { url: 'https://cdn.example.com/img.jpg', error: '404' }, - ]); - }); - - it('counts media files on disk', async () => { - writeMinimalWxr(); - writeLog([]); - writeFileSync(join(TMP, 'media', 'photo.jpg'), 'fake'); - - const report = await verifyExtraction(TMP); - expect(report.mediaOnDisk).toBe(1); - }); - - it('counts post_type items in the PLAIN-text WXR form (not just CDATA-wrapped)', async () => { - // The WXR builder emits page (no CDATA); the - // counter must match both forms or it reports 0 on real output. - const wxr = ` - - page - page - attachment - -`; - writeFileSync(join(TMP, 'output.wxr'), wxr); - writeLog([]); - const report = await verifyExtraction(TMP); - expect(report.pages).toBe(2); - expect(report.mediaAttachments).toBe(1); - expect(report.posts).toBe(1); - }); - - it('reports missing WXR gracefully', async () => { - writeLog([]); - - const report = await verifyExtraction(TMP); - expect(report.wxrFound).toBe(false); - expect(report.contentItems).toBe(0); - }); - - it('includes quality score breakdown', async () => { - writeMinimalWxr(); - writeLog([ - { type: 'processed', url: 'https://example.com/a', slug: 'a', qualityScore: 'high' }, - { type: 'processed', url: 'https://example.com/b', slug: 'b', qualityScore: 'medium' }, - { type: 'processed', url: 'https://example.com/c', slug: 'c', qualityScore: 'low' }, - ]); - - const report = await verifyExtraction(TMP); - expect(report.qualityScores).toEqual({ high: 1, medium: 1, low: 1 }); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/verification/verify.ts b/packages/data-liberation-agent/src/lib/verification/verify.ts deleted file mode 100644 index 09540ec0e7..0000000000 --- a/packages/data-liberation-agent/src/lib/verification/verify.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { readFileSync, existsSync, readdirSync } from 'fs'; -import { join } from 'path'; - -export interface VerificationReport { - outputDir: string; - wxrFound: boolean; - contentItems: number; - pages: number; - posts: number; - mediaAttachments: number; - mediaOnDisk: number; - /** - * Genuinely-risky CDN URLs: referenced in post/page CONTENT but with NO - * locally-downloaded copy. Back-compat alias for `cdnInContentNoLocalCopy`. - * These are the ones that may break when the source site changes. - */ - staleCdnUrls: string[]; - /** - * CDN URLs found in `` that have NO successful media stub — - * the real "images may break" set. Same as `staleCdnUrls`. - */ - cdnInContentNoLocalCopy: string[]; - /** - * CDN URLs found in `` that DO have a successful media stub - * (the asset is captured locally). This is a post_content rewrite gap, NOT a - * breakage risk — the file exists in media/ and as a WXR attachment. - */ - cdnInContentDownloadedNotRewritten: string[]; - failedUrls: Array<{ url: string; error: string }>; - failedMedia: Array<{ url: string; error: string }>; - redirectCount: number; - qualityScores: { high: number; medium: number; low: number }; - manualAttentionItems: string[]; -} - -const STALE_CDN_PATTERNS = [ - /https?:\/\/[^\s"'<>]*wixstatic\.com[^\s"'<>]*/g, - /https?:\/\/[^\s"'<>]*wixmp\.com[^\s"'<>]*/g, - /https?:\/\/[^\s"'<>]*squarespace-cdn\.com[^\s"'<>]*/g, - /https?:\/\/[^\s"'<>]*static\.squarespace\.com[^\s"'<>]*/g, - /https?:\/\/[^\s"'<>]*assets\.squarespace\.com[^\s"'<>]*/g, - /https?:\/\/[^\s"'<>]*cdn\.shopify\.com[^\s"'<>]*/g, - /https?:\/\/[^\s"'<>]*assets-global\.website-files\.com[^\s"'<>]*/g, -]; - -/** - * Collect CDN URLs that appear inside `` (post/page bodies) - * ONLY. Deliberately excludes `` — those are expected - * provenance for downloaded-and-attached media, not a breakage risk. Scanning - * the whole WXR (the old behavior) double-counted every downloaded image (once - * as attachment_url, once as a content ) and falsely flagged captured - * assets as "may break". - */ -function scanContentForCdnUrls(wxrContent: string): string[] { - const found = new Set(); - const contentBlocks = wxrContent.match(/\s*/g) || []; - const allContent = contentBlocks.join('\n'); - for (const pattern of STALE_CDN_PATTERNS) { - const matches = allContent.match(pattern) || []; - for (const m of matches) found.add(m); - } - return [...found]; -} - -interface MediaStubFile { - version?: number; - stubs?: Record; -} - -/** - * Set of media URLs that were successfully downloaded this run (a `success` - * stub). Returns `null` when media-stubs.json is absent or unreadable — older - * runs predate the store, so the caller degrades to "unknown" and keeps the - * legacy behavior of treating every content CDN URL as potentially stale. - */ -function loadDownloadedMediaUrls(outputDir: string): Set | null { - const stubPath = join(outputDir, 'media-stubs.json'); - if (!existsSync(stubPath)) return null; - try { - const data = JSON.parse(readFileSync(stubPath, 'utf8')) as MediaStubFile; - if (!data.stubs) return null; - const downloaded = new Set(); - for (const [url, stub] of Object.entries(data.stubs)) { - if (stub?.status === 'success') downloaded.add(url); - } - return downloaded; - } catch { - return null; - } -} - -/** - * Bucket content-referenced CDN URLs by whether the run captured a local copy. - * - `noLocalCopy`: in content, NO success stub → genuine breakage risk. - * - `downloadedNotRewritten`: in content, HAS a success stub → rewrite gap only. - * When `downloaded` is null (no media-stubs.json), every URL is treated as - * no-local-copy to preserve legacy behavior on older runs. - */ -function bucketContentCdnUrls( - contentCdnUrls: string[], - downloaded: Set | null, -): { noLocalCopy: string[]; downloadedNotRewritten: string[] } { - const noLocalCopy: string[] = []; - const downloadedNotRewritten: string[] = []; - for (const url of contentCdnUrls) { - if (downloaded && downloaded.has(url)) downloadedNotRewritten.push(url); - else noLocalCopy.push(url); - } - return { noLocalCopy, downloadedNotRewritten }; -} - -function parseExtractionLog(logPath: string): { - failedUrls: Array<{ url: string; error: string }>; - failedMedia: Array<{ url: string; error: string }>; - qualityScores: { high: number; medium: number; low: number }; -} { - const failedUrls: Array<{ url: string; error: string }> = []; - const failedMedia: Array<{ url: string; error: string }> = []; - const qualityScores = { high: 0, medium: 0, low: 0 }; - if (!existsSync(logPath)) return { failedUrls, failedMedia, qualityScores }; - const content = readFileSync(logPath, 'utf8'); - for (const line of content.split('\n')) { - if (!line.trim()) continue; - try { - const entry = JSON.parse(line) as { type: string; url?: string; error?: string; qualityScore?: string; }; - if (entry.type === 'failed' && entry.url) failedUrls.push({ url: entry.url, error: entry.error || 'unknown' }); - if (entry.type === 'media_failed' && entry.url) failedMedia.push({ url: entry.url, error: entry.error || 'unknown' }); - if (entry.type === 'processed' && entry.qualityScore) { - const score = entry.qualityScore as keyof typeof qualityScores; - if (score in qualityScores) qualityScores[score]++; - } - } catch { /* skip malformed lines */ } - } - return { failedUrls, failedMedia, qualityScores }; -} - -function countWxrItems(wxrContent: string): { pages: number; posts: number; media: number } { - // Match BOTH the CDATA-wrapped form () - // AND the plain-text form (page) — the WXR builder - // emits the plain form, so a CDATA-only regex counted 0 on real output. - const countType = (type: string): number => - (wxrContent.match(new RegExp(`\\s*(?:)?\\s*`, 'g')) || []).length; - return { pages: countType('page'), posts: countType('post'), media: countType('attachment') }; -} - -export async function verifyExtraction(outputDir: string): Promise { - const wxrPath = join(outputDir, 'output.wxr'); - const logPath = join(outputDir, 'extraction-log.jsonl'); - const redirectPath = join(outputDir, 'redirect-map.json'); - const mediaDir = join(outputDir, 'media'); - - const wxrFound = existsSync(wxrPath); - let wxrContent = ''; - let pages = 0, posts = 0, mediaAttachments = 0; - let cdnInContentNoLocalCopy: string[] = []; - let cdnInContentDownloadedNotRewritten: string[] = []; - if (wxrFound) { - wxrContent = readFileSync(wxrPath, 'utf8'); - const counts = countWxrItems(wxrContent); - pages = counts.pages; posts = counts.posts; mediaAttachments = counts.media; - const contentCdnUrls = scanContentForCdnUrls(wxrContent); - const downloaded = loadDownloadedMediaUrls(outputDir); - const buckets = bucketContentCdnUrls(contentCdnUrls, downloaded); - cdnInContentNoLocalCopy = buckets.noLocalCopy; - cdnInContentDownloadedNotRewritten = buckets.downloadedNotRewritten; - } - // Back-compat: staleCdnUrls is the genuinely-risky set (no local copy). - const staleCdnUrls = cdnInContentNoLocalCopy; - - const { failedUrls, failedMedia, qualityScores } = parseExtractionLog(logPath); - - let redirectCount = 0; - if (existsSync(redirectPath)) { - try { - const redirects = JSON.parse(readFileSync(redirectPath, 'utf8')); - redirectCount = Array.isArray(redirects) ? redirects.length : 0; - } catch { /* malformed */ } - } - - let mediaOnDisk = 0; - if (existsSync(mediaDir)) { - try { mediaOnDisk = readdirSync(mediaDir).filter((f) => !f.startsWith('.')).length; } catch { /* ignore */ } - } - - const manualAttentionItems: string[] = []; - if (cdnInContentNoLocalCopy.length > 0) manualAttentionItems.push(`${cdnInContentNoLocalCopy.length} stale CDN URL(s) in content with NO local copy — images may break after source site changes`); - if (cdnInContentDownloadedNotRewritten.length > 0) manualAttentionItems.push(`${cdnInContentDownloadedNotRewritten.length} content image(s) reference the source CDN but ARE downloaded locally — rewrite post_content to the local copies (not a breakage risk)`); - if (failedUrls.length > 0) manualAttentionItems.push(`${failedUrls.length} page(s) failed extraction — re-run with --resume or extract manually`); - if (failedMedia.length > 0) manualAttentionItems.push(`${failedMedia.length} media file(s) failed to download — check URLs and retry`); - if (qualityScores.low > 0) manualAttentionItems.push(`${qualityScores.low} page(s) with low quality scores — review content manually`); - - return { - outputDir, wxrFound, contentItems: pages + posts, pages, posts, - mediaAttachments, mediaOnDisk, staleCdnUrls, - cdnInContentNoLocalCopy, cdnInContentDownloadedNotRewritten, - failedUrls, failedMedia, - redirectCount, qualityScores, manualAttentionItems, - }; -} diff --git a/packages/data-liberation-agent/src/lib/woo-csv/index.ts b/packages/data-liberation-agent/src/lib/woo-csv/index.ts deleted file mode 100644 index a99e6219cb..0000000000 --- a/packages/data-liberation-agent/src/lib/woo-csv/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { WooProductCsvBuilder } from './woo-product-csv.js'; -export type { WooProduct } from './woo-product-csv.js'; diff --git a/packages/data-liberation-agent/src/lib/woo-csv/woo-product-csv.test.ts b/packages/data-liberation-agent/src/lib/woo-csv/woo-product-csv.test.ts deleted file mode 100644 index 84e65859c7..0000000000 --- a/packages/data-liberation-agent/src/lib/woo-csv/woo-product-csv.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { WooProductCsvBuilder } from './woo-product-csv.js'; - -describe('WooProductCsvBuilder sourceUrl round-trip', () => { - it('preserves sourceUrl when streaming to JSONL', () => { - const dir = mkdtempSync(join(tmpdir(), 'woo-')); - try { - const b = new WooProductCsvBuilder(); - b.openStream(dir); - b.addProduct({ name: 'A', sourceUrl: 'https://origin.example.com/p/a' }); - b.closeStream(); - const lines = readFileSync(join(dir, 'products.jsonl'), 'utf8').trim().split('\n'); - const parsed = JSON.parse(lines[0]); - expect(parsed.sourceUrl).toBe('https://origin.example.com/p/a'); - } finally { - rmSync(dir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/lib/woo-csv/woo-product-csv.ts b/packages/data-liberation-agent/src/lib/woo-csv/woo-product-csv.ts deleted file mode 100644 index 5535d33b23..0000000000 --- a/packages/data-liberation-agent/src/lib/woo-csv/woo-product-csv.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { writeFileSync, appendFileSync, readFileSync, existsSync } from 'fs'; -import { dirname, join } from 'path'; -import { mkdirSync } from 'fs'; -import Papa from 'papaparse'; - -// --------------------------------------------------------------------------- -// WooCommerce Product CSV Builder -// --------------------------------------------------------------------------- - -export interface WooProduct { - name: string; - type?: 'simple' | 'variable' | 'grouped' | 'external' | 'variation'; - sku?: string; - published?: boolean; - description?: string; - shortDescription?: string; - regularPrice?: string; - salePrice?: string; - categories?: string[]; - tags?: string[]; - images?: string[]; - weight?: string; - length?: string; - width?: string; - height?: string; - inStock?: boolean; - stock?: number; - attributes?: Array<{ - name: string; - values: string[]; - visible?: boolean; - global?: boolean; - }>; - parentSku?: string; - /** SEO title — emitted as `meta:_yoast_wpseo_title` */ - seoTitle?: string; - /** SEO description — emitted as `meta:_yoast_wpseo_metadesc` */ - seoDescription?: string; - /** Cost of goods sold — emitted as `meta:_wc_cog_cost` (WooCommerce COGS plugin) */ - costOfGoods?: string; - /** Source URL of the product on the origin site. Used to cross-reference screenshots against the manifest. */ - sourceUrl?: string; - /** Arbitrary custom post meta — each key becomes a `meta:` column */ - meta?: Record; -} - -// WooCommerce post-meta keys for the three first-class SEO/cost fields. -// These columns are always present in the output (even when empty) so that -// the CSV shape is stable across runs and predictable for import tooling. -const META_KEY_SEO_TITLE = '_yoast_wpseo_title'; -const META_KEY_SEO_DESC = '_yoast_wpseo_metadesc'; -const META_KEY_COGS = '_wc_cog_cost'; -const FIXED_META_KEYS = [META_KEY_SEO_TITLE, META_KEY_SEO_DESC, META_KEY_COGS] as const; - - -export class WooProductCsvBuilder { - private products: WooProduct[] = []; - - addProduct(product: WooProduct): void { - if (this._streaming) { - this.flushProduct(product); - } else { - this.products.push(product); - } - } - - /** - * Determine the maximum number of attribute columns needed across all products. - */ - private maxAttributes(): number { - let max = 0; - for (const p of this.products) { - if (p.attributes && p.attributes.length > max) { - max = p.attributes.length; - } - } - return max; - } - - /** - * Collect the union of custom `meta` keys across all products, excluding - * the three fixed first-class keys (which always get a column regardless). - */ - private customMetaKeys(): string[] { - const keys = new Set(); - for (const p of this.products) { - if (!p.meta) continue; - for (const k of Object.keys(p.meta)) { - if ((FIXED_META_KEYS as readonly string[]).includes(k)) continue; - keys.add(k); - } - } - return [...keys].sort(); - } - - /** - * Build the header row. - */ - private buildHeaders(customMetaKeys: string[]): string[] { - const headers = [ - 'id', - 'type', - 'sku', - 'name', - 'published', - 'short_description', - 'description', - 'regular_price', - 'sale_price', - 'category_ids', - 'tag_ids', - 'images', - 'weight', - 'length', - 'width', - 'height', - 'stock_status', - 'stock_quantity', - ]; - - const attrCount = this.maxAttributes(); - for (let i = 1; i <= attrCount; i++) { - headers.push(`attributes:name${i}`); - headers.push(`attributes:value${i}`); - headers.push(`attributes:visible${i}`); - headers.push(`attributes:taxonomy${i}`); - } - - headers.push('parent_id'); - - // First-class meta columns — always present for a stable shape. - for (const key of FIXED_META_KEYS) { - headers.push(`meta:${key}`); - } - // Adapter-supplied custom meta keys. - for (const key of customMetaKeys) { - headers.push(`meta:${key}`); - } - - return headers; - } - - /** - * Collapse newlines in a string value so CSV fields don't contain raw line breaks. - * HTML content is unaffected visually since newlines are whitespace in HTML. - */ - private static collapseNewlines(value: string): string { - return value.replace(/\r?\n/g, ' '); - } - - /** - * Resolve the value for a given meta key on a product. The three fixed - * keys read from their first-class fields first, then fall through to - * `product.meta`. Custom keys read only from `product.meta`. - */ - private static metaValue(product: WooProduct, key: string): string { - if (key === META_KEY_SEO_TITLE && product.seoTitle) return product.seoTitle; - if (key === META_KEY_SEO_DESC && product.seoDescription) return product.seoDescription; - if (key === META_KEY_COGS && product.costOfGoods) return product.costOfGoods; - return product.meta?.[key] || ''; - } - - /** - * Build a CSV row for a single product. - */ - private buildRow(product: WooProduct, attrCount: number, customMetaKeys: string[]): string[] { - const c = WooProductCsvBuilder.collapseNewlines; - const row: string[] = [ - '', // ID — empty for new products - product.type || 'simple', - product.sku || '', - c(product.name), - product.published === false ? '0' : '1', - c(product.shortDescription || ''), - c(product.description || ''), - product.regularPrice || '', - product.salePrice || '', - product.categories ? product.categories.join(' | ') : '', - product.tags ? product.tags.join(' | ') : '', - product.images ? product.images.join(', ') : '', - product.weight || '', - product.length || '', - product.width || '', - product.height || '', - product.inStock === false ? 'outofstock' : product.inStock === true ? 'instock' : '', - product.stock != null ? String(product.stock) : '', - ]; - - for (let i = 0; i < attrCount; i++) { - const attr = product.attributes?.[i]; - if (attr) { - row.push(attr.name); - row.push(attr.values.join(', ')); - row.push(attr.visible === false ? '0' : '1'); - row.push(attr.global === true ? '1' : '0'); - } else { - row.push('', '', '', ''); - } - } - - row.push(product.parentSku || ''); - - for (const key of FIXED_META_KEYS) { - row.push(c(WooProductCsvBuilder.metaValue(product, key))); - } - for (const key of customMetaKeys) { - row.push(c(WooProductCsvBuilder.metaValue(product, key))); - } - - return row; - } - - /** - * Serialize all products to a CSV file at the given path. - */ - serialize(outputPath: string): void { - mkdirSync(dirname(outputPath), { recursive: true }); - - const customMetaKeys = this.customMetaKeys(); - const headers = this.buildHeaders(customMetaKeys); - const attrCount = this.maxAttributes(); - - const data = this.products.map(p => this.buildRow(p, attrCount, customMetaKeys)); - const csv = Papa.unparse({ fields: headers, data }, { newline: '\r\n' }); - writeFileSync(outputPath, csv, 'utf8'); - } - - // --------------------------------------------------------------------------- - // Streaming mode — write products as JSONL, then build CSV from that - // --------------------------------------------------------------------------- - - private _streamDir: string | null = null; - private _jsonlPath: string | null = null; - private _streaming = false; - - get isStreaming(): boolean { - return this._streaming; - } - - /** - * Begin streaming mode. Products are appended as JSONL lines. - * Pass `{ resume: true }` to append to an existing file instead of - * truncating — required for adapters that persist cross-run state - * (e.g. Shopify GraphQL product handles) in the extraction session. - */ - openStream(outputDir: string, { resume = false }: { resume?: boolean } = {}): void { - mkdirSync(outputDir, { recursive: true }); - this._streamDir = outputDir; - this._jsonlPath = join(outputDir, 'products.jsonl'); - this._streaming = true; - if (!resume) { - writeFileSync(this._jsonlPath, '', 'utf8'); - } - } - - /** - * Append a product as a JSONL line. No memory accumulation. - */ - flushProduct(product: WooProduct): void { - if (!this._streaming || !this._jsonlPath) { - throw new Error('Cannot flushProduct: streaming is not active. Call openStream() first.'); - } - appendFileSync(this._jsonlPath, JSON.stringify(product) + '\n', 'utf8'); - } - - /** - * End streaming. Reads the JSONL, computes maxAttributes, writes products.csv. - * Returns the path to the CSV file. - */ - closeStream(): string { - if (!this._streaming || !this._jsonlPath || !this._streamDir) { - throw new Error('Cannot closeStream: streaming is not active.'); - } - - const csvPath = join(this._streamDir, 'products.csv'); - const products = readJsonl(this._jsonlPath); - - if (products.length > 0) { - // Temporarily load products to use existing serialize logic - this.products = products; - this.serialize(csvPath); - this.products = []; - } - - this._streaming = false; - return csvPath; - } -} - -/** - * Read a products.jsonl file back into WooProduct objects. - */ -function readJsonl(path: string): WooProduct[] { - if (!existsSync(path)) return []; - const content = readFileSync(path, 'utf8'); - const products: WooProduct[] = []; - for (const line of content.split('\n')) { - if (!line.trim()) continue; - try { - products.push(JSON.parse(line) as WooProduct); - } catch { - // Skip malformed lines - } - } - return products; -} diff --git a/packages/data-liberation-agent/src/lib/wordpress/block-policy.test.ts b/packages/data-liberation-agent/src/lib/wordpress/block-policy.test.ts deleted file mode 100644 index e4188f1b96..0000000000 --- a/packages/data-liberation-agent/src/lib/wordpress/block-policy.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { - containsCustomHtmlBlock, - containsUnmarkedCustomHtmlBlock, - PIPELINE_ISLAND_OPENER, - PIPELINE_ISLAND_NAME, -} from './block-policy.js'; - -describe('containsUnmarkedCustomHtmlBlock', () => { - it('flags a bare wp:html opener (hand-authored / legacy island)', () => { - expect(containsUnmarkedCustomHtmlBlock('\n
x
\n')).toBe(true); - }); - - it('does NOT flag a pipeline-marked island', () => { - const island = `${PIPELINE_ISLAND_OPENER}\n
x
\n`; - expect(containsUnmarkedCustomHtmlBlock(island)).toBe(false); - // The strict project-wide ban still sees it as a Custom HTML block — - // compose paths (block-compose / block-transform-apply) keep rejecting it. - expect(containsCustomHtmlBlock(island)).toBe(true); - }); - - it('flags the core/html long-form name when unmarked', () => { - expect(containsUnmarkedCustomHtmlBlock('x')).toBe(true); - }); - - it('flags an opener whose attrs lack the marker name', () => { - expect( - containsUnmarkedCustomHtmlBlock('x'), - ).toBe(true); - }); - - it('flags a mixed document containing one marked and one bare island', () => { - const mixed = - `${PIPELINE_ISLAND_OPENER}\n
a
\n\n` + - `\n
b
\n`; - expect(containsUnmarkedCustomHtmlBlock(mixed)).toBe(true); - }); - - it('flags a broken, unclosed opener (safe side)', () => { - expect(containsUnmarkedCustomHtmlBlock('')).toBe(false); - expect(containsUnmarkedCustomHtmlBlock('

x

')).toBe(false); - expect(containsUnmarkedCustomHtmlBlock('')).toBe(false); - expect(containsUnmarkedCustomHtmlBlock('plain text, no blocks')).toBe(false); - }); - - it('opener constant carries the marker name', () => { - expect(PIPELINE_ISLAND_OPENER).toContain(`"name":"${PIPELINE_ISLAND_NAME}"`); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/wordpress/block-policy.ts b/packages/data-liberation-agent/src/lib/wordpress/block-policy.ts deleted file mode 100644 index dea15fa625..0000000000 --- a/packages/data-liberation-agent/src/lib/wordpress/block-policy.ts +++ /dev/null @@ -1,47 +0,0 @@ -const CUSTOM_HTML_BLOCK_RE = /)/i; - -export function containsCustomHtmlBlock(markup: string): boolean { - return CUSTOM_HTML_BLOCK_RE.test(markup); -} - -export function customHtmlBlockError(context: string): string { - return `${context} contains a Custom HTML block. Use existing WordPress core blocks first; create a custom block when needed, and move CSS into style.css instead of wp:html.`; -} - -// -// Pipeline-emitted coverage islands -// --------------------------------- -// The reconstruction pipeline legitimately emits `core/html` fallback islands -// when a section's structured render drops content (engine coverage island). Those -// islands carry a deterministic `metadata.name` marker in the block delimiter -// so the install-time wp:html ban can tell them apart from hand-authored -// Custom HTML blocks. This is a QUALITY gate, not a security boundary against -// the operator — the marker is recognizable, not unforgeable, which is -// sufficient: the ban exists to stop an agent from dumping raw HTML instead -// of composing blocks, and agents are instructed never to emit wp:html at all. -// (metadata.name is a WP-supported block attribute — it round-trips through -// @wordpress/blocks and labels the island in the editor List View.) -// - -export const PIPELINE_ISLAND_NAME = 'lib-coverage-island'; - -/** The exact opening delimiter the pipeline emits for a coverage island. */ -export const PIPELINE_ISLAND_OPENER = ``; - -// Opening wp:html delimiters only (a leading `/` closer never matches `wp:`), -// with optional attribute JSON, matched to the first `-->` (or end of input -// for a broken, unclosed opener — which counts as unmarked, the safe side). -const HTML_BLOCK_OPENER_RE = /)[\s\S]*?(?:-->|$)/gi; - -/** - * True when the markup contains a `wp:html` OPENING delimiter that does NOT - * bear the pipeline island marker. Marked islands (pipeline-emitted coverage - * fallbacks) are allowed; any other Custom HTML block is treated as - * hand-authored and rejected by callers via {@link customHtmlBlockError}. - */ -export function containsUnmarkedCustomHtmlBlock(markup: string): boolean { - for (const m of markup.matchAll(HTML_BLOCK_OPENER_RE)) { - if (!m[0].includes(`"name":"${PIPELINE_ISLAND_NAME}"`)) return true; - } - return false; -} diff --git a/packages/data-liberation-agent/src/lib/wxr/index.ts b/packages/data-liberation-agent/src/lib/wxr/index.ts deleted file mode 100644 index ec94092e84..0000000000 --- a/packages/data-liberation-agent/src/lib/wxr/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { WxrBuilder } from './wxr-builder.js'; -export type { Category, Comment, MediaItem, MenuItem, PageItem, PostItem, Tag, Term, WxrBuilderOpts, WxrItem } from './wxr-builder.js'; -export { readWxr } from './wxr-reader.js'; -export type { WxrData } from './wxr-reader.js'; -export { rehydrateBuilderFromWxr } from './wxr-rehydrate.js'; diff --git a/packages/data-liberation-agent/src/lib/wxr/wxr-builder-status.test.ts b/packages/data-liberation-agent/src/lib/wxr/wxr-builder-status.test.ts deleted file mode 100644 index 1c0a14e993..0000000000 --- a/packages/data-liberation-agent/src/lib/wxr/wxr-builder-status.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { WxrBuilder, type WxrBuilderOpts } from './index.js'; - -const SITE = 'https://example.com'; -const TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(TMP, { recursive: true }); - -function buildXml(opts?: WxrBuilderOpts): string { - const dir = mkdtempSync(join(TMP, 'wxr-status-')); - try { - const w = new WxrBuilder({ title: 'Example', url: SITE, language: 'en-US' }, opts); - w.addPage({ title: 'About', slug: 'about', content: '

a

', sourceUrl: `${SITE}/about` }); - w.addPost({ title: 'Hello', slug: 'hello', content: '

h

', sourceUrl: `${SITE}/hello` }); - w.addMedia({ title: 'Img', slug: 'img', url: `${SITE}/img.jpg`, altText: '', caption: '' }); - w.addMenuItem({ title: 'Home', url: SITE, menuSlug: 'primary' }); - const out = join(dir, 'output.wxr'); - w.serialize(out); - return readFileSync(out, 'utf8'); - } finally { - rmSync(dir, { recursive: true, force: true }); - } -} - -function statusNear(xml: string, postType: string): string | null { - const idx = xml.indexOf(`${postType}`); - if (idx === -1) return null; - const m = xml.slice(idx - 700, idx + 60).match(/([^<]+)<\/wp:status>/g); - return m ? m[m.length - 1] : null; -} - -describe('WxrBuilder post status', () => { - it('DEFAULTS pages/posts to draft (documented "import as drafts" convention)', () => { - const xml = buildXml(); - expect(statusNear(xml, 'page')).toContain('draft'); - expect(statusNear(xml, 'post')).toContain('draft'); - }); - - it('attachments always use WP inherit convention (regardless of contentStatus)', () => { - expect(statusNear(buildXml(), 'attachment')).toContain('inherit'); - expect(statusNear(buildXml({ contentStatus: 'publish' }), 'attachment')).toContain('inherit'); - }); - - it('publishes pages/posts when contentStatus="publish" (replica/preview flow)', () => { - const xml = buildXml({ contentStatus: 'publish' }); - expect(statusNear(xml, 'page')).toContain('publish'); - expect(statusNear(xml, 'post')).toContain('publish'); - expect(xml).not.toContain('draft'); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/wxr/wxr-builder.ts b/packages/data-liberation-agent/src/lib/wxr/wxr-builder.ts deleted file mode 100644 index 4e71b6b07f..0000000000 --- a/packages/data-liberation-agent/src/lib/wxr/wxr-builder.ts +++ /dev/null @@ -1,859 +0,0 @@ -import { writeFileSync, appendFileSync, mkdirSync } from 'fs'; -import { dirname, join } from 'path'; -import { XMLBuilder } from 'fast-xml-parser'; - -export interface SiteMetaInput { - title?: string; - url?: string; - description?: string; - language?: string; -} - -export interface SiteMeta { - title: string; - url: string; - description: string; - language: string; -} - -export interface AuthorInput { - login: string; - email?: string; - displayName?: string; - firstName?: string; - lastName?: string; -} - -export interface Author { - id: number; - login: string; - email: string; - displayName: string; - firstName: string; - lastName: string; -} - -export interface CategoryInput { - slug: string; - name: string; - parent?: string; - description?: string; -} - -export interface Category { - id: number; - slug: string; - name: string; - parent: string; - description: string; -} - -export interface TagInput { - slug: string; - name: string; - description?: string; -} - -export interface Tag { - id: number; - slug: string; - name: string; - description: string; -} - -export interface MediaInput { - url: string; - localPath?: string; - title?: string; - slug?: string; - altText?: string; - caption?: string; -} - -export interface MediaItem { - id: number; - type: 'attachment'; - title: string; - slug: string; - url: string; - localPath?: string; - altText: string; - caption: string; -} - -export interface PageInput { - title: string; - slug: string; - content?: string; - excerpt?: string; - date?: string; - parent?: number; - menuOrder?: number; - author?: string; - seoTitle?: string; - seoDescription?: string; - sourceUrl?: string; -} - -export interface PageItem { - id: number; - type: 'page'; - title: string; - slug: string; - content: string; - excerpt: string; - date: string; - parent: number; - menuOrder: number; - author: string; - seoTitle: string; - seoDescription: string; - sourceUrl: string; -} - -export interface PostInput { - title: string; - slug: string; - content?: string; - excerpt?: string; - date?: string; - categories?: string[]; - tags?: string[]; - featuredMediaId?: number; - author?: string; - seoTitle?: string; - seoDescription?: string; - sourceUrl?: string; - customTerms?: Array<{ taxonomy: string; slug: string }>; -} - -export interface PostItem { - id: number; - type: 'post'; - title: string; - slug: string; - content: string; - excerpt: string; - date: string; - categories: string[]; - tags: string[]; - featuredMediaId: number; - author: string; - seoTitle: string; - seoDescription: string; - sourceUrl: string; - customTerms: Array<{ taxonomy: string; slug: string }>; -} - -export interface MenuItemInput { - title: string; - url: string; - menuSlug: string; - parent?: number; - order?: number; -} - -export interface MenuItem { - id: number; - type: 'nav_menu_item'; - title: string; - slug: string; - url: string; - menuSlug: string; - parent: number; - menuOrder: number; -} - -export interface RedirectInput { - from: string; - to: string; -} - -export interface Redirect { - from: string; - to: string; -} - -export interface CommentInput { - postId: number; - author?: string; - authorEmail?: string; - authorUrl?: string; - authorIp?: string; - date?: string; - content: string; - approved?: boolean; - type?: string; - parent?: number; - userId?: number; -} - -export interface Comment { - id: number; - postId: number; - author: string; - authorEmail: string; - authorUrl: string; - authorIp: string; - date: string; - content: string; - approved: string; - type: string; - parent: number; - userId: number; -} - -export interface TermInput { - taxonomy: string; - slug: string; - name: string; - parent?: string; - description?: string; -} - -export interface Term { - id: number; - taxonomy: string; - slug: string; - name: string; - parent: string; - description: string; -} - -export type WxrItem = MediaItem | PageItem | PostItem | MenuItem; - -export interface ValidationResult { - valid: boolean; - warnings: string[]; -} - -const xmlBuilder = new XMLBuilder({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - cdataPropName: '__cdata', - format: true, - indentBy: ' ', - suppressEmptyNode: false, - processEntities: true, -}); - -/** Escape ]]> inside CDATA content (fast-xml-parser doesn't handle this). */ -function safeCdata(str: string): string { - if (!str) return str; - return str.replace(/]]>/g, ']]]]>'); -} - -/** - * Collapse whitespace that fast-xml-parser inserts around CDATA sections. - * - * fast-xml-parser with format:true produces: - * \n \n - * - * WordPress's importer reads the text content including that whitespace, - * which corrupts values. This collapses it to: - * - */ -function collapseCdataWhitespace(xml: string): string { - return xml.replace(/>\s*()\s*$1<'); -} - -/** Wrap a value for XMLBuilder CDATA output with ]]> escaping. */ -function cd(value: string): { __cdata: string } { - return { __cdata: safeCdata(value) }; -} - -function formatWpDate(isoDate: string): string { - if (!isoDate) return '0000-00-00 00:00:00'; - const d = new Date(isoDate); - const pad = (n: number) => String(n).padStart(2, '0'); - return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`; -} - -function toRFC822(isoDate: string): string { - if (!isoDate) return ''; - const d = new Date(isoDate); - const days = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']; - const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; - const pad = (n: number) => String(n).padStart(2, '0'); - return `${days[d.getUTCDay()]}, ${pad(d.getUTCDate())} ${months[d.getUTCMonth()]} ${d.getUTCFullYear()} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())} +0000`; -} - -/** Optional WxrBuilder configuration. */ -export interface WxrBuilderOpts { - /** - * Status for extracted pages/posts. Default `'draft'` — honors the documented - * "all content imported as drafts; the user reviews and publishes manually" - * convention for the WXR a user imports into their production WordPress. The - * replica/preview flow passes `'publish'` so its nav targets resolve. - * Attachments always use WP's `'inherit'` convention regardless. - */ - contentStatus?: 'draft' | 'publish'; -} - -export class WxrBuilder { - siteMeta: SiteMeta; - contentStatus: 'draft' | 'publish'; - _nextId: number; - authors: Author[]; - categories: Category[]; - tags: Tag[]; - items: WxrItem[]; - redirects: Redirect[]; - comments: Comment[]; - terms: Term[]; - private _streamPath: string | null = null; - private _streaming = false; - /** - * Fallback ISO date used for any item (attachments, nav menu items, or - * pages/posts whose adapter didn't populate a date) that would otherwise - * serialize as `0000-00-00 00:00:00`. A zero date causes WordPress's WXR - * importer to route attachment uploads into `wp-content/uploads/0000/00/`, - * which breaks Studio blueprint application. - */ - private readonly _fallbackDate: string = new Date().toISOString(); - - get isStreaming(): boolean { - return this._streaming; - } - - constructor(siteMeta: SiteMetaInput, opts: WxrBuilderOpts = {}) { - this.contentStatus = opts.contentStatus ?? 'draft'; - this.siteMeta = { - title: siteMeta.title || 'Untitled', - url: (siteMeta.url || '').replace(/\/+$/, ''), - description: siteMeta.description || '', - language: siteMeta.language || 'en-US', - }; - this._nextId = 1; - this.authors = []; - this.categories = []; - this.tags = []; - this.items = []; - this.redirects = []; - this.comments = []; - this.terms = []; - this._streamPath = null; - this._streaming = false; - } - - _id(): number { - return this._nextId++; - } - - addAuthor(author: AuthorInput): number { - const id = this._id(); - this.authors.push({ - id, - login: author.login, - email: author.email || '', - displayName: author.displayName || author.login, - firstName: author.firstName || '', - lastName: author.lastName || '', - }); - return id; - } - - addCategory(cat: CategoryInput): number { - const id = this._id(); - this.categories.push({ - id, - slug: cat.slug, - name: cat.name, - parent: cat.parent || '', - description: cat.description || '', - }); - return id; - } - - addTag(tag: TagInput): number { - const id = this._id(); - this.tags.push({ - id, - slug: tag.slug, - name: tag.name, - description: tag.description || '', - }); - return id; - } - - addMedia(media: MediaInput): number { - const id = this._id(); - this.items.push({ - id, - type: 'attachment', - title: media.title || '', - slug: media.slug || '', - url: media.url, - localPath: media.localPath, - altText: media.altText || '', - caption: media.caption || '', - }); - return id; - } - - addPage(page: PageInput): number { - const id = this._id(); - this.items.push({ - id, - type: 'page', - title: page.title, - slug: page.slug, - content: page.content || '', - excerpt: page.excerpt || '', - date: page.date || '', - parent: page.parent || 0, - menuOrder: page.menuOrder || 0, - author: page.author || '', - seoTitle: page.seoTitle || '', - seoDescription: page.seoDescription || '', - sourceUrl: page.sourceUrl || '', - }); - return id; - } - - addPost(post: PostInput): number { - const id = this._id(); - this.items.push({ - id, - type: 'post', - title: post.title, - slug: post.slug, - content: post.content || '', - excerpt: post.excerpt || '', - date: post.date || '', - categories: post.categories || [], - tags: post.tags || [], - featuredMediaId: post.featuredMediaId || 0, - author: post.author || '', - seoTitle: post.seoTitle || '', - seoDescription: post.seoDescription || '', - sourceUrl: post.sourceUrl || '', - customTerms: post.customTerms || [], - }); - return id; - } - - addMenuItem(item: MenuItemInput): void { - this.items.push({ - id: this._id(), - type: 'nav_menu_item', - title: item.title, - slug: '', - url: item.url, - menuSlug: item.menuSlug, - parent: item.parent || 0, - menuOrder: item.order || 0, - }); - } - - addRedirect(redirect: RedirectInput): void { - this.redirects.push({ from: redirect.from, to: redirect.to }); - } - - addComment(comment: CommentInput): number { - const id = this._id(); - this.comments.push({ - id, - postId: comment.postId, - author: comment.author || '', - authorEmail: comment.authorEmail || '', - authorUrl: comment.authorUrl || '', - authorIp: comment.authorIp || '', - date: comment.date || '', - content: comment.content, - approved: comment.approved === false ? '0' : '1', - type: comment.type || 'comment', - parent: comment.parent || 0, - userId: comment.userId || 0, - }); - return id; - } - - addTerm(term: TermInput): number { - const id = this._id(); - this.terms.push({ - id, - taxonomy: term.taxonomy, - slug: term.slug, - name: term.name, - parent: term.parent || '', - description: term.description || '', - }); - return id; - } - - /** - * Backfill channel-header term declarations for any category/tag referenced - * inline on a post but never explicitly registered via {@link addCategory} / - * {@link addTag}. - * - * Some adapters (e.g. Squarespace) attach terms directly to posts as inline - * `` refs without declaring - * them in the channel header. A spec-compliant WXR 1.2 also declares each term - * once in the header; absent declarations trip validators ("references unknown - * category/tag slug …") even though WordPress can still create the terms from - * the inline refs. - * - * The inline `nicename` emitted by {@link _serializeItem} is the raw slug - * string from `post.categories` / `post.tags`, so the backfilled declaration's - * resolvable key (`wp:category_nicename` / `wp:tag_slug`) MUST equal that raw - * string verbatim for references to resolve. We therefore use the inline value - * as both the declared slug and the human-readable name (Squarespace term refs - * already carry display-shaped values like "The New York Times"). - * - * Idempotent and order-independent: only terms missing from `this.categories` - * / `this.tags` are appended, so it is safe to call from both `serialize()` - * and `openStream()` and safe to call more than once. - */ - private _backfillInlineTerms(): void { - const knownCategorySlugs = new Set(this.categories.map((c) => c.slug)); - const knownTagSlugs = new Set(this.tags.map((t) => t.slug)); - const seenCategorySlugs = new Set(); - const seenTagSlugs = new Set(); - - for (const item of this.items) { - if (item.type !== 'post') continue; - for (const slug of item.categories) { - if (!slug || knownCategorySlugs.has(slug) || seenCategorySlugs.has(slug)) continue; - seenCategorySlugs.add(slug); - this.addCategory({ slug, name: slug }); - } - for (const slug of item.tags) { - if (!slug || knownTagSlugs.has(slug) || seenTagSlugs.has(slug)) continue; - seenTagSlugs.add(slug); - this.addTag({ slug, name: slug }); - } - } - } - - validate(): ValidationResult { - const warnings: string[] = []; - const attachmentIds = new Set( - this.items.filter((i): i is MediaItem => i.type === 'attachment').map((i) => i.id) - ); - const categorySlugs = new Set(this.categories.map((c) => c.slug)); - const tagSlugs = new Set(this.tags.map((t) => t.slug)); - - const termKeys = new Set(this.terms.map((t) => `${t.taxonomy}:${t.slug}`)); - const itemIds = new Set(this.items.map((i) => i.id)); - - for (const item of this.items) { - if (item.type === 'post' || item.type === 'page') { - if (!item.content || item.content.trim() === '') { - warnings.push(`"${item.title}" (${item.type}) has empty content`); - } - } - if (item.type === 'post') { - if (item.featuredMediaId && !attachmentIds.has(item.featuredMediaId)) { - warnings.push( - `Post "${item.title}" references featuredMediaId ${item.featuredMediaId} which does not exist` - ); - } - for (const slug of item.categories) { - if (!categorySlugs.has(slug)) { - warnings.push(`Post "${item.title}" references unknown category slug "${slug}"`); - } - } - for (const slug of item.tags) { - if (!tagSlugs.has(slug)) { - warnings.push(`Post "${item.title}" references unknown tag slug "${slug}"`); - } - } - for (const ct of item.customTerms) { - if (!termKeys.has(`${ct.taxonomy}:${ct.slug}`)) { - warnings.push( - `Post "${item.title}" references unregistered custom term "${ct.taxonomy}:${ct.slug}"` - ); - } - } - } - } - - for (const comment of this.comments) { - if (!itemIds.has(comment.postId)) { - warnings.push(`Comment ${comment.id} references non-existent postId ${comment.postId}`); - } - } - - return { valid: true, warnings }; - } - - private _serializeHeader(): string { - const now = new Date(); - const obj = { - '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, - rss: { - '@_version': '2.0', - '@_xmlns:excerpt': 'http://wordpress.org/export/1.2/excerpt/', - '@_xmlns:content': 'http://purl.org/rss/1.0/modules/content/', - '@_xmlns:wfw': 'http://wellformedweb.org/CommentAPI/', - '@_xmlns:dc': 'http://purl.org/dc/elements/1.1/', - '@_xmlns:wp': 'http://wordpress.org/export/1.2/', - channel: { - title: this.siteMeta.title, - link: this.siteMeta.url, - description: this.siteMeta.description, - pubDate: toRFC822(now.toISOString()), - language: this.siteMeta.language, - 'wp:wxr_version': '1.2', - 'wp:base_site_url': this.siteMeta.url, - 'wp:base_blog_url': this.siteMeta.url, - }, - }, - }; - let xml = collapseCdataWhitespace(xmlBuilder.build(obj) as string); - // Strip closing tags — items and taxonomies are appended after - xml = xml.replace(/\s*<\/channel>\s*\n?\s*<\/rss>\s*$/, ''); - return xml; - } - - private _serializeTaxonomies(): string { - const fragments: string[] = []; - - for (const author of this.authors) { - fragments.push(xmlBuilder.build({ - 'wp:author': { - 'wp:author_id': author.id, - 'wp:author_login': cd(author.login), - 'wp:author_email': cd(author.email), - 'wp:author_display_name': cd(author.displayName), - 'wp:author_first_name': cd(author.firstName), - 'wp:author_last_name': cd(author.lastName), - }, - })); - } - - for (const cat of this.categories) { - const catObj: Record = { - 'wp:term_id': cat.id, - 'wp:category_nicename': cd(cat.slug), - 'wp:category_parent': cd(cat.parent), - 'wp:cat_name': cd(cat.name), - }; - if (cat.description) { - catObj['wp:category_description'] = cd(cat.description); - } - fragments.push(xmlBuilder.build({ 'wp:category': catObj })); - } - - for (const tag of this.tags) { - const tagObj: Record = { - 'wp:term_id': tag.id, - 'wp:tag_slug': cd(tag.slug), - 'wp:tag_name': cd(tag.name), - }; - if (tag.description) { - tagObj['wp:tag_description'] = cd(tag.description); - } - fragments.push(xmlBuilder.build({ 'wp:tag': tagObj })); - } - - for (const term of this.terms) { - const termObj: Record = { - 'wp:term_id': term.id, - 'wp:term_taxonomy': cd(term.taxonomy), - 'wp:term_slug': cd(term.slug), - 'wp:term_parent': cd(term.parent), - 'wp:term_name': cd(term.name), - }; - if (term.description) { - termObj['wp:term_description'] = cd(term.description); - } - fragments.push(xmlBuilder.build({ 'wp:term': termObj })); - } - - return collapseCdataWhitespace(fragments.join('')); - } - - private _serializeItem(item: WxrItem): string { - const slug = item.slug || ''; - const originalDate = (item.type === 'post' || item.type === 'page') ? item.date : ''; - const date = originalDate || this._fallbackDate; - const content = (item.type === 'post' || item.type === 'page') ? item.content : ''; - const excerpt = (item.type === 'post' || item.type === 'page') ? item.excerpt : ''; - const author = (item.type === 'post' || item.type === 'page') ? (item.author || '') : ''; - const parent = (item.type === 'page') ? item.parent : ((item.type === 'nav_menu_item') ? item.parent : 0); - const menuOrder = (item.type === 'page') ? item.menuOrder : ((item.type === 'nav_menu_item') ? item.menuOrder : 0); - // Per-type post status. Attachments use WP's `inherit` convention; pages/ - // posts/nav use `contentStatus` — default 'draft' per the documented - // "import as drafts; the user reviews and publishes manually" convention, - // which the replica/preview flow overrides to 'publish' so its imported nav - // targets resolve instead of 404ing. - const status = - item.type === 'attachment' ? 'inherit' - : this.contentStatus; - - const obj: Record = { - title: item.title, - link: this.siteMeta.url + '/' + slug, - pubDate: date ? toRFC822(date) : '', - 'dc:creator': cd(author), - guid: { '@_isPermaLink': 'false', '#text': this.siteMeta.url + '/?p=' + item.id }, - description: '', - 'content:encoded': cd(content), - 'excerpt:encoded': cd(excerpt), - 'wp:post_id': item.id, - 'wp:post_date': formatWpDate(date), - 'wp:post_date_gmt': formatWpDate(date), - 'wp:comment_status': 'closed', - 'wp:ping_status': 'closed', - 'wp:post_name': slug, - 'wp:status': status, - 'wp:post_parent': parent, - 'wp:menu_order': menuOrder, - 'wp:post_type': item.type, - 'wp:post_password': '', - 'wp:is_sticky': 0, - }; - - // Categories, tags, custom terms (posts only) - if (item.type === 'post' || item.type === 'page') { - if (item.type === 'post') { - const categories: Array> = []; - for (const catSlug of item.categories) { - const cat = this.categories.find((c) => c.slug === catSlug); - categories.push({ '@_domain': 'category', '@_nicename': catSlug, __cdata: safeCdata(cat ? cat.name : catSlug) }); - } - for (const tagSlug of item.tags) { - const tag = this.tags.find((t) => t.slug === tagSlug); - categories.push({ '@_domain': 'post_tag', '@_nicename': tagSlug, __cdata: safeCdata(tag ? tag.name : tagSlug) }); - } - for (const ct of item.customTerms) { - const term = this.terms.find((t) => t.taxonomy === ct.taxonomy && t.slug === ct.slug); - categories.push({ '@_domain': ct.taxonomy, '@_nicename': ct.slug, __cdata: safeCdata(term ? term.name : ct.slug) }); - } - if (categories.length > 0) obj.category = categories; - - if (item.featuredMediaId) { - this._addPostmeta(obj, '_thumbnail_id', String(item.featuredMediaId)); - } - } - - if (item.seoTitle) this._addPostmeta(obj, '_seo_title', item.seoTitle); - if (item.seoDescription) this._addPostmeta(obj, '_seo_description', item.seoDescription); - if (item.sourceUrl) this._addPostmeta(obj, '_source_url', item.sourceUrl); - } - - if (item.type === 'attachment') { - obj['wp:attachment_url'] = cd(item.url); - if (item.altText) { - this._addPostmeta(obj, '_wp_attachment_image_alt', item.altText); - } - } - - if (item.type === 'nav_menu_item') { - this._addPostmeta(obj, '_menu_item_url', item.url); - this._addPostmeta(obj, '_menu_item_type', 'custom'); - this._addPostmeta(obj, '_menu_slug', item.menuSlug); - } - - // Comments - const itemComments = this.comments.filter((c) => c.postId === item.id); - if (itemComments.length > 0) { - obj['wp:comment'] = itemComments.map((comment) => ({ - 'wp:comment_id': comment.id, - 'wp:comment_author': cd(comment.author), - 'wp:comment_author_email': comment.authorEmail, - 'wp:comment_author_url': comment.authorUrl, - 'wp:comment_author_IP': comment.authorIp, - 'wp:comment_date': formatWpDate(comment.date), - 'wp:comment_date_gmt': formatWpDate(comment.date), - 'wp:comment_content': cd(comment.content), - 'wp:comment_approved': comment.approved, - 'wp:comment_type': comment.type, - 'wp:comment_parent': comment.parent, - 'wp:comment_user_id': comment.userId, - })); - } - - return collapseCdataWhitespace(xmlBuilder.build({ item: obj }) as string); - } - - /** Append a wp:postmeta entry to an item object. */ - private _addPostmeta(obj: Record, key: string, value: string): void { - const meta = { 'wp:meta_key': key, 'wp:meta_value': cd(value) }; - if (!obj['wp:postmeta']) { - obj['wp:postmeta'] = [meta]; - } else { - (obj['wp:postmeta'] as Array).push(meta); - } - } - - serialize(outputPath: string): { validation: ValidationResult; wxrPath: string } { - this._backfillInlineTerms(); - const validation = this.validate(); - - mkdirSync(dirname(outputPath), { recursive: true }); - - const parts: string[] = []; - parts.push(this._serializeHeader()); - - const taxonomies = this._serializeTaxonomies(); - if (taxonomies) { - parts.push(taxonomies); - } - - for (const item of this.items) { - parts.push(this._serializeItem(item)); - } - - parts.push(''); - parts.push(''); - - writeFileSync(outputPath, parts.join('\n'), 'utf8'); - - if (this.redirects.length > 0) { - const redirectPath = join(dirname(outputPath), 'redirect-map.json'); - writeFileSync(redirectPath, JSON.stringify(this.redirects, null, 2), 'utf8'); - } - - return { validation, wxrPath: outputPath }; - } - - openStream(outputPath: string): void { - mkdirSync(dirname(outputPath), { recursive: true }); - - this._backfillInlineTerms(); - const header = this._serializeHeader(); - const taxonomies = this._serializeTaxonomies(); - - let content = header; - if (taxonomies) { - content += '\n' + taxonomies; - } - - writeFileSync(outputPath, content, 'utf8'); - - this._streamPath = outputPath; - this._streaming = true; - } - - flushItem(item: WxrItem): void { - if (!this._streaming || !this._streamPath) { - throw new Error('Cannot flushItem: streaming is not active. Call openStream() first.'); - } - - const itemXml = this._serializeItem(item); - appendFileSync(this._streamPath, '\n' + itemXml, 'utf8'); - } - - closeStream(): { validation: ValidationResult; wxrPath: string } { - if (!this._streaming || !this._streamPath) { - throw new Error('Cannot closeStream: streaming is not active.'); - } - - appendFileSync(this._streamPath, '\n\n', 'utf8'); - - if (this.redirects.length > 0) { - const redirectPath = join(dirname(this._streamPath), 'redirect-map.json'); - writeFileSync(redirectPath, JSON.stringify(this.redirects, null, 2), 'utf8'); - } - - const validation = this.validate(); - const wxrPath = this._streamPath; - - this._streaming = false; - - return { validation, wxrPath }; - } -} diff --git a/packages/data-liberation-agent/src/lib/wxr/wxr-reader.ts b/packages/data-liberation-agent/src/lib/wxr/wxr-reader.ts deleted file mode 100644 index bdcc3df27c..0000000000 --- a/packages/data-liberation-agent/src/lib/wxr/wxr-reader.ts +++ /dev/null @@ -1,326 +0,0 @@ -import { readFileSync, existsSync } from 'fs'; -import { dirname, join } from 'path'; -import { XMLParser } from 'fast-xml-parser'; -import type { - SiteMeta, - Author, - Category, - Tag, - Term, - MediaItem, - PageItem, - PostItem, - MenuItem, - Comment, - Redirect, - WxrItem, -} from './wxr-builder.js'; - -export interface WxrData { - site: SiteMeta; - authors: Author[]; - categories: Category[]; - tags: Tag[]; - terms: Term[]; - items: WxrItem[]; - comments: Comment[]; - redirects: Redirect[]; -} - -/** - * Convert WP date format (YYYY-MM-DD HH:MM:SS) to ISO 8601. - * Returns empty string for zero dates. - */ -function wpDateToIso(wpDate: string): string { - if (!wpDate || wpDate === '0000-00-00 00:00:00') return ''; - // wpDate is "YYYY-MM-DD HH:MM:SS" in UTC - const date = new Date(wpDate.replace(' ', 'T') + 'Z'); - if (isNaN(date.getTime())) return ''; - return date.toISOString(); -} - -/** Elements that should always be parsed as arrays even when only one is present. */ -const arrayElements = new Set([ - 'item', - 'wp:author', - 'wp:category', - 'wp:tag', - 'wp:term', - 'wp:comment', - 'wp:postmeta', - 'category', -]); - -function ensureArray(val: T | T[] | undefined | null): T[] { - if (val == null) return []; - return Array.isArray(val) ? val : [val]; -} - -/** - * Extract a string from a parsed XML node. - * Handles plain strings, numbers, and objects with __cdata or #text properties. - */ -function str(node: unknown): string { - if (node == null) return ''; - if (typeof node === 'string') return node; - if (typeof node === 'number') return String(node); - if (typeof node === 'object' && node !== null) { - const obj = node as Record; - if ('__cdata' in obj) return String(obj['__cdata'] ?? ''); - if ('#text' in obj) return String(obj['#text']); - } - return String(node); -} - -function numOf(node: unknown): number { - const t = str(node); - const n = parseInt(t, 10); - return isNaN(n) ? 0 : n; -} - -/** - * Read a WXR file and parse it into structured typed objects. - */ -export function readWxr(wxrPath: string): WxrData { - const xml = readFileSync(wxrPath, 'utf8'); - - const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - textNodeName: '#text', - cdataPropName: '__cdata', - processEntities: true, - trimValues: true, - isArray: (name) => arrayElements.has(name), - }); - - const doc = parser.parse(xml); - const channel = doc.rss?.channel; - if (!channel) { - throw new Error('Invalid WXR: missing rss > channel'); - } - - const site = parseSiteMeta(channel); - const authors = parseAuthors(channel); - const categories = parseCategories(channel); - const tags = parseTags(channel); - const terms = parseTerms(channel); - - const items: WxrItem[] = []; - const comments: Comment[] = []; - - for (const rawItem of ensureArray(channel.item) as Record[]) { - const postType = str(rawItem['wp:post_type']); - - switch (postType) { - case 'attachment': - items.push(parseMediaItem(rawItem)); - break; - case 'page': - items.push(parsePageItem(rawItem)); - break; - case 'post': - items.push(parsePostItem(rawItem)); - break; - case 'nav_menu_item': - items.push(parseMenuItem(rawItem)); - break; - } - - // Parse comments within this item - for (const rawComment of ensureArray(rawItem['wp:comment']) as Record[]) { - comments.push(parseComment(rawComment, numOf(rawItem['wp:post_id']))); - } - } - - // Load redirects from sibling file - const redirects = loadRedirects(wxrPath); - - return { site, authors, categories, tags, terms, items, comments, redirects }; -} - -function parseSiteMeta(channel: Record): SiteMeta { - return { - title: str(channel.title), - url: str(channel['wp:base_blog_url']) || str(channel.link), - description: str(channel.description), - language: str(channel.language), - }; -} - -function parseAuthors(channel: Record): Author[] { - return (ensureArray(channel['wp:author']) as Record[]).map((a) => ({ - id: numOf(a['wp:author_id']), - login: str(a['wp:author_login']), - email: str(a['wp:author_email']), - displayName: str(a['wp:author_display_name']), - firstName: str(a['wp:author_first_name']), - lastName: str(a['wp:author_last_name']), - })); -} - -function parseCategories(channel: Record): Category[] { - return (ensureArray(channel['wp:category']) as Record[]).map((c) => ({ - id: numOf(c['wp:term_id']), - slug: str(c['wp:category_nicename']), - name: str(c['wp:cat_name']), - parent: str(c['wp:category_parent']), - description: str(c['wp:category_description']), - })); -} - -function parseTags(channel: Record): Tag[] { - return (ensureArray(channel['wp:tag']) as Record[]).map((t) => ({ - id: numOf(t['wp:term_id']), - slug: str(t['wp:tag_slug']), - name: str(t['wp:tag_name']), - description: str(t['wp:tag_description']), - })); -} - -function parseTerms(channel: Record): Term[] { - return (ensureArray(channel['wp:term']) as Record[]).map((t) => ({ - id: numOf(t['wp:term_id']), - taxonomy: str(t['wp:term_taxonomy']), - slug: str(t['wp:term_slug']), - name: str(t['wp:term_name']), - parent: str(t['wp:term_parent']), - description: str(t['wp:term_description']), - })); -} - -function getPostmeta(item: Record): Map { - const map = new Map(); - for (const meta of ensureArray(item['wp:postmeta']) as Record[]) { - const key = str(meta['wp:meta_key']); - const value = str(meta['wp:meta_value']); - if (key) map.set(key, value); - } - return map; -} - -function getItemCategories(item: Record): { - categories: string[]; - tags: string[]; - customTerms: Array<{ taxonomy: string; slug: string }>; -} { - const categories: string[] = []; - const tags: string[] = []; - const customTerms: Array<{ taxonomy: string; slug: string }> = []; - - for (const cat of ensureArray(item.category) as Record[]) { - const domain = String(cat['@_domain'] || ''); - const nicename = String(cat['@_nicename'] || ''); - - if (domain === 'category') { - categories.push(nicename); - } else if (domain === 'post_tag') { - tags.push(nicename); - } else if (domain && nicename) { - customTerms.push({ taxonomy: domain, slug: nicename }); - } - } - - return { categories, tags, customTerms }; -} - -function parseMediaItem(item: Record): MediaItem { - const meta = getPostmeta(item); - return { - id: numOf(item['wp:post_id']), - type: 'attachment', - title: str(item.title), - slug: str(item['wp:post_name']), - url: str(item['wp:attachment_url']), - altText: meta.get('_wp_attachment_image_alt') || '', - // Caption is read here but doesn't round-trip: WxrBuilder doesn't - // serialize caption for attachment items. - caption: str(item['excerpt:encoded']), - }; -} - -function parsePageItem(item: Record): PageItem { - const meta = getPostmeta(item); - const wpDate = str(item['wp:post_date']); - return { - id: numOf(item['wp:post_id']), - type: 'page', - title: str(item.title), - slug: str(item['wp:post_name']), - content: str(item['content:encoded']), - excerpt: str(item['excerpt:encoded']), - date: wpDateToIso(wpDate), - parent: numOf(item['wp:post_parent']), - menuOrder: numOf(item['wp:menu_order']), - author: str(item['dc:creator']), - seoTitle: meta.get('_seo_title') || '', - seoDescription: meta.get('_seo_description') || '', - sourceUrl: meta.get('_source_url') || '', - }; -} - -function parsePostItem(item: Record): PostItem { - const meta = getPostmeta(item); - const wpDate = str(item['wp:post_date']); - const { categories, tags, customTerms } = getItemCategories(item); - - return { - id: numOf(item['wp:post_id']), - type: 'post', - title: str(item.title), - slug: str(item['wp:post_name']), - content: str(item['content:encoded']), - excerpt: str(item['excerpt:encoded']), - date: wpDateToIso(wpDate), - categories, - tags, - featuredMediaId: parseInt(meta.get('_thumbnail_id') || '0', 10) || 0, - author: str(item['dc:creator']), - seoTitle: meta.get('_seo_title') || '', - seoDescription: meta.get('_seo_description') || '', - sourceUrl: meta.get('_source_url') || '', - customTerms, - }; -} - -function parseMenuItem(item: Record): MenuItem { - const meta = getPostmeta(item); - return { - id: numOf(item['wp:post_id']), - type: 'nav_menu_item', - title: str(item.title), - slug: str(item['wp:post_name']), - url: meta.get('_menu_item_url') || '', - menuSlug: meta.get('_menu_slug') || '', - parent: numOf(item['wp:post_parent']), - menuOrder: numOf(item['wp:menu_order']), - }; -} - -function parseComment(raw: Record, postId: number): Comment { - const wpDate = str(raw['wp:comment_date']); - return { - id: numOf(raw['wp:comment_id']), - postId, - author: str(raw['wp:comment_author']), - authorEmail: str(raw['wp:comment_author_email']), - authorUrl: str(raw['wp:comment_author_url']), - authorIp: str(raw['wp:comment_author_IP']), - date: wpDateToIso(wpDate), - content: str(raw['wp:comment_content']), - approved: str(raw['wp:comment_approved']), - type: str(raw['wp:comment_type']), - parent: numOf(raw['wp:comment_parent']), - userId: numOf(raw['wp:comment_user_id']), - }; -} - -function loadRedirects(wxrPath: string): Redirect[] { - const redirectPath = join(dirname(wxrPath), 'redirect-map.json'); - if (!existsSync(redirectPath)) return []; - try { - return JSON.parse(readFileSync(redirectPath, 'utf8')); - } catch { - return []; - } -} diff --git a/packages/data-liberation-agent/src/lib/wxr/wxr-rehydrate.test.ts b/packages/data-liberation-agent/src/lib/wxr/wxr-rehydrate.test.ts deleted file mode 100644 index 2b8ecc0438..0000000000 --- a/packages/data-liberation-agent/src/lib/wxr/wxr-rehydrate.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { WxrBuilder } from './wxr-builder.js'; -import { rehydrateBuilderFromWxr } from './wxr-rehydrate.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function tmpWxr(): string { - return join(mkdtempSync(join(FIXTURE_TMP, 'rh-')), 'output.wxr'); -} - -const SITE = 'https://example.com'; - -describe('rehydrateBuilderFromWxr', () => { - it('merges prior pages, drops nav_menu_items, and reseeds _nextId past the max id', () => { - const seed = new WxrBuilder({ title: 'Example', url: SITE, language: 'en-US' }); - seed.addPage({ title: 'Home', slug: 'home', content: '

h

', sourceUrl: SITE }); - seed.addPage({ title: 'About', slug: 'about', content: '

a

', sourceUrl: `${SITE}/about` }); - seed.addMenuItem({ title: 'Home', url: SITE, menuSlug: 'primary' }); - const wxrPath = tmpWxr(); - seed.serialize(wxrPath); - - const fresh = new WxrBuilder({ title: 'Example', url: SITE, language: 'en-US' }); - const merged = rehydrateBuilderFromWxr(fresh, wxrPath); - - expect(merged).toBe(true); - // nav_menu_items are intentionally dropped (regenerated each run). - expect(fresh.items.filter((i) => i.type === 'nav_menu_item')).toHaveLength(0); - expect(fresh.items.filter((i) => i.type === 'page').map((p) => p.slug).sort()).toEqual(['about', 'home']); - - // _nextId is reseeded past the largest *retained* id so newly added items - // never collide with rehydrated ones. - const maxRetainedId = Math.max(...fresh.items.map((i) => i.id)); - expect(fresh._nextId).toBeGreaterThan(maxRetainedId); - fresh.addPage({ title: 'New', slug: 'new', content: '', sourceUrl: `${SITE}/new` }); - const ids = fresh.items.map((i) => i.id); - expect(new Set(ids).size).toBe(ids.length); - }); - - it('is a no-op returning false when there is no prior WXR', () => { - const fresh = new WxrBuilder({ title: 'Example', url: SITE, language: 'en-US' }); - const merged = rehydrateBuilderFromWxr(fresh, join(FIXTURE_TMP, 'does-not-exist', 'output.wxr')); - expect(merged).toBe(false); - expect(fresh.items).toHaveLength(0); - }); - - it('treats a corrupt prior WXR as a fresh start (returns false, builder untouched)', () => { - const wxrPath = tmpWxr(); - writeFileSync(wxrPath, '<<< not valid xml at all', 'utf8'); - const fresh = new WxrBuilder({ title: 'Example', url: SITE, language: 'en-US' }); - fresh.addPage({ title: 'Existing', slug: 'existing', content: '', sourceUrl: SITE }); - const before = fresh.items.length; - - const merged = rehydrateBuilderFromWxr(fresh, wxrPath); - - // Corrupt prior must not throw and must not wipe what the builder already holds. - expect(merged).toBe(false); - expect(fresh.items).toHaveLength(before); - }); -}); diff --git a/packages/data-liberation-agent/src/lib/wxr/wxr-rehydrate.ts b/packages/data-liberation-agent/src/lib/wxr/wxr-rehydrate.ts deleted file mode 100644 index c86504e7d0..0000000000 --- a/packages/data-liberation-agent/src/lib/wxr/wxr-rehydrate.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { existsSync } from 'node:fs'; -import { readWxr } from './wxr-reader.js'; -import type { WxrBuilder } from './wxr-builder.js'; - -/** - * Rehydrate a fresh WxrBuilder from an existing WXR file so a subsequent - * serialize() preserves prior items instead of overwriting them with only the - * items extracted in the current run. - * - * Used by the resume path of liberate_extract and by every liberate_extract_one - * call (which appends a single URL to an existing extraction). Without this, both - * handlers serialize a builder holding only the current run's items, silently - * truncating the WXR — see DISCOVERIES.md (2026-04-30). - * - * nav_menu_items are dropped because the extraction loop regenerates them - * deterministically from the current inventory's navigation each run; keeping the - * prior ones would duplicate them. _nextId is reseeded past the largest existing - * id so newly added items never collide with rehydrated ones. - * - * A missing prior WXR is a no-op; a corrupt/unreadable one is treated as a fresh - * start (the builder is left untouched). - * - * @returns true if prior items were merged, false if there was nothing to merge. - */ -export function rehydrateBuilderFromWxr(wxr: WxrBuilder, wxrPath: string): boolean { - if (!existsSync(wxrPath)) return false; - try { - const prior = readWxr(wxrPath); - wxr.authors = prior.authors; - wxr.categories = prior.categories; - wxr.tags = prior.tags; - wxr.terms = prior.terms; - wxr.comments = prior.comments; - wxr.redirects = prior.redirects; - wxr.items = prior.items.filter((item) => item.type !== 'nav_menu_item'); - - let maxId = 0; - for (const item of wxr.items) maxId = Math.max(maxId, item.id); - for (const author of wxr.authors) maxId = Math.max(maxId, author.id); - for (const category of wxr.categories) maxId = Math.max(maxId, category.id); - for (const tag of wxr.tags) maxId = Math.max(maxId, tag.id); - for (const term of wxr.terms) maxId = Math.max(maxId, term.id); - for (const comment of wxr.comments) maxId = Math.max(maxId, comment.id); - wxr._nextId = maxId + 1; - return true; - } catch { - // Corrupt prior WXR: fall through and treat this as a fresh run. - return false; - } -} diff --git a/packages/data-liberation-agent/src/mcp-server.boot.test.ts b/packages/data-liberation-agent/src/mcp-server.boot.test.ts index b1d2538293..3b63b92fa0 100644 --- a/packages/data-liberation-agent/src/mcp-server.boot.test.ts +++ b/packages/data-liberation-agent/src/mcp-server.boot.test.ts @@ -1,98 +1,66 @@ -import { describe, it, expect } from 'vitest'; import { spawn } from 'node:child_process'; -import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; -// Boots the MCP server EXACTLY as production does — `tsx src/mcp-server.ts` in a -// child process, with cwd = repo root — and asserts it links and serves its tools. -// -// Why a subprocess and not an in-process import: the failure mode this guards -// against is a barrel re-exporting a TYPE with value `export {}` syntax (e.g. -// `export { DetectionResult } from './detect-platform.js'`). That crashes Node's -// native ESM linker at boot (which is how `tsx` runs the server) with -// "does not provide an export named ...". It is invisible to `tsc` (even with -// noEmitOnError) and silently erased by vitest's own esbuild transform — so a -// same-process `import('./mcp-server.js')` would NOT reproduce it. Only a real -// `tsx` launch links the barrel graph the way production does. Regression guard -// for the four barrels fixed alongside this test (detect-platform, woo-csv, -// resume-state, wxr) and any future barrel that makes the same mistake. -// -// Self-locating via import.meta.url so a worktree copy boots its own server from -// its own path (no shared cwd state, no cross-copy collision). +const SERVER = fileURLToPath( new URL( './mcp-server.ts', import.meta.url ) ); -const here = dirname(fileURLToPath(import.meta.url)); -const serverEntry = join(here, 'mcp-server.ts'); -const repoRoot = join(here, '..'); +/** + * Boot the server over stdio and ask what it offers. + * + * The surface is the product's three verbs. This asserts the count as well as + * the names: the previous server grew to thirty-nine tools by exposing every + * internal pipeline phase, and nothing failed when it did. + */ +function listTools(): Promise< Array< { name: string } > > { + return new Promise( ( resolve, reject ) => { + const child = spawn( 'npx', [ 'tsx', SERVER ], { stdio: [ 'pipe', 'pipe', 'pipe' ] } ); + let buffer = ''; + const timer = setTimeout( () => { + child.kill(); + reject( new Error( `MCP server did not answer. stdout: ${ buffer }` ) ); + }, 60_000 ); -interface Tool { - name: string; -} - -describe('mcp-server boots under tsx/native-ESM', () => { - it('links the barrel graph and serves the tool list', async () => { - const child = spawn('npx', ['tsx', serverEntry], { - cwd: repoRoot, - stdio: ['pipe', 'pipe', 'pipe'], - }); - - let stdout = ''; - let stderr = ''; - child.stderr.on('data', (d) => { - stderr += d.toString(); - }); + child.stdout.on( 'data', ( chunk ) => { + buffer += String( chunk ); + for ( const line of buffer.split( '\n' ) ) { + if ( ! line.trim().startsWith( '{' ) ) continue; + try { + const message = JSON.parse( line ) as { id?: number; result?: { tools?: Array< { name: string } > } }; + if ( message.id === 2 && message.result?.tools ) { + clearTimeout( timer ); + child.kill(); + resolve( message.result.tools ); + return; + } + } catch { + /* partial line; wait for more */ + } + } + } ); + child.on( 'error', reject ); - const send = (msg: unknown) => child.stdin.write(`${JSON.stringify(msg)}\n`); - send({ - jsonrpc: '2.0', - id: 1, - method: 'initialize', - params: { - protocolVersion: '2024-11-05', - capabilities: {}, - clientInfo: { name: 'boot-smoke', version: '0' }, - }, - }); - send({ jsonrpc: '2.0', method: 'notifications/initialized' }); - send({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }); - - try { - const tools = await new Promise((resolve, reject) => { - const timer = setTimeout( - () => reject(new Error(`timed out waiting for tools/list\nstderr:\n${stderr}`)), - 60_000, - ); - child.stdout.on('data', (d) => { - stdout += d.toString(); - for (const line of stdout.split('\n')) { - if (!line.trim()) continue; - try { - const msg = JSON.parse(line); - if (msg.id === 2 && msg.result?.tools) { - clearTimeout(timer); - resolve(msg.result.tools as Tool[]); - } - } catch { - // partial line — wait for the rest - } - } - }); - child.on('exit', (code) => { - clearTimeout(timer); - reject(new Error(`server exited (code ${code}) before tools/list\nstderr:\n${stderr}`)); - }); - }); + child.stdin.write( + `${ JSON.stringify( { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'test', version: '1' }, + }, + } ) }\n` + ); + child.stdin.write( + `${ JSON.stringify( { jsonrpc: '2.0', method: 'notifications/initialized' } ) }\n` + ); + child.stdin.write( `${ JSON.stringify( { jsonrpc: '2.0', id: 2, method: 'tools/list' } ) }\n` ); + } ); +} - // No ESM-linker crash leaked to stderr (the original boot failure). - expect(stderr).not.toMatch(/SyntaxError|does not provide an export/); - // Tools actually registered — a real list, with stable core tools present. - const names = tools.map((t) => t.name); - expect(names.length).toBeGreaterThan(0); - expect(names).toContain('liberate_paths'); - expect(names).toContain('liberate_detect'); - expect(names).toContain('liberate_extract'); - } finally { - child.stdin.end(); - child.kill(); - } - }, 70_000); -}); +describe( 'mcp server', () => { + it( 'boots and offers the product verbs, and only those', async () => { + const tools = await listTools(); + expect( tools.map( ( tool ) => tool.name ).sort() ).toEqual( [ 'compare', 'liberate', 'publish' ] ); + }, 90_000 ); +} ); diff --git a/packages/data-liberation-agent/src/mcp-server.schema.test.ts b/packages/data-liberation-agent/src/mcp-server.schema.test.ts deleted file mode 100644 index b87eef4126..0000000000 --- a/packages/data-liberation-agent/src/mcp-server.schema.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { readFileSync } from 'node:fs'; -import { join, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -// Read mcp-server.ts as TEXT (not import — importing starts the stdio server) and slice -// each tool's definition block so assertions are scoped to the right tool. -const here = dirname(fileURLToPath(import.meta.url)); -const SERVER = readFileSync(join(here, 'mcp-server.ts'), 'utf8'); - -function toolBlock(name: string): string { - const start = SERVER.indexOf(`name: '${name}'`); - expect(start, `tool ${name} not found in mcp-server.ts`).toBeGreaterThan(-1); - const next = SERVER.indexOf("name: 'liberate_", start + 1); - return SERVER.slice(start, next === -1 ? undefined : next); -} - -// The carry handler reads p.htmlSlug + p.postType and args.islandsOutDir; the server -// passes args RAW (no validation), so a schema that omits them "works" but lies — and -// without htmlSlug every page silently live-fetches (html/.html ≠ the namespaced -// capture filename). Keep the advertised schema honest. (Drift caught 2026-06-04.) -describe('liberate_reconstruct_pages_carry schema declares the handler contract', () => { - const block = toolBlock('liberate_reconstruct_pages_carry'); - it.each(['htmlSlug', 'postType', 'islandsOutDir'])('declares %s', (field) => { - expect(block).toContain(field); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server.ts b/packages/data-liberation-agent/src/mcp-server.ts index c6b5676d26..b849bd0585 100644 --- a/packages/data-liberation-agent/src/mcp-server.ts +++ b/packages/data-liberation-agent/src/mcp-server.ts @@ -1,795 +1,142 @@ // src/mcp-server.ts // -// Thin router. Tool listing lives below; per-tool logic lives in -// src/mcp-server/handlers/.ts. The dispatch map at the bottom maps -// tool names to handler modules. +// MCP as a transport, not an architecture. +// +// The product is three verbs, so this exposes three tools that call the same +// entry points the CLI calls. It deliberately does not expose pipeline phases: +// a caller that has to drive discovery, capture, and export in sequence is +// reimplementing the CLI, and the surface then has to be maintained against +// every internal change. // import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; -import { - CallToolRequestSchema, - ListToolsRequestSchema, -} from '@modelcontextprotocol/sdk/types.js'; -import type { PlatformAdapter } from './types.js'; - -import type { Handler, HandlerContext, ToolResult } from './mcp-server/handler-types.js'; -import { detectHandler } from './mcp-server/handlers/detect.js'; -import { discoverHandler } from './mcp-server/handlers/discover.js'; -import { inspectHandler } from './mcp-server/handlers/inspect.js'; -import { extractHandler } from './mcp-server/handlers/extract.js'; -import { extractOneHandler } from './mcp-server/handlers/extract-one.js'; -import { mediaInstallHandler } from './mcp-server/handlers/media-install.js'; -import { replicateTickHandler } from './mcp-server/handlers/replicate-tick.js'; -import { blockTransformApplyHandler } from './mcp-server/handlers/block-transform-apply.js'; -import { blockComposeHandler } from './mcp-server/handlers/block-compose.js'; -import { qaHandler } from './mcp-server/handlers/qa.js'; -import { mapApisHandler } from './mcp-server/handlers/map-apis.js'; -import { probeHandler } from './mcp-server/handlers/probe.js'; -import { verifyHandler } from './mcp-server/handlers/verify.js'; -import { setupHandler } from './mcp-server/handlers/setup.js'; -import { wpImportHandler } from './mcp-server/handlers/wp-import.js'; -import { statusHandler } from './mcp-server/handlers/status.js'; -import { pathsHandler } from './mcp-server/handlers/paths.js'; -import { previewHandler } from './mcp-server/handlers/preview.js'; -import { installThemeHandler } from './mcp-server/handlers/install-theme.js'; -import { themeScaffoldHandler } from './mcp-server/handlers/theme-scaffold.js'; -import { reconstructPagesHandler } from './mcp-server/handlers/reconstruct-pages.js'; -import { reconstructPagesCarryHandler } from './mcp-server/handlers/reconstruct-pages-carry.js'; -import { blockifyWxrHandler } from './mcp-server/handlers/blockify-wxr.js'; -import { screenshotHandler } from './mcp-server/handlers/screenshot.js'; -import { dataModelScaffoldHandler } from './mcp-server/handlers/data-model-scaffold.js'; -import { designFoundationScaffoldHandler } from './mcp-server/handlers/design-foundation-scaffold.js'; -import { designFoundationValidateHandler } from './mcp-server/handlers/design-foundation-validate.js'; -import { designFoundationSaveHandler } from './mcp-server/handlers/design-foundation-save.js'; -import { replicateInventoryHandler } from './mcp-server/handlers/replicate-inventory.js'; -import { replicateVerifyHandler } from './mcp-server/handlers/replicate-verify.js'; -import { compareHandler } from './mcp-server/handlers/compare.js'; -import { clusterPagesHandler } from './mcp-server/handlers/cluster-pages.js'; -import { sectionExtractHandler } from './mcp-server/handlers/section-extract.js'; -import { composeInstantiateHandler } from './mcp-server/handlers/compose-instantiate.js'; -import { ingestLocalSiteHandler } from './mcp-server/handlers/ingest-local-site.js'; -import { convertLocalSiteHandler } from './mcp-server/handlers/convert-local-site.js'; -import { validateArtifactsHandler } from './mcp-server/handlers/validate-artifacts.js'; -import { refineReportHandler } from './mcp-server/handlers/refine-report.js'; -import { NEW_TOOL_SCHEMAS } from './mcp-server/handlers/tool-schemas.js'; - -// Static adapter imports — add new adapters here (alphabetical) -import { defaultAdapter } from './adapters/default/index.js'; -import { godaddyWmAdapter } from './adapters/godaddy-wm/index.js'; -import { hostingerAdapter } from './adapters/hostinger/index.js'; -import { hubspotAdapter } from './adapters/hubspot/index.js'; -import { shopifyAdapter } from './adapters/shopify/index.js'; -import { squarespaceAdapter } from './adapters/squarespace/index.js'; -import { webflowAdapter } from './adapters/webflow/index.js'; -import { weeblyAdapter } from './adapters/weebly/index.js'; -import { wixAdapter } from './adapters/wix/index.js'; -import { resolveAdapter } from './adapters/resolve-adapter.js'; -const adapters: PlatformAdapter[] = [defaultAdapter, godaddyWmAdapter, hostingerAdapter, hubspotAdapter, shopifyAdapter, squarespaceAdapter, webflowAdapter, weeblyAdapter, wixAdapter]; +import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import { resolveOutputBase } from './lib/paths.js'; -function findAdapter(platform: string): PlatformAdapter | null { - return resolveAdapter(adapters, platform); +/** + * MCP tool result envelope. The index signature is what keeps it assignable to + * the SDK's ServerResult union, which is declared with `[key: string]: unknown`. + */ +interface ToolResult { + content: Array<{ type: 'text'; text: string }>; + isError?: boolean; + [key: string]: unknown; } -function textResult(data: unknown): ToolResult { - return { content: [{ type: 'text' as const, text: JSON.stringify(data, null, 2) }] }; -} +const textResult = (data: unknown): ToolResult => ({ + content: [{ type: 'text', text: JSON.stringify(data, null, 2) }], +}); -function errorResult(message: string): ToolResult { - return { - content: [{ type: 'text' as const, text: JSON.stringify({ error: message }) }], - isError: true, - }; -} +const errorResult = (message: string): ToolResult => ({ + content: [{ type: 'text', text: JSON.stringify({ error: message }) }], + isError: true, +}); + +const TOOLS = [ + { + name: 'liberate', + description: + 'Liberate a website into a complete, portable HTML site. Returns the run directory, the website directory, and route counts.', + inputSchema: { + type: 'object', + properties: { + url: { type: 'string', description: 'Site to liberate.' }, + outputDir: { type: 'string', description: 'Output base directory. Defaults to ~/data-liberation.' }, + resume: { type: 'boolean', description: 'Reuse artifacts already on disk instead of recapturing.' }, + screenshots: { type: 'boolean', description: 'Also capture full-page and scrolled PNGs.' }, + }, + required: ['url'], + }, + }, + { + name: 'compare', + description: + 'Verify a liberated copy: self-consistency across every route, and source fidelity across a sample. Returns the report, including whether it passed.', + inputSchema: { + type: 'object', + properties: { + directory: { type: 'string', description: 'A liberated run directory.' }, + screenshots: { type: 'boolean', description: 'Write source/copy/diff PNGs as evidence.' }, + }, + required: ['directory'], + }, + }, + { + name: 'publish', + description: 'Publish a liberated site to a live URL. Returns the live URL and any claim link.', + inputSchema: { + type: 'object', + properties: { + directory: { type: 'string', description: 'A liberated run directory.' }, + target: { type: 'string', description: 'Publish target. Defaults to spacefast.' }, + token: { type: 'string', description: 'Token for the target, if publishing into an account.' }, + }, + required: ['directory'], + }, + }, +]; const server = new Server( - { name: 'data-liberation', version: '0.1.0' }, + { name: 'data-liberation', version: '1.0.0' }, { capabilities: { tools: {} } } ); -server.setRequestHandler(ListToolsRequestSchema, async () => ({ - tools: [ - { - name: 'liberate_detect', - description: 'Detect the platform of a website (GoDaddy Websites & Marketing, Hostinger, HubSpot, Shopify, Squarespace, Webflow, Weebly, Wix, or unknown)', - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'The URL of the website to detect' }, - }, - required: ['url'], - }, - }, - { - name: 'liberate_discover', - description: 'Inventory a website: fetch sitemap, categorize URLs, extract navigation structure', - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'The URL of the website to inventory' }, - token: { type: 'string', description: 'API token for platforms requiring auth' }, - cdpPort: { type: 'number', description: 'CDP port for browser-based extraction' }, - verbose: { type: 'boolean', description: 'Enable detailed logging' }, - }, - required: ['url'], - }, - }, - { - name: 'liberate_inspect', - description: "Probe a site to assess extractability: detect platform, check sitemap, probe sample pages", - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'The URL of the website to inspect' }, - token: { type: 'string', description: 'API token if needed' }, - cdpPort: { type: 'number', description: 'CDP port for browser-based inspection' }, - }, - required: ['url'], - }, - }, - { - name: 'liberate_extract', - description: 'Extract all content from a website. Produces WXR file + media directory + redirect map.', - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'The URL of the website to extract' }, - outputDir: { type: 'string', description: 'Directory to write WXR, media, and logs' }, - token: { type: 'string', description: 'API token for platforms requiring auth (e.g. Webflow)' }, - cdpPort: { type: 'number', description: 'CDP port for browser-based extraction' }, - adminToken: { type: 'string', description: 'Shopify Admin API access token. When set, products are fetched via the Shopify Admin GraphQL API for richer data (compareAtPrice, inventoryPolicy, unitCost, collections, SEO metafields, variant images). Falls back to the public JSON API on failure.' }, - shopDomain: { type: 'string', description: 'Shopify *.myshopify.com hostname. Usually auto-detected by liberate_discover from the storefront HTML; only pass explicitly if detection failed (e.g. Cloudflare-protected site).' }, - delay: { type: 'number', description: 'Delay between requests in ms (default: 500)' }, - resume: { type: 'boolean', description: 'Resume a previous extraction' }, - dryRun: { type: 'boolean', description: 'Extract 2-3 pages and report without writing WXR' }, - limit: { type: 'number', description: 'Cap extraction to the first N URLs and write a real WXR for them' }, - verbose: { type: 'boolean', description: 'Enable detailed per-page logging' }, - screenshots: { type: 'boolean', description: 'After extract completes, capture screenshots (desktop + mobile) for every processed URL. Results are written to output//screenshots/ with a manifest.json keyed by URL.' }, - captureDesign: { type: 'boolean', description: 'Enable html-first design replication: carry source HTML+CSS as the page/post design. Note: full html-first design capture (site.css aggregation, blank theme install) runs via the CLI (`data-liberation --html-first`); this flag is reserved for future MCP support.' }, - contentStatus: { type: 'string', enum: ['draft', 'publish'], description: 'WXR post status for extracted pages/posts. Default "draft" — the documented "import as drafts; the user reviews and publishes manually" convention for a production import. The replica/preview flow (e.g. building a Studio replica) passes "publish" so imported nav targets resolve. Attachments always use "inherit".' }, - }, - required: ['url', 'outputDir'], - }, - }, - { - name: 'liberate_extract_one', - description: 'Extract a single URL through the streaming pipeline. Used by the watch loop and agent-driven streaming. Each call runs adapter discovery to set up state, then narrows to the target URL. Append-mode WXR — results accumulate in output.wxr across calls.', - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'The single URL to extract.' }, - outputDir: { type: 'string', description: 'Liberation output directory. WXR + media + logs are appended here.' }, - siteUrl: { type: 'string', description: 'Origin of the source site, used for adapter discovery. Defaults to the origin parsed from `url`.' }, - token: { type: 'string', description: 'API token for platforms requiring auth (e.g. Webflow).' }, - cdpPort: { type: 'number', description: 'CDP port for browser-based extraction.' }, - adminToken: { type: 'string', description: 'Shopify Admin API token (see liberate_extract).' }, - shopDomain: { type: 'string', description: 'Shopify *.myshopify.com hostname (see liberate_extract).' }, - delay: { type: 'number', description: 'Delay floor in ms.' }, - verbose: { type: 'boolean', description: 'Per-step logging.' }, - contentStatus: { type: 'string', enum: ['draft', 'publish'], description: 'WXR post status for extracted pages/posts. Default "draft" (import-as-drafts convention); the replica/preview flow passes "publish". Attachments always use "inherit".' }, - }, - required: ['url', 'outputDir'], - }, - }, - { - name: 'liberate_paths', - description: 'Resolve where liberation output lives. Returns { base, siteDir }. base = the default output base (DLA_OUTPUT_DIR or /_liberations). siteDir = base/ when a url is given. Skills MUST use this instead of assuming output// relative to cwd.', - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'Optional source URL; when present, siteDir is returned.' }, - }, - }, - }, - { - name: 'liberate_status', - description: 'Check progress of a running or completed extraction', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'The output directory of the extraction' }, - }, - required: ['outputDir'], - }, - }, - { - name: 'liberate_map_apis', - description: 'Map all API endpoints used by a website by navigating pages via CDP and capturing JSON network traffic. Produces a categorized endpoint catalog with sample responses and auth headers. Use during /adapt reconnaissance to reverse-engineer a new platform.', - inputSchema: { - type: 'object' as const, - properties: { - cdpPort: { type: 'number', description: 'Chrome DevTools Protocol port (e.g. 9222)' }, - url: { type: 'string', description: 'The URL of the site to map' }, - crawlUrls: { - type: 'array', - items: { type: 'string' }, - description: 'Additional URLs to navigate (e.g. admin dashboard sections)', - }, - followLinks: { type: 'boolean', description: 'Follow same-origin links from the main page (up to 20, default: false)' }, - }, - required: ['cdpPort', 'url'], - }, - }, - { - name: 'liberate_probe', - description: 'Probe a browser page via CDP for extraction-relevant data: window globals, JSON-LD, cookies, localStorage, network entries, and platform identity fields. Requires a running Chrome with --remote-debugging-port. Use for debugging extraction failures.', - inputSchema: { - type: 'object' as const, - properties: { - cdpPort: { type: 'number', description: 'Chrome DevTools Protocol port (e.g. 9222)' }, - url: { type: 'string', description: 'Only probe pages on this domain (optional — probes all tabs if omitted)' }, - }, - required: ['cdpPort'], - }, - }, - { - name: 'liberate_qa', - description: 'Compare extracted WXR content against the original source site page by page. Reports text similarity, missing headings/images/links, and grades each page (pass/warn/fail). Optionally patches fixable issues like missing alt text.', - inputSchema: { - type: 'object' as const, - properties: { - wxrFile: { type: 'string', description: 'Path to the WXR file to QA' }, - fix: { type: 'boolean', description: 'Patch fixable issues in the WXR (default: false)' }, - }, - required: ['wxrFile'], - }, - }, - { - name: 'liberate_verify', - description: 'Verify a completed extraction: check for stale CDN URLs, failed pages, missing media, and items needing manual attention', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'The output directory of the extraction to verify' }, - }, - required: ['outputDir'], - }, - }, - { - name: 'liberate_setup', - description: 'Validate WordPress connection: check site reachability, REST API, and authentication. Returns guidance if anything fails. Pass delegate: true to skip validation and receive a structured manifest describing what the import target needs — useful when the calling environment handles site setup itself.', - inputSchema: { - type: 'object' as const, - properties: { - site: { type: 'string', description: 'WordPress site domain (e.g. mysite.com or localhost:8881)' }, - username: { type: 'string', description: 'WordPress username' }, - token: { type: 'string', description: 'WordPress application password' }, - delegate: { type: 'boolean', description: 'Skip validation and return a setup manifest for the calling environment to handle. Use when the environment has its own site management (e.g. local dev tools).' }, - }, - required: [], - }, - }, - { - name: 'liberate_import', - description: 'Import a WXR file into a WordPress site. Pass delegate: true to skip REST import and receive a structured import manifest — useful when the calling environment handles imports itself (e.g. local dev tools with direct database/CLI access).', - inputSchema: { - type: 'object' as const, - properties: { - wxrFile: { type: 'string', description: 'Path to the WXR file to import' }, - site: { type: 'string', description: 'WordPress site domain (e.g. example.com)' }, - username: { type: 'string', description: 'WordPress username' }, - token: { type: 'string', description: 'WordPress application password' }, - dryRun: { type: 'boolean', description: 'Preview without importing' }, - delay: { type: 'number', description: 'Delay between requests in ms (default: 500)' }, - only: { type: 'string', description: 'Only import specific type (categories, tags, media, pages, posts, comments, menus)' }, - verbose: { type: 'boolean', description: 'Enable detailed logging' }, - resume: { type: 'boolean', description: '(deprecated — import is always idempotent, this flag has no effect)' }, - importAuthors: { type: 'boolean', description: 'Create WordPress users for each author in the WXR (default: false — all content owned by authenticated user)' }, - woocommerceKey: { type: 'string', description: 'WooCommerce consumer key for product import' }, - woocommerceSecret: { type: 'string', description: 'WooCommerce consumer secret for product import' }, - delegate: { type: 'boolean', description: 'Skip REST import and return a structured import manifest for the calling environment to handle.' }, - }, - required: ['wxrFile'], - }, - }, - { - name: 'liberate_preview', - description: 'Spawn a local Studio preview of an extraction output. Returns { url, port, status, warnings }. Kills any existing preview on the same outputDir before starting. Optionally installs a generated replica theme + block plugins via themeFiles[] + blockPlugins[]; the theme is activated after content import. Used by the replicate skill in Step 5 (Install).', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Path to the extraction output directory (contains output.wxr).' }, - open: { type: 'boolean', description: 'If true, open the URL in the default browser after readiness.' }, - port: { type: 'number', description: 'Override the auto-picked port (default range: 9400-9499).' }, - themeFiles: { - type: 'array', - description: 'Generated replica theme files. Each entry is { relativePath, content } rooted at the theme directory (e.g. relativePath: "templates/index.html"). Theme is written to wp-content/themes// and activated after content import.', - items: { - type: 'object' as const, - properties: { - relativePath: { type: 'string' }, - content: { type: 'string' }, - }, - required: ['relativePath', 'content'], - }, - }, - blockPlugins: { - type: 'array', - description: 'DEPRECATED — embed custom blocks inside the theme at blocks//{src,build}/ via themeFiles[] instead (Telex blocks-inside-themes pattern, registered from functions.php). Kept for backwards compatibility. Each entry is { slug, files: [{relativePath, content}] }; plugin is written to wp-content/plugins// and activated.', - items: { - type: 'object' as const, - properties: { - slug: { type: 'string' }, - files: { - type: 'array', - items: { - type: 'object' as const, - properties: { - relativePath: { type: 'string' }, - content: { type: 'string' }, - }, - required: ['relativePath', 'content'], - }, - }, - }, - required: ['slug', 'files'], - }, - }, - themeSlug: { type: 'string', description: 'Theme directory name (kebab-case). Required when themeFiles is non-empty. Conventionally -replica.' }, - }, - required: ['outputDir'], - }, - }, - { - name: 'liberate_install_theme', - description: 'Install replica theme files + block plugins into an ALREADY-RUNNING Studio site (no site creation, no content import). Use this from the streaming watch loop\'s theme-piece / archetype-template judgments — `liberate_preview` would create a `-2` duplicate Studio site and re-import content over the streamed posts. Writes to /wordpress/wp-content/{themes,plugins}/, then runs `studio wp plugin activate` and `studio wp theme activate`. Returns warnings[] for non-fatal activate failures.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Path to the extraction output directory (used for log routing only — files are written to studioSitePath, not outputDir).' }, - studioSitePath: { type: 'string', description: 'On-disk path to the running Studio site (parent dir, NOT the wordpress sub-dir). Streaming watch logs this as `preview-pre-started.sitePath`.' }, - themeFiles: { - type: 'array', - description: 'Replica theme files. Same shape as liberate_preview — { relativePath, content }, rooted at the theme directory. Activated after writing.', - items: { - type: 'object' as const, - properties: { - relativePath: { type: 'string' }, - content: { type: 'string' }, - }, - required: ['relativePath', 'content'], - }, - }, - blockPlugins: { - type: 'array', - description: 'DEPRECATED — kept for backwards compatibility. New replica work should embed custom blocks inside the theme at blocks//{src,build}/ via themeFiles[], following the Telex blocks-inside-themes pattern. The skill registers them from functions.php. Each plugin entry is { slug, files: [{relativePath, content}] }, activated after writing.', - items: { - type: 'object' as const, - properties: { - slug: { type: 'string' }, - files: { - type: 'array', - items: { - type: 'object' as const, - properties: { - relativePath: { type: 'string' }, - content: { type: 'string' }, - }, - required: ['relativePath', 'content'], - }, - }, - }, - required: ['slug', 'files'], - }, - }, - themeSlug: { type: 'string', description: 'Theme directory name (kebab-case). Required when themeFiles is non-empty. Conventionally -replica.' }, - }, - required: ['outputDir', 'studioSitePath'], - }, - }, - { - name: 'liberate_theme_scaffold', - description: 'Read /design-foundation.json and emit a complete-and-activatable WordPress block theme bundle deterministically: style.css (theme header), theme.json (settings/styles mapped from foundation tokens), functions.php (theme setup + custom-block registration loop), templates/index.html (homepage shell with header part + post-content + footer part), parts/header.html (site-title + page-list nav), parts/footer.html (copyright). No agent reasoning, no vision, no LLM call — pure deterministic mapping. Pair with `liberate_install_theme` to install the result into a running Studio site. Per-archetype templates (page.html, single.html, etc.) and patterns are NOT emitted here — they belong to the replicate skill\'s archetype-template tick.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (must contain design-foundation.json).' }, - themeSlug: { type: 'string', description: 'Theme directory slug (kebab-case). Conventionally -replica.' }, - themeName: { type: 'string', description: 'Display name. Defaults to themeSlug.' }, - siteTitle: { type: 'string', description: 'Source site title — used in style.css description and footer copyright.' }, - themeDescription: { type: 'string', description: 'Override the default style.css Description line.' }, - sourceUrl: { type: 'string', description: 'The source site origin (e.g. https://www.example.com/). Used to resolve the captured header/footer chrome links to absolute URLs so they remap to local permalinks (without it, nav hrefs are not remapped and point off-site). Defaults to a placeholder origin.' }, - persist: { type: 'boolean', description: 'When true, also write the emitted text theme files to /theme (alongside the font/logo assets always written there), materializing a complete on-disk theme. Default false: themeFiles[] is returned for the caller to install into a live site.' }, - reconstructedPages: { - type: 'array', - description: 'Block-reconstructed content pages. Each entry emits templates/page-.html wiring the page to its reconstructed pattern (and front-page.html for isHome), so the page renders block sections instead of falling through page.html to raw carried post_content. The pattern files (reconstructed block markup) are added to themeFiles[] separately by the replicate skill.', - items: { - type: 'object', - properties: { - slug: { type: 'string', description: 'Source-faithful WP page slug (last path segment), e.g. "about-us".' }, - patternSlug: { type: 'string', description: 'Fully-qualified theme pattern slug, e.g. "/page-about-us".' }, - isHome: { type: 'boolean', description: 'When true, also emit templates/front-page.html for this page (static front page).' }, - }, - required: ['slug', 'patternSlug'], - }, - }, - }, - required: ['outputDir', 'themeSlug'], - }, - }, - { - name: 'liberate_blockify_wxr', - description: 'BULK blog-body block conversion (blocks reconstruct path ONLY). Rewrites every post/page content:encoded body in output.wxr through the source platform adapter\'s block recipe (seam 2) so imported posts land as editable Gutenberg blocks instead of one Classic block (e.g. Squarespace sqs-block bodies). Lossless: bodies the recipe can\'t convert are left verbatim, and all other items (attachments, nav menu items, comments, terms) are preserved unchanged. No-op when the platform adapter has no block recipe. Resolves the platform from session.json (recorded at extraction); pass `platform` to override. Run AFTER extraction and BEFORE liberate_import. The theme/carry path must NOT call this.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory holding output.wxr + session.json.' }, - wxrPath: { type: 'string', description: 'Override the WXR path. Defaults to /output.wxr.' }, - platform: { type: 'string', description: 'Override the platform adapter id (else read from session.json).' }, - }, - required: ['outputDir'], - }, - }, - { - name: 'liberate_reconstruct_pages', - description: 'Deterministically reconstruct EVERY content page from its OWN captured section specs. For each page: capture specs, install section media, reconstruct verbatim block markup, GATE through validate_artifacts, write the pattern + reconstructed post_content. Page TEMPLATES are collapsed to a small set of variant templates (templates/page-replica[-].html) registered in theme.json customTemplates and assigned per page via _wp_page_template; output.wxr is patched to match. Set collapseTemplates:false to fall back to one templates/page-.html per page. The theme shell must already be installed via liberate_theme_scaffold/install.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (holds media/ + media-stubs.json).' }, - studioSitePath: { type: 'string', description: 'On-disk path to the running Studio site (e.g. ~/Studio/example-com).' }, - themeSlug: { type: 'string', description: 'Installed theme slug. Defaults to -replica derived from outputDir.' }, - collapseTemplates: { - type: 'boolean', - description: 'Collapse per-page templates into variant-keyed templates + _wp_page_template assignments (default true). false = one template per page (legacy).', - }, - variationHoist: { type: 'boolean', description: 'Hoist recurring instance-style constellations into theme block-style variations (default true). Set false to disable (escape hatch).' }, - editableIslands: { type: 'boolean', description: 'Convert the coverage-gated core/html fallback islands into in-canvas dla/editable-html blocks (visible + styled + text/image-editable in the block editor; ships + activates the block plugin). Default true. Set false to keep plain core/html.' }, - pages: { - type: 'array', - description: 'Content pages to reconstruct. Reconstruct every page (not just cluster reps).', - items: { - type: 'object', - properties: { - slug: { type: 'string', description: 'Source-faithful WP page slug (sanitize_title-shaped), e.g. "about-us".' }, - sourceUrl: { type: 'string', description: 'The page\'s source URL to capture + reconstruct.' }, - title: { type: 'string', description: 'Human-readable page title (pattern doc-comment).' }, - isHome: { type: 'boolean', description: 'When true, also emit front-page.html.' }, - }, - required: ['slug', 'sourceUrl', 'title'], - }, - }, - }, - required: ['outputDir', 'studioSitePath', 'pages'], - }, - }, - { - name: 'liberate_reconstruct_pages_carry', - description: 'Carry-and-scope parity path: for each page, load cached body HTML (or fetch live), collect CSS, carry the sanitized HTML + scoped CSS into core/html block islands, self-host the run media (rewriting /srcset/url() to the local WP library) + localize internal links, write a carry FSE block theme under wp-content/themes/-carry (incl. WooCommerce single-product/archive-product templates wrapping the carried header/footer when the run has products), and return per-page islands for building output-carry.wxr. Requires liberate_screenshot (html/ cache); falls back to live fetch. Pass islandsOutDir to write islands to disk and return paths instead of inline content (avoids the MCP response-size cap on large sites).', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (holds html/ cache from liberate_screenshot).' }, - studioSitePath: { type: 'string', description: 'On-disk path to the running Studio site (e.g. ~/Studio/example-com).' }, - themeName: { type: 'string', description: 'Display name for the carry theme (default: "Liberated (Carry)").' }, - islandsOutDir: { type: 'string', description: 'When set, write each carried island to /.html and return its path + byte count instead of inline postContent. Use this from MCP to avoid the response-size cap (islands are whole page bodies). Omit to get postContent inline (the tsx driver default).' }, - editableIslands: { type: 'boolean', description: 'Emit carried bodies as in-canvas dla/editable-html blocks (visible + styled + text/image-editable in the block editor) instead of sandboxed core/html islands. Front-end output is byte-identical (static save). Ships + activates the block plugin. Default true; set false to force plain core/html.' }, - pages: { - type: 'array', - description: 'Content pages to carry and scope. Pass every page in the site for full coverage.', - items: { - type: 'object', - properties: { - slug: { type: 'string', description: 'URL-safe page slug (sanitize_title-shaped), e.g. "about-us". Must match the WP post_name so the island-swap finds the post and functions.php body-class scoping targets it.' }, - sourceUrl: { type: 'string', description: 'The page\'s source URL (used as base for CSS resolution and live HTML fallback).' }, - title: { type: 'string', description: 'Human-readable page title.' }, - isHome: { type: 'boolean', description: 'When true, emits front-page.html template and uses is_front_page() body-class condition.' }, - postType: { type: 'string', enum: ['page', 'post'], description: 'Post type (default "page"). "post" scopes via is_single() and renders through single.html; also selects the functions.php body-class condition.' }, - htmlSlug: { type: 'string', description: 'Override the cached-HTML filename stem loaded as html/.html when it differs from the WP slug — e.g. posts captured as "blogs--snoozweek--" or pages as "pages--". Falls back to slug; without it the tool live-fetches sourceUrl.' }, - }, - required: ['slug', 'sourceUrl', 'title'], - }, - }, - }, - required: ['outputDir', 'studioSitePath', 'pages'], - }, - }, - { - name: 'liberate_screenshot', - description: 'Capture full-page + scrolled screenshots (desktop + mobile) and rendered HTML for every URL on a site. Writes to /screenshots/ and /html/, plus palette.json, typography.json, breakpoints.json, and computed-styles.json via DOM/CSS site-analysis. Reuses sitemap discovery or accepts explicit urls[].', - inputSchema: { - type: 'object' as const, - properties: { - url: { type: 'string', description: 'Site URL (used for sitemap discovery and same-origin enforcement)' }, - outputDir: { type: 'string', description: 'Output directory' }, - urls: { type: 'array', items: { type: 'string' }, description: 'Explicit URL list (skips sitemap fetch; all must share origin with `url` if both provided)' }, - types: { type: 'array', items: { type: 'string' }, description: 'Filter by URL type: page, post, product, homepage, gallery, event' }, - limit: { type: 'number', description: 'Cap to first N URLs' }, - concurrency: { type: 'number', description: 'Parallel URL captures (default 3, max 10)' }, - browserRestartEvery: { type: 'number', description: 'Close and relaunch browser every N URLs (default 100)' }, - cdpPort: { type: 'number', description: 'Connect to existing Chrome via CDP' }, - force: { type: 'boolean', description: 'Re-capture even if output files already exist' }, - verbose: { type: 'boolean', description: 'Per-URL progress logging' }, - }, - required: ['url', 'outputDir'], - }, - }, - { - name: 'liberate_data_model_scaffold', - description: - 'Deterministic pre-pass for the JS-data path: reads an owned local site dir, discovers record arrays / mount containers / id-lookups by AST (resilient to malformed/vendored JS files), infers field roles, and writes a PARTIAL data-model.draft.json. Returns { model, skillTodos, discovered, validation }. The model-local-data skill fills only the skillTodos (card.template, ambiguous ordering, low-confidence role guesses), then writes the final data-model.json. Run before liberate_convert_local_site when the source renders content from a JS data array.', - inputSchema: { - type: 'object' as const, - properties: { - dir: { type: 'string', description: 'Absolute path to the local static-site directory.' }, - outputDir: { type: 'string', description: 'Where data-model.draft.json is written. Defaults to dir.' }, - }, - required: ['dir'], - }, - }, - { - name: 'liberate_design_foundation_scaffold', - description: - 'Runs the deterministic scaffold on a liberation output directory: reads palette.json / typography.json / breakpoints.json / screenshots/manifest.json from SP1 output, applies pure rules (darkest high-frequency → text.default, lightest → surface.base, breakpoint tier mapping, gradient regex extraction from html/*.html), and returns a PartialDesignFoundation. Empty role slots are left for the design-foundations skill to assign. Emits skillTodos listing every path the skill must fill. The design-foundations skill may additionally read computed-styles.json for HTML/CSS role assignment.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (must contain SP1 files).' }, - origin: { type: 'string', description: 'Origin URL (e.g. https://example.com). Stored in the foundation `origin` field.' }, - }, - required: ['outputDir', 'origin'], - }, - }, - { - name: 'liberate_design_foundation_validate', - description: - 'Validates a design-foundation JSON blob against the schema. Returns { ok: true } or { ok: false, errors: [...] }. Used by the design-foundations skill after filling role slots to catch structural mistakes and unfilled skillTodos before saving to disk.', - inputSchema: { - type: 'object' as const, - properties: { - foundation: { type: 'object', description: 'JSON blob to validate (not a path).' }, - }, - required: ['foundation'], - }, - }, - { - name: 'liberate_design_foundation_save', - description: - 'Persists a validated design-foundation JSON to disk and generates the human-readable design-foundation.md companion. Writes both files atomically to outputDir. Skips write when inputsDigest matches prior file (unless force=true).', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Destination directory.' }, - foundation: { type: 'object', description: 'Complete design foundation JSON blob.' }, - force: { type: 'boolean', description: 'Overwrite even if inputsDigest matches prior file.' }, - }, - required: ['outputDir', 'foundation'], - }, - }, - { - name: 'liberate_media_install', - description: 'Install one URL\'s pending media into the running replica WP site. Idempotent: skips media already registered as attachments (tracked via MediaStubStore.wpPostId). Uses `studio wp eval-file` to run a vendored PHP installer script.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (contains media-stubs.json + media/).' }, - url: { type: 'string', description: 'Source URL whose media we are installing (used for logging; the install acts on all pending media in MediaStubStore).' }, - target: { - type: 'object' as const, - description: 'Where to install the media. Studio: { kind: "studio", sitePath: "/Users/.../Studio/site-name" }.', - properties: { - kind: { type: 'string', enum: ['studio'] }, - sitePath: { type: 'string' }, - siteUrl: { type: 'string', description: 'Optional site URL used to compute browser-visible upload URLs.' }, - }, - required: ['kind', 'sitePath'], - }, - }, - required: ['outputDir', 'url', 'target'], - }, - }, - { - name: 'liberate_replicate_tick', - description: 'Run one tick of the replicate streaming scheduler. Reads replicate-state.json, computes deltas (new archetypes since last tick, foundation drift), returns judgmentNeeded[] markers describing what skills the calling agent should invoke (replicate, design-foundations, compose-page-blocks). The MCP tool is deterministic — it does not invoke skills directly.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory.' }, - reason: { type: 'string', description: 'Optional reason override (manual / new-archetype / periodic / foundation-drift). Defaults to inferred.' }, - }, - required: ['outputDir'], - }, - }, - { - name: 'liberate_block_transform_apply', - description: 'Apply composed block markup to a post in the running replica site. Validates: parse_blocks roundtrip + output-verify text-substring check + post-existence poll (3 retries with backoff). Idempotent via block-transform-log.jsonl (same source+output hashes skip re-apply). Studio path uses `wp post update` via studio CLI.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (block-transform-log.jsonl lives here).' }, - url: { type: 'string', description: 'Source URL (post is matched via _source_url meta in the replica).' }, - blocks: { type: 'string', description: 'Composed block markup to apply as post_content.' }, - sourceHtml: { type: 'string', description: 'Original sanitized source HTML — passed to output-verify for text-substring validation.' }, - target: { - type: 'object' as const, - description: 'Replica site target. Studio: { kind: "studio", sitePath: "..." }.', - properties: { - kind: { type: 'string', enum: ['studio'] }, - sitePath: { type: 'string' }, - siteUrl: { type: 'string' }, - }, - required: ['kind'], - }, - composedBy: { type: 'string', description: 'Provenance string for the log entry (e.g. "compose-page-blocks@v1.0" or "heuristic@v1.0").' }, - }, - required: ['outputDir', 'url', 'blocks', 'sourceHtml', 'target'], - }, - }, - { - name: 'liberate_block_compose', - description: 'Validate composed block markup and write it to a sidecar file (/composed/.blocks.html) for the streaming watch loop to install as post_content. Compose-then-install counterpart to liberate_block_transform_apply: same parse_blocks roundtrip + output-verify validation, same block-transform-log.jsonl idempotency, but NO database write. The runner reads the sidecar after the agent returns and passes the contents to wp_insert_post via contentOverride, so the very first DB write of each post carries block markup (not raw HTML that gets transformed afterward). Use this in the streaming watch loop\'s compose-page-blocks judgment; reach for liberate_block_transform_apply only for re-composing already-imported posts.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (sidecar lives at /composed/, log is block-transform-log.jsonl).' }, - url: { type: 'string', description: 'Source URL — used for log entries and as the manifest lookup key when sourceHtml is omitted.' }, - slug: { type: 'string', description: 'Post slug — determines the sidecar filename (composed/.blocks.html). Must match the WxrItem.slug the runner buffered.' }, - blocks: { type: 'string', description: 'Composed block markup. Validated for parse_blocks roundtrip and against sourceHtml for text-substring containment.' }, - sourceHtml: { type: 'string', description: 'Sanitized source HTML used for anti-hallucination output-verify. Optional — falls back to /screenshots/manifest.json lookup.' }, - composedBy: { type: 'string', description: 'Provenance string for the log entry (default "compose-page-blocks@v1.0").' }, - source: { type: 'string', enum: ['heuristic', 'ai'], description: 'Compose source flavour for the log entry (default "ai").' }, - }, - required: ['outputDir', 'url', 'slug', 'blocks'], - }, - }, - { - name: 'liberate_replicate_inventory', - description: - 'Read a liberation outputDir and return a structured archetype inventory: counts per archetype (homepage/page/post/product/gallery/event), up to 3 representative URLs per archetype with their screenshot+html paths (selected by largest HTML — proxy for section count), product count from products.jsonl, and presence of design-foundation.json. Used by the replicate skill in Step 1 (Inventory). Throws when output.wxr is missing.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (must contain output.wxr).' }, - }, - required: ['outputDir'], - }, - }, - { - name: 'liberate_replicate_verify', - description: - 'Capture replica screenshots at given URLs (desktop + mobile by default) against a running replica WP install and pair each viewport with the matching source screenshot from screenshots/manifest.json. Returns a structured pairing manifest the calling agent (vision-capable) uses for side-by-side comparison. Used by the replicate skill in Step 6 (Verify). Replica screenshots are written to ///.png — same shape as the source layout.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (contains screenshots/manifest.json and the source screenshots/).' }, - replicaBaseUrl: { type: 'string', description: 'Base URL of the running replica (e.g. https://my-site-replica.wp.local or http://localhost:8881). No trailing slash.' }, - urls: { type: 'array', items: { type: 'string' }, description: 'Path-only URLs to verify (e.g. ["/", "/blog/post-1"]).' }, - viewports: { type: 'array', items: { type: 'string', enum: ['desktop', 'mobile'] }, description: 'Viewports to capture. Default: ["desktop", "mobile"].' }, - outputSubdir: { type: 'string', description: 'Where in outputDir to write replica screenshots. Default: "replica-screenshots". Files land at //.png.' }, - cdpPort: { type: 'number', description: 'Connect to existing Chrome via CDP (otherwise launches a new browser).' }, - }, - required: ['outputDir', 'replicaBaseUrl', 'urls'], - }, - }, - { - name: 'liberate_refine_report', - description: 'Validate refine coverage for one page: reads /refine//*.json (one file per section, written by match-section) and enforces that EVERY finding id appears in exactly one of applied[]/skipped[]. Fails loudly, naming unaccounted IDs. match-page must not mark a page done until this passes.', - inputSchema: { - type: 'object' as const, - properties: { - outputDir: { type: 'string', description: 'Liberation output directory (contains refine// written by match-section).' }, - slug: { type: 'string', description: 'Page slug whose refine// directory to validate.' }, - }, - required: ['outputDir', 'slug'], - }, - }, - { - name: 'liberate_compare', - description: - 'Pixel-parity scorer (fixed viewport). Joins an origin screenshots dir to a replica screenshots dir by URL pathname, crops both full-page PNGs to the top 1440×900 / 390×844 region, and returns per-pathname desktop/mobile similarity scores (1 − diffPixels/total). Writes comparison.json (v2 with originHeight/replicaHeight/heightMismatchRatio per viewport) + diff PNGs into the replica dir. Writes magenta-padded .padded.png diff when height mismatch exceeds 2%. Both dirs must have the standard layout: manifest.json + desktop/.png + mobile/.png.', - inputSchema: { - type: 'object' as const, - properties: { - originDir: { type: 'string', description: 'Origin screenshots dir (manifest.json + desktop/ mobile/).' }, - replicaDir: { type: 'string', description: 'Replica screenshots dir, same layout. comparison.json + diff/ are written here.' }, - viewports: { type: 'array', items: { type: 'string', enum: ['desktop', 'mobile'] }, description: 'Viewports to score. Default: both.' }, - diffOutputDir: { type: 'string', description: 'Where to write diff PNGs. Default: /diff.' }, - floor: { type: 'number', description: 'Pass/fail score floor used for repair-tasks.json records. Default 0.99.' }, - maxHeightDelta: { type: 'number', description: 'Height-gate tolerance in capture px (pre-crop |originH - replicaH|). Default 8.' }, - }, - required: ['originDir', 'replicaDir'], - }, - }, - { - name: 'liberate_ingest_local_site', - description: - 'Stage 1a of the owned-source path: ingest a local static-site directory (HTML/CSS/JS) and normalize each page into validated native Gutenberg block markup. Writes /composed/.blocks.html sidecars + /normalize-report.json. No Playwright/Studio. Downstream theme-scaffold/install/compare stages consume the sidecars.', - inputSchema: { - type: 'object' as const, - properties: { - dir: { type: 'string', description: 'Absolute path to the local static-site directory to ingest.' }, - outputDir: { type: 'string', description: 'Liberation output directory for composed sidecars + normalize-report.json. Defaults to `dir`.' }, - nativeBehaviors: { type: 'boolean', description: 'Detect catalog behaviors in the source css/js and emit dla/* Interactivity wrappers in the sidecars instead of core/group: uniform dla/reveal plus per-section DOM patterns (dla/tabs, dla/slider, dla/modal — verbatim inner markup). liberate_convert_local_site threads its own flag through here.' }, - }, - required: ['dir'], - }, - }, - { - name: 'liberate_convert_local_site', - description: - 'Stage 1b+1c of the owned-source path: full local-static-site → live Studio site. Reuses liberate_ingest_local_site (sidecars + normalize-report), optionally captures the source design (palette/typography/screenshots) and self-hosts Google Fonts, assembles the local block theme (core/navigation header from the nav graph, foundation-styled footer, no-title page templates), writes + activates it, creates WP Pages from the sidecars (idempotent via _source_url), sets the front page, assigns the page-local template, and optionally captures the WP replica + scores parity.', - inputSchema: { - type: 'object' as const, - properties: { - dir: { type: 'string', description: 'Absolute path to the local static-site directory.' }, - studioSitePath: { type: 'string', description: 'Studio site path on host (e.g. ~/Studio/my-site — the dir studio site list prints, not wp-root).' }, - createSite: { type: 'boolean', description: 'Provision the Studio site via `studio site create` when none exists at studioSitePath (idempotent — an existing site is reused). Default false (errors if the site is absent). Admin creds via env WP_ADMIN_USER/WP_ADMIN_PASS; omitted → Studio auto-generates.' }, - outputDir: { type: 'string', description: 'Liberation output dir for sidecars + reports. Defaults to `dir`.' }, - themeSlug: { type: 'string', description: 'Theme slug (kebab-case). Default: local-site-theme.' }, - siteTitle: { type: 'string', description: 'Site title for header/footer. Default: home page .' }, - skipDesign: { type: 'boolean', description: 'Skip source design capture (tokens/fonts) and compare; theme uses default styling.' }, - skipCompare: { type: 'boolean', description: 'Skip the WP-replica screenshot + parity compare stage.' }, - wpUrl: { type: 'string', description: 'Base URL for replica capture. Default: auto-resolved via wp option get siteurl (Studio assigns random ports); explicit value overrides.' }, - carryCss: { type: 'boolean', description: 'Carry the source stylesheet into the theme (adapted for the block DOM). Default true — the stage-1d parity mechanism; tokens-only theming when false.' }, - carryJs: { type: 'boolean', description: 'Carry the source scripts into the theme (enqueued footer, html.js gate added). Default true for identical replication.' }, - nativeBehaviors: { type: 'boolean', description: 'Replace carried source JS with native Interactivity blocks (reveal, sticky, plus per-section tabs/slider/modal with verbatim inner markup); unmapped behaviors land in behavior-gaps.json. Forces carryJs off.' }, - editableIslands: { type: 'boolean', description: 'Convert carried core/html islands into editable dla/editable-html blocks (text+image bindable, static-save, render-anywhere). Default true; set false to force plain core/html.' }, - dataModel: { type: 'boolean', description: 'WordPress-driven data path: when a data-model.json (from the model-local-data skill) is present in outputDir/dir, register a CPT+taxonomy via generated mu-plugins, insert items idempotently, and replace empty JS-mount grids with native core/query loops (dla/data-card cards) while neutralizing the JS data-mounts and rebinding modal lookups to per-card DOM islands. Default on when the file exists; pass false to force off.' }, - repair: { type: 'boolean', description: 'Deterministic parity repair loop: diff regions → computed-style probe → generated parity-patch.css → re-compare, bounded. Default true. No AI involved.' }, - maxRepairRounds: { type: 'number', description: 'Max repair rounds (0-5). Default 2. Loop also stops early on allPass or an unchanged divergence fingerprint.' }, - failOnConservationRailDrop: { type: 'boolean', description: 'Opt-in hard fail for local region-audit conservation: when true, unassigned nav/complementary rails with at least two links set isError. Default false (warn-only).' }, - }, - required: ['dir', 'studioSitePath'], - }, - }, - ...(JSON.parse(JSON.stringify( - Object.entries(NEW_TOOL_SCHEMAS).map(([name, def]) => ({ name, ...def })), - )) as Array<{ name: string; description: string; inputSchema: { type: 'object'; [k: string]: unknown } }>), - ], -})); +server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS })); + +server.setRequestHandler(CallToolRequestSchema, async (request) => { + const { name } = request.params; + const args = (request.params.arguments ?? {}) as Record<string, unknown>; + const log = (message: string) => { + void server.sendLoggingMessage({ level: 'info', data: message }).catch(() => undefined); + }; -/** Tool name → handler module. */ -const handlers: Record<string, Handler> = { - liberate_compare: compareHandler, - liberate_data_model_scaffold: dataModelScaffoldHandler, - liberate_design_foundation_save: designFoundationSaveHandler, - liberate_design_foundation_scaffold: designFoundationScaffoldHandler, - liberate_design_foundation_validate: designFoundationValidateHandler, - liberate_detect: detectHandler, - liberate_discover: discoverHandler, - liberate_block_transform_apply: blockTransformApplyHandler, - liberate_block_compose: blockComposeHandler, - liberate_extract: extractHandler, - liberate_extract_one: extractOneHandler, - liberate_media_install: mediaInstallHandler, - liberate_replicate_tick: replicateTickHandler, - liberate_import: wpImportHandler, - liberate_inspect: inspectHandler, - liberate_map_apis: mapApisHandler, - liberate_preview: previewHandler, - liberate_install_theme: installThemeHandler, - liberate_theme_scaffold: themeScaffoldHandler, - liberate_probe: probeHandler, - liberate_qa: qaHandler, - liberate_replicate_inventory: replicateInventoryHandler, - liberate_replicate_verify: replicateVerifyHandler, - liberate_refine_report: refineReportHandler, - liberate_screenshot: screenshotHandler, - liberate_setup: setupHandler, - liberate_paths: pathsHandler, - liberate_status: statusHandler, - liberate_verify: verifyHandler, - liberate_cluster_pages: clusterPagesHandler, - liberate_section_extract: sectionExtractHandler, - liberate_compose_instantiate: composeInstantiateHandler, - liberate_ingest_local_site: ingestLocalSiteHandler, - liberate_convert_local_site: convertLocalSiteHandler, - liberate_validate_artifacts: validateArtifactsHandler, - liberate_reconstruct_pages: reconstructPagesHandler, - liberate_reconstruct_pages_carry: reconstructPagesCarryHandler, - liberate_blockify_wxr: blockifyWxrHandler, -}; + try { + if (name === 'liberate') { + const { liberateSite } = await import('./ui/liberate.js'); + const result = await liberateSite({ + url: String(args.url ?? ''), + outputBase: typeof args.outputDir === 'string' ? args.outputDir : resolveOutputBase(), + resume: args.resume === true, + screenshots: args.screenshots === true, + // A tool call has no terminal to hold, so it never serves. + serve: false, + log, + }); + return textResult({ + websiteDir: result.websiteDir, + routesDiscovered: result.routesDiscovered, + routesCaptured: result.routesCaptured, + routesSkipped: result.routesSkipped, + routesFailed: result.routesFailed, + }); + } -function makeContext(): HandlerContext { - return { adapters, findAdapter, textResult, errorResult, server }; -} + if (name === 'compare') { + const { checkFidelity } = await import('./lib/fidelity/check.js'); + const report = await checkFidelity({ + directory: String(args.directory ?? ''), + screenshots: args.screenshots === true, + log, + }); + return textResult(report); + } -server.setRequestHandler(CallToolRequestSchema, async (request) => { - const { name, arguments: args } = request.params; - const handler = handlers[name]; - if (!handler) return errorResult(`Unknown tool: ${name}`); - return handler((args ?? {}) as Record<string, unknown>, makeContext()); -}); + if (name === 'publish') { + const { publishSite } = await import('./ui/publish.js'); + const result = await publishSite({ + directory: String(args.directory ?? ''), + target: typeof args.target === 'string' ? args.target : 'spacefast', + token: typeof args.token === 'string' ? args.token : process.env.SPACEFAST_TOKEN, + log, + }); + return textResult(result); + } + return errorResult(`Unknown tool: ${name}`); + } catch (error) { + return errorResult(error instanceof Error ? error.message : String(error)); + } +}); async function main() { const transport = new StdioServerTransport(); diff --git a/packages/data-liberation-agent/src/mcp-server/handler-types.ts b/packages/data-liberation-agent/src/mcp-server/handler-types.ts deleted file mode 100644 index ae2d149d24..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handler-types.ts +++ /dev/null @@ -1,49 +0,0 @@ -// -// MCP Handler Contract -// ==================== -// Each tool's logic lives in its own handler module under handlers/<tool>.ts. -// mcp-server.ts is a thin router that imports handlers and dispatches by tool -// name. Handlers receive their args + a context with shared helpers (adapter -// registry, result wrappers, server reference for notifications). -// -// The context-object pattern (vs module-level globals) makes handlers -// unit-testable in isolation — pass a mock context, assert on the returned -// ToolResult. -// -import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import type { PlatformAdapter } from '../types.js'; - -/** - * MCP tool result envelope. Mirrors what setRequestHandler returns. - * The index signature keeps it assignable to the SDK's ServerResult union - * (which has [key: string]: unknown). - */ -export interface ToolResult { - content: Array<{ type: 'text'; text: string }>; - isError?: boolean; - [key: string]: unknown; -} - -/** Shared dependencies handlers receive. */ -export interface HandlerContext { - /** All registered platform adapters. */ - adapters: PlatformAdapter[]; - /** Find a registered adapter by platform id; returns null if unknown. */ - findAdapter(platform: string): PlatformAdapter | null; - /** Wrap a JSON-serializable value as a tool result. */ - textResult(data: unknown): ToolResult; - /** Wrap an error message as an isError tool result. */ - errorResult(message: string): ToolResult; - /** The active MCP server instance — for sending notifications during long-running tools. */ - server: Server; -} - -/** - * Handler signature. The router converts the raw `arguments` payload into a - * Record before dispatching, so handlers don't have to deal with optional - * undefined args. - */ -export type Handler = ( - args: Record<string, unknown>, - ctx: HandlerContext, -) => Promise<ToolResult>; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/__snapshots__/tool-schemas.test.ts.snap b/packages/data-liberation-agent/src/mcp-server/handlers/__snapshots__/tool-schemas.test.ts.snap deleted file mode 100644 index aca89771a5..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/__snapshots__/tool-schemas.test.ts.snap +++ /dev/null @@ -1,87 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`new tool contracts > locks the agent-facing tool surface (update the snapshot deliberately) 1`] = ` -{ - "liberate_cluster_pages": { - "description": "Cluster page signatures by exact layout signature; pick a representative per cluster.", - "inputSchema": { - "properties": { - "signatures": { - "description": "PageSignature[]", - "type": "array", - }, - }, - "required": [ - "signatures", - ], - "type": "object", - }, - }, - "liberate_compose_instantiate": { - "description": "Deterministically fill a cluster layout skeleton with a page's content; flag misfits.", - "inputSchema": { - "properties": { - "pageContent": { - "type": "object", - }, - "skeleton": { - "type": "object", - }, - }, - "required": [ - "skeleton", - "pageContent", - ], - "type": "object", - }, - }, - "liberate_section_extract": { - "description": "Extract a page signature (off saved HTML) or full computed-style section specs (representatives).", - "inputSchema": { - "properties": { - "cdpPort": { - "description": "detail=full only: connect to an existing Chromium over CDP instead of launching", - "type": "number", - }, - "detail": { - "enum": [ - "signature", - "full", - ], - "type": "string", - }, - "html": { - "type": "string", - }, - "mediaMap": { - "description": "detail=full only: {sourceCdnUrl: uploadedWpUrl} rewrite map", - "type": "object", - }, - "url": { - "type": "string", - }, - }, - "required": [ - "url", - "detail", - ], - "type": "object", - }, - }, - "liberate_validate_artifacts": { - "description": "Pre-install gate: drift + escaping/injection + provenance over generated patterns.", - "inputSchema": { - "properties": { - "patterns": { - "description": "ArtifactPattern[]", - "type": "array", - }, - }, - "required": [ - "patterns", - ], - "type": "object", - }, - }, -} -`; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/_design-foundation-shared.ts b/packages/data-liberation-agent/src/mcp-server/handlers/_design-foundation-shared.ts deleted file mode 100644 index a1211f27bc..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/_design-foundation-shared.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Resolve a dotted path ("color.accent.primary") to a Role-shaped value in - * the foundation. Returns true iff the path exists and the role object has - * non-empty value/role/evidence and no "TODO" sentinel. Used by the - * design-foundation-validate handler's skillTodos check. - */ -export function pathResolvesToValidRole(f: unknown, dottedPath: string): boolean { - const parts = dottedPath.split('.'); - let cur: unknown = f; - for (const p of parts) { - if (!cur || typeof cur !== 'object') return false; - cur = (cur as Record<string, unknown>)[p]; - } - if (!cur || typeof cur !== 'object') return false; - const role = cur as { value?: unknown; role?: unknown; evidence?: unknown; css?: unknown }; - const hasEvidence = Array.isArray(role.evidence) && role.evidence.length > 0; - const hasValue = (typeof role.value === 'string' && role.value.length > 0 && role.value !== 'TODO') - || (typeof role.css === 'string' && role.css.length > 0 && role.css !== 'TODO'); - const hasRole = typeof role.role === 'string' && role.role.length > 0 && role.role !== 'TODO'; - return hasEvidence && hasValue && hasRole; -} diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/block-compose.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/block-compose.test.ts deleted file mode 100644 index 1393edf67f..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/block-compose.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; -import { join } from 'node:path'; -import { blockComposeHandler } from './block-compose.js'; -import { appendTransform } from '../../lib/streaming/block-transform-log.js'; -import type { HandlerContext, ToolResult } from '../handler-types.js'; -import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function makeCtx(): HandlerContext { - return { - adapters: [], - findAdapter: () => null, - textResult: (data: unknown): ToolResult => ({ - content: [{ type: 'text', text: JSON.stringify(data) }], - structured: data, - }), - errorResult: (message: string): ToolResult => ({ - content: [{ type: 'text', text: message }], - isError: true, - }), - server: {} as unknown as Server, - }; -} - -function readResult(r: ToolResult): { isError?: boolean; data?: Record<string, unknown>; text?: string } { - if (r.isError) return { isError: true, text: r.content[0]?.text }; - return { - data: (r as { structured?: Record<string, unknown> }).structured ?? undefined, - text: r.content[0]?.text, - }; -} - -const VALID_BLOCKS = '<!-- wp:paragraph --><p>About us at example.</p><!-- /wp:paragraph -->'; - -describe('blockComposeHandler', () => { - it('rejects calls missing required args', async () => { - const result = await blockComposeHandler({}, makeCtx()); - expect(result.isError).toBe(true); - }); - - it('rejects malformed block markup (mismatched close)', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const result = await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/about', - slug: 'about', - blocks: '<!-- wp:paragraph --><p>Hi</p><!-- /wp:heading -->', - }, - makeCtx(), - ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('roundtrip'); - }); - - it('rejects empty markup', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const result = await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/x', - slug: 'x', - blocks: ' ', - }, - makeCtx(), - ); - expect(result.isError).toBe(true); - }); - - it('rejects Custom HTML blocks even when the text is source-grounded', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const result = await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/about', - slug: 'about', - blocks: '<!-- wp:html --><div>About us at example.</div><!-- /wp:html -->', - sourceHtml: '<html><body><p>About us at example.</p></body></html>', - }, - makeCtx(), - ); - - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('Custom HTML'); - expect(existsSync(join(dir, 'composed', 'about.blocks.html'))).toBe(false); - }); - - it('writes the sidecar at <outputDir>/composed/<slug>.blocks.html on valid input', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const result = await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/about', - slug: 'about', - blocks: VALID_BLOCKS, - // sourceHtml omitted — output-verify falls back to manifest, which doesn't exist → skipped - }, - makeCtx(), - ); - expect(result.isError).toBeUndefined(); - const sidecar = join(dir, 'composed', 'about.blocks.html'); - expect(existsSync(sidecar)).toBe(true); - expect(readFileSync(sidecar, 'utf8')).toBe(VALID_BLOCKS); - const parsed = readResult(result); - expect(parsed.data?.composedPath).toBe(sidecar); - expect(parsed.data?.blocksCount).toBe(1); - }); - - it('appends a block-transform-log entry on success', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/about', - slug: 'about', - blocks: VALID_BLOCKS, - }, - makeCtx(), - ); - const log = readFileSync(join(dir, 'block-transform-log.jsonl'), 'utf8'); - const lines = log.trim().split('\n'); - // header + 1 entry - expect(lines.length).toBe(2); - const entry = JSON.parse(lines[1]); - expect(entry.url).toBe('https://example.com/about'); - expect(entry.slug).toBe('about'); - expect(entry.blocksCount).toBe(1); - expect(entry.composedBy).toBe('compose-page-blocks@v1.0'); - }); - - it('verifies output against sourceHtml — rejects markup with hallucinated text', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const sourceHtml = '<html><body><p>Welcome to our site</p></body></html>'; - const blocksWithHallucination = - '<!-- wp:paragraph --><p>This text is totally not in the source HTML at all.</p><!-- /wp:paragraph -->'; - const result = await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/x', - slug: 'x', - blocks: blocksWithHallucination, - sourceHtml, - }, - makeCtx(), - ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('verification failed'); - // No sidecar written when validation fails. - expect(existsSync(join(dir, 'composed', 'x.blocks.html'))).toBe(false); - }); - - it('reads sourceHtml from screenshot manifest when not passed explicitly', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - // Stage a manifest pointing at an html file with text matching VALID_BLOCKS. - const screenshotsDir = join(dir, 'screenshots'); - const htmlDir = join(dir, 'html'); - mkdirSync(screenshotsDir, { recursive: true }); - mkdirSync(htmlDir, { recursive: true }); - writeFileSync( - join(htmlDir, 'about.html'), - '<html><body><p>About us at example.</p></body></html>', - 'utf8', - ); - writeFileSync( - join(screenshotsDir, 'manifest.json'), - JSON.stringify({ - entries: { - 'https://example.com/about': { html: 'html/about.html' }, - }, - }), - 'utf8', - ); - const result = await blockComposeHandler( - { - outputDir: dir, - url: 'https://example.com/about', - slug: 'about', - blocks: VALID_BLOCKS, - }, - makeCtx(), - ); - expect(result.isError).toBeUndefined(); - }); - - it('idempotent — second call with identical input is skipped (no double-write)', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const args = { - outputDir: dir, - url: 'https://example.com/about', - slug: 'about', - blocks: VALID_BLOCKS, - }; - const first = await blockComposeHandler(args, makeCtx()); - expect(first.isError).toBeUndefined(); - const log1 = readFileSync(join(dir, 'block-transform-log.jsonl'), 'utf8'); - - const second = await blockComposeHandler(args, makeCtx()); - expect(second.isError).toBeUndefined(); - const parsed = readResult(second); - expect(parsed.data?.skipped).toBe(true); - - const log2 = readFileSync(join(dir, 'block-transform-log.jsonl'), 'utf8'); - // Log unchanged on idempotent skip. - expect(log2).toBe(log1); - }); - - it('re-composing different markup for same URL writes a new sidecar and logs again', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'compose-')); - const sourceHtml = '<html><body><p>About us at example.</p><p>Second pass content here.</p></body></html>'; - const v1 = VALID_BLOCKS; - const v2 = '<!-- wp:paragraph --><p>Second pass content here.</p><!-- /wp:paragraph -->'; - await blockComposeHandler( - { outputDir: dir, url: 'https://example.com/about', slug: 'about', blocks: v1, sourceHtml }, - makeCtx(), - ); - await blockComposeHandler( - { outputDir: dir, url: 'https://example.com/about', slug: 'about', blocks: v2, sourceHtml }, - makeCtx(), - ); - const sidecar = readFileSync(join(dir, 'composed', 'about.blocks.html'), 'utf8'); - expect(sidecar).toBe(v2); - // Two entries in the log (header + 2). - const log = readFileSync(join(dir, 'block-transform-log.jsonl'), 'utf8').trim().split('\n'); - expect(log.length).toBe(3); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/block-compose.ts b/packages/data-liberation-agent/src/mcp-server/handlers/block-compose.ts deleted file mode 100644 index 8f74cac1c3..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/block-compose.ts +++ /dev/null @@ -1,140 +0,0 @@ -// -// liberate_block_compose -// ======================= -// Validates composed block markup and writes it to a sidecar file -// (`<outputDir>/composed/<slug>.blocks.html`) for the streaming watch -// loop to pick up before the post is inserted into WordPress. This is -// the **compose-then-install** counterpart to -// `liberate_block_transform_apply` (install-then-update). -// -// Why a separate tool: the streaming flow buffers extracted URLs and -// only installs them once the design foundation exists, so the first -// (and only) `wp_insert_post` carries block markup as `post_content`. -// The agent that produces the markup needs a way to hand the result -// back to the runner WITHOUT touching the database. This handler is -// that hand-off — same validation rules as apply (markup roundtrip + -// output-verify against source HTML), same idempotency log, but no -// `wp post update`. -// -// The runner reads `<outputDir>/composed/<slug>.blocks.html` after the -// agent returns and passes the contents as `installPost.contentOverride`. -// - -import { createHash } from 'node:crypto'; -import { mkdirSync, writeFileSync } from 'node:fs'; -import { dirname } from 'node:path'; -import type { Handler } from '../handler-types.js'; -import { verifyComposedOutput } from '../../lib/streaming/output-verify.js'; -import { - appendTransform, - findLastTransform, - type BlockTransformEntry, -} from '../../lib/streaming/block-transform-log.js'; -import { - blockMarkupRoundtrips, - readSourceHtmlFromManifest, - composedSidecarPath, - countBlocks, -} from '../../lib/streaming/block-markup-validate.js'; -import { - containsCustomHtmlBlock, - customHtmlBlockError, -} from '../../lib/wordpress/block-policy.js'; - -function sha256(text: string): string { - return createHash('sha256').update(text).digest('hex'); -} - -export const blockComposeHandler: Handler = async (args, ctx) => { - const outputDir = args.outputDir as string; - const url = args.url as string; - const slug = args.slug as string; - const blocks = args.blocks as string; - const sourceHtmlArg = args.sourceHtml as string | undefined; - const composedBy = (args.composedBy as string) ?? 'compose-page-blocks@v1.0'; - const source = (args.source as 'heuristic' | 'ai') ?? 'ai'; - - if (!outputDir || !url || !slug || !blocks) { - return ctx.errorResult( - 'liberate_block_compose requires outputDir + url + slug + blocks', - ); - } - - // Validation gate 1: markup is structurally well-formed. - const roundtrip = blockMarkupRoundtrips(blocks); - if (!roundtrip.ok) { - return ctx.errorResult(`Block markup failed roundtrip validation: ${roundtrip.reason}`); - } - - if (containsCustomHtmlBlock(blocks)) { - return ctx.errorResult(customHtmlBlockError('Composed post_content')); - } - - // Validation gate 2: every text node in the proposed blocks must appear - // in the source HTML (anti-hallucination). Same rule as apply — if the - // caller didn't pass sourceHtml, fall back to reading from the - // screenshot manifest. - const sourceHtml = sourceHtmlArg ?? readSourceHtmlFromManifest(outputDir, url) ?? null; - if (sourceHtml) { - const verifyResult = verifyComposedOutput(blocks, sourceHtml); - if (!verifyResult.valid) { - return ctx.errorResult( - `Output verification failed — text not found in source: ${verifyResult.hallucinated.slice(0, 3).join(' | ')}`, - ); - } - } - - // Idempotency: if we already composed identical input → output, the - // sidecar should already be on disk. Short-circuit so resume runs - // don't re-write or re-log. - const sourceHash = sha256(sourceHtml ?? url); - const outputHash = sha256(blocks); - const last = findLastTransform(outputDir, url); - const sidecarPath = composedSidecarPath(outputDir, slug); - if (last && last.sourceHash === sourceHash && last.outputHash === outputHash) { - return ctx.textResult({ - ok: true, - url, - slug, - composedPath: sidecarPath, - skipped: true, - reason: 'identical sourceHash + outputHash already composed', - previousAt: last.transformedAt, - }); - } - - // Write sidecar atomically-ish (writeFileSync + mkdir parents). The - // runner reads this exact path; mismatch → installPost falls back to - // raw HTML (NO_AGENT path), so the contract is "the file at this path - // is the canonical block markup for <slug>". - try { - mkdirSync(dirname(sidecarPath), { recursive: true }); - writeFileSync(sidecarPath, blocks, 'utf8'); - } catch (err) { - return ctx.errorResult(`Failed to write composed sidecar: ${(err as Error).message}`); - } - - // Append to the same block-transform-log apply uses, so downstream - // tooling (audit, idempotency checks) sees a unified history. - const entry: BlockTransformEntry = { - url, - slug, - blocksCount: countBlocks(blocks), - transformedAt: new Date().toISOString(), - source, - warnings: [], - composedBy, - sourceHash, - outputHash, - }; - appendTransform(outputDir, entry); - - return ctx.textResult({ - ok: true, - url, - slug, - composedPath: sidecarPath, - blocksCount: entry.blocksCount, - composedAt: entry.transformedAt, - }); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/block-transform-apply.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/block-transform-apply.test.ts deleted file mode 100644 index 9067c6f564..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/block-transform-apply.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { blockTransformApplyHandler } from './block-transform-apply.js'; -import { appendTransform } from '../../lib/streaming/block-transform-log.js'; -import type { HandlerContext, ToolResult } from '../handler-types.js'; -import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); -mkdirSync(FIXTURE_TMP, { recursive: true }); - -function makeCtx(): HandlerContext { - return { - adapters: [], - findAdapter: () => null, - textResult: (data: unknown): ToolResult => ({ - content: [{ type: 'text', text: JSON.stringify(data) }], - structured: data, - }), - errorResult: (message: string): ToolResult => ({ - content: [{ type: 'text', text: message }], - isError: true, - }), - server: {} as unknown as Server, - }; -} - -function readResult(r: ToolResult): { isError?: boolean; data?: unknown; text?: string } { - if (r.isError) return { isError: true, text: r.content[0]?.text }; - // Structured payload is on `structured` (set by makeCtx textResult above). - return { data: (r as { structured?: unknown }).structured ?? null, text: r.content[0]?.text }; -} - -describe('blockTransformApplyHandler — validation and idempotency', () => { - it('rejects calls missing required args', async () => { - const ctx = makeCtx(); - const result = await blockTransformApplyHandler({}, ctx); - expect(result.isError).toBe(true); - }); - - it('rejects malformed block markup that does not roundtrip (mismatched close)', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'apply-')); - const ctx = makeCtx(); - const result = await blockTransformApplyHandler( - { - outputDir: dir, - url: 'https://example.com/x', - blocks: '<!-- wp:paragraph --><p>Hi</p><!-- /wp:heading -->', - }, - ctx, - ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('roundtrip'); - }); - - it('rejects markup with unclosed blocks', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'apply-')); - const ctx = makeCtx(); - const result = await blockTransformApplyHandler( - { - outputDir: dir, - url: 'https://example.com/x', - blocks: '<!-- wp:paragraph --><p>Hi</p>', - }, - ctx, - ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('unclosed'); - }); - - it('rejects when output verification finds hallucinated text', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'apply-')); - // Fake a manifest pointing to a source HTML file - const htmlDir = join(dir, 'html'); - mkdirSync(htmlDir, { recursive: true }); - writeFileSync(join(htmlDir, 'about.html'), '<article><p>Foo Industries</p></article>'); - const screenshotsDir = join(dir, 'screenshots'); - mkdirSync(screenshotsDir, { recursive: true }); - writeFileSync( - join(screenshotsDir, 'manifest.json'), - JSON.stringify({ - entries: { 'https://example.com/about': { html: 'html/about.html' } }, - }), - ); - const ctx = makeCtx(); - const result = await blockTransformApplyHandler( - { - outputDir: dir, - url: 'https://example.com/about', - blocks: - '<!-- wp:paragraph --><p>Bar Inc</p><!-- /wp:paragraph -->', - target: { kind: 'studio', studioSitePath: '/tmp/site' }, - }, - ctx, - ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('verification'); - expect(result.content[0].text).toContain('Bar Inc'); - }); - - it('skips re-application when sourceHash + outputHash unchanged (idempotency)', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'apply-')); - // Fake an existing log entry that matches the source + blocks we're about to send. - // Source HTML for the URL: - const htmlDir = join(dir, 'html'); - mkdirSync(htmlDir, { recursive: true }); - writeFileSync(join(htmlDir, 'about.html'), '<article><p>Hello world content</p></article>'); - mkdirSync(join(dir, 'screenshots'), { recursive: true }); - writeFileSync( - join(dir, 'screenshots', 'manifest.json'), - JSON.stringify({ - entries: { 'https://example.com/about': { html: 'html/about.html' } }, - }), - ); - - const blocks = '<!-- wp:paragraph -->\n<p>Hello world content</p>\n<!-- /wp:paragraph -->'; - - // Compute the hashes the handler will compute and pre-seed the log. - const { createHash } = await import('node:crypto'); - const sourceHtml = '<article><p>Hello world content</p></article>'; - const sourceHash = createHash('sha256').update(sourceHtml).digest('hex'); - const outputHash = createHash('sha256').update(blocks).digest('hex'); - appendTransform(dir, { - url: 'https://example.com/about', - slug: 'about', - blocksCount: 1, - transformedAt: '2026-04-29T00:00:00.000Z', - source: 'heuristic', - warnings: [], - composedBy: 'compose-page-blocks@v1.0', - sourceHash, - outputHash, - }); - - const ctx = makeCtx(); - const result = await blockTransformApplyHandler( - { - outputDir: dir, - url: 'https://example.com/about', - blocks, - target: { kind: 'studio', studioSitePath: '/tmp/site' }, - }, - ctx, - ); - // Skip path: NOT an error, and tells caller skipped:true with reason. - expect(result.isError).toBeUndefined(); - expect(result.content[0].text).toContain('skipped'); - expect(result.content[0].text).toContain('identical'); - }); - - it('errors when studio target is selected without a studioSitePath', async () => { - const dir = mkdtempSync(join(FIXTURE_TMP, 'apply-')); - const ctx = makeCtx(); - const result = await blockTransformApplyHandler( - { - outputDir: dir, - url: 'https://example.com/x', - blocks: '<!-- wp:paragraph --><p>Hi</p><!-- /wp:paragraph -->', - target: { kind: 'studio' }, - }, - ctx, - ); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('studioSitePath'); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/block-transform-apply.ts b/packages/data-liberation-agent/src/mcp-server/handlers/block-transform-apply.ts deleted file mode 100644 index 6f2ce910bf..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/block-transform-apply.ts +++ /dev/null @@ -1,180 +0,0 @@ -// -// liberate_block_transform_apply -// ============================== -// Applies composed block markup to the running WP site for a given URL. -// Sequencing (per the streaming plan, Phase 3): -// -// 1. parse_blocks roundtrip validation — ensure the input is at least -// lexically valid block markup. We don't have wp_parse_blocks() here, -// so do a structural sanity check (matching open/close comments). -// 2. output-verify — confirm every text node in the proposed blocks is -// a substring of the source HTML's plain text (anti-hallucination). -// 3. post-existence poll — find the post id matching `_source_url=<url>` -// with 3 retries + 500ms/2s/5s backoff (avoid the compose-then-apply -// race when WXR import hasn't landed yet). -// 4. Idempotency — short-circuit when block-transform-log records a -// successful apply with the same `sourceHash`. -// 5. Apply — `wp post update <postId> --post_content=<blocks>`. -// 6. Append to block-transform-log.jsonl on success. -// -// `target` selects the application path. Studio is the only supported target; -// other kinds return a not-yet-supported error so callers handle it explicitly. -// - -import { createHash } from 'node:crypto'; -import { execFile } from 'node:child_process'; -import { writeFileSync, mkdtempSync } from 'node:fs'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { promisify } from 'node:util'; -import type { Handler } from '../handler-types.js'; -import { verifyComposedOutput } from '../../lib/streaming/output-verify.js'; -import { pollForPost } from '../../lib/streaming/post-existence-poll.js'; -import { - appendTransform, - findLastTransform, - type BlockTransformEntry, -} from '../../lib/streaming/block-transform-log.js'; -import { - blockMarkupRoundtrips, - readSourceHtmlFromManifest, - slugFromUrl, - countBlocks, -} from '../../lib/streaming/block-markup-validate.js'; -import { - containsCustomHtmlBlock, - customHtmlBlockError, -} from '../../lib/wordpress/block-policy.js'; - -const execFileAsync = promisify(execFile); - -interface ApplyTarget { - /** "studio". */ - kind?: string; - /** Studio site path (parent dir, NOT the wordpress sub-dir). */ - studioSitePath?: string; - /** Site URL — kept for forward compatibility with the poller opts shape. */ - siteUrl?: string; -} - -function sha256(text: string): string { - return createHash('sha256').update(text).digest('hex'); -} - -export const blockTransformApplyHandler: Handler = async (args, ctx) => { - const outputDir = args.outputDir as string; - const url = args.url as string; - const blocks = args.blocks as string; - const target = (args.target ?? {}) as ApplyTarget; - - if (!outputDir || !url || !blocks) { - return ctx.errorResult( - 'liberate_block_transform_apply requires outputDir + url + blocks', - ); - } - - // Pre-apply 1: parse_blocks roundtrip / structural sanity. - const roundtrip = blockMarkupRoundtrips(blocks); - if (!roundtrip.ok) { - return ctx.errorResult(`Block markup failed roundtrip validation: ${roundtrip.reason}`); - } - - if (containsCustomHtmlBlock(blocks)) { - return ctx.errorResult(customHtmlBlockError('Applied post_content')); - } - - // Pre-apply 2: output-verify against source HTML when available. - const sourceHtml = readSourceHtmlFromManifest(outputDir, url); - const verifyResult = sourceHtml ? verifyComposedOutput(blocks, sourceHtml) : null; - if (verifyResult && !verifyResult.valid) { - return ctx.errorResult( - `Output verification failed — text not found in source: ${verifyResult.hallucinated.slice(0, 3).join(' | ')}`, - ); - } - - // Pre-apply 3: idempotency check. - const sourceHash = sha256(sourceHtml ?? url); - const outputHash = sha256(blocks); - const last = findLastTransform(outputDir, url); - if (last && last.sourceHash === sourceHash && last.outputHash === outputHash) { - return ctx.textResult({ - ok: true, - url, - skipped: true, - reason: 'identical sourceHash + outputHash already applied', - previousAppliedAt: last.transformedAt, - }); - } - - // Pre-apply 4: post existence (3 retries with backoff). - const studioSitePath = target.studioSitePath; - if (!studioSitePath) { - return ctx.errorResult( - 'Studio target requires `target.studioSitePath`. Pass `target: {kind: "studio", studioSitePath: "..."}` to specify the running Studio site.', - ); - } - - const poll = await pollForPost({ - siteUrl: target.siteUrl ?? '', - sourceUrl: url, - studioSitePath, - }); - if (!poll.found || !poll.postId) { - return ctx.textResult({ - ok: false, - url, - skipped: true, - reason: `Post for source URL not found after ${poll.attempts} attempts; WXR import may not have landed.`, - pollAttempts: poll.attempts, - }); - } - - // Apply via `studio wp post update`. We pass the blocks via a temp file - // (-) rather than command-line argv to avoid shell quoting + length limits. - const tmpDir = mkdtempSync(join(tmpdir(), 'dla-blocks-')); - const blocksPath = join(tmpDir, `${poll.postId}.blocks.html`); - writeFileSync(blocksPath, blocks); - - const warnings: string[] = []; - try { - await execFileAsync( - 'studio', - [ - 'wp', - '--path', - studioSitePath as string, - 'post', - 'update', - String(poll.postId), - blocksPath, - ], - { timeout: 60_000, maxBuffer: 50 * 1024 * 1024 }, - ); - } catch (err) { - return ctx.errorResult(`wp post update failed: ${(err as Error).message}`); - } - - const blocksCount = countBlocks(blocks); - const entry: BlockTransformEntry = { - url, - slug: slugFromUrl(url), - blocksCount, - transformedAt: new Date().toISOString(), - source: (args.source as 'heuristic' | 'ai') ?? 'ai', - warnings, - composedBy: (args.composedBy as string) ?? 'compose-page-blocks@v1.0', - sourceHash, - outputHash, - }; - appendTransform(outputDir, entry); - - return ctx.textResult({ - ok: true, - url, - postId: poll.postId, - pollAttempts: poll.attempts, - blocksCount, - appliedAt: entry.transformedAt, - warnings, - }); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/blockify-wxr.ts b/packages/data-liberation-agent/src/mcp-server/handlers/blockify-wxr.ts deleted file mode 100644 index 9d3d9dd9cb..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/blockify-wxr.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { join } from 'node:path'; -import { existsSync } from 'node:fs'; -import { ImportSession } from '../../lib/resume-state/index.js'; -import { blockifyWxrFile } from '../../lib/extraction/blockify-wxr.js'; -import type { Handler } from '../handler-types.js'; - -/** - * Bulk-convert post/page bodies in output.wxr to Gutenberg blocks via the source - * platform adapter's block recipe (seam 2). Blocks reconstruct path only — the - * blocks flow calls this after extraction and before import; the theme/carry path - * never does. No-op when the platform has no block recipe (returns skipped:true). - */ -export const blockifyWxrHandler: Handler = async (args, ctx) => { - const outputDir = args.outputDir as string | undefined; - if (!outputDir) return ctx.errorResult('liberate_blockify_wxr requires `outputDir`.'); - - const wxrPath = (args.wxrPath as string | undefined) ?? join(outputDir, 'output.wxr'); - if (!existsSync(wxrPath)) return ctx.errorResult(`WXR not found at ${wxrPath}`); - - // Prefer an explicit override, else the platform recorded at extraction. - const platform = (args.platform as string | undefined) ?? ImportSession.readAdapter(outputDir) ?? undefined; - const adapter = platform ? ctx.findAdapter(platform) : null; - if (!adapter?.blocks) { - return ctx.textResult({ - wxrPath, - converted: 0, - skipped: true, - reason: platform - ? `adapter '${platform}' has no block recipe — bodies left as source HTML` - : 'no platform recorded in session.json (pass `platform` to override)', - }); - } - - try { - const result = blockifyWxrFile(wxrPath, adapter.blocks); - return ctx.textResult({ wxrPath, platform, ...result }); - } catch (err) { - // A corrupt/unreadable WXR shouldn't surface as an unhandled throw. - return ctx.errorResult(`blockify failed for ${wxrPath}: ${err instanceof Error ? err.message : String(err)}`); - } -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/cluster-pages.ts b/packages/data-liberation-agent/src/mcp-server/handlers/cluster-pages.ts deleted file mode 100644 index 4dbad35fd8..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/cluster-pages.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { Handler } from '../handler-types.js'; -import { clusterPages } from '../../lib/replicate/cluster-pages.js'; -import type { PageSignature } from '../../lib/replicate/page-signature.js'; - -export const clusterPagesHandler: Handler = async (args, ctx) => { - const signatures = args.signatures as PageSignature[] | undefined; - if (!Array.isArray(signatures)) return ctx.errorResult('signatures[] is required'); - return ctx.textResult(clusterPages(signatures)); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/compare.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/compare.test.ts deleted file mode 100644 index 70893ff8e3..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/compare.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { mkdirSync, writeFileSync, rmSync, existsSync, readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { PNG } from 'pngjs'; -import { compareHandler } from './compare.js'; -import type { HandlerContext, ToolResult } from '../handler-types.js'; - -function fakeCtx(): HandlerContext { - return { - adapters: [], - findAdapter: () => null, - textResult: (data: unknown): ToolResult => ({ content: [{ type: 'text', text: JSON.stringify(data) }] }), - errorResult: (message: string): ToolResult => ({ content: [{ type: 'text', text: message }], isError: true }), - server: {} as never, - }; -} - -const TMP = join(process.cwd(), '.tmp-test', 'compare-handler'); - -function writeSolidPng(path: string, w: number, h: number, rgba: [number, number, number, number]) { - const png = new PNG({ width: w, height: h }); - for (let i = 0; i < w * h; i++) { - const o = i * 4; - png.data[o] = rgba[0]; png.data[o + 1] = rgba[1]; png.data[o + 2] = rgba[2]; png.data[o + 3] = rgba[3]; - } - mkdirSync(dirname(path), { recursive: true }); - writeFileSync(path, PNG.sync.write(png)); -} - -function buildDir(dir: string, url: string, slug: string, png: { w: number; h: number; color: [number, number, number, number] }) { - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, 'manifest.json'), JSON.stringify({ version: 1, entries: { [url]: { slug, capturedAt: '2026-05-20T00:00:00Z' } } }, null, 2)); - for (const vp of ['desktop', 'mobile']) writeSolidPng(join(dir, vp, `${slug}.png`), png.w, png.h, png.color); -} - -describe('compareHandler', () => { - it('errors when originDir/replicaDir are missing', async () => { - const res = await compareHandler({}, fakeCtx()); - expect(res.isError).toBe(true); - expect(res.content[0].text).toMatch(/originDir.*replicaDir/); - }); -}); - -describe('compareHandler repair tasks + height tally', () => { - beforeEach(() => rmSync(TMP, { recursive: true, force: true })); - afterEach(() => rmSync(TMP, { recursive: true, force: true })); - - it('writes repair-tasks.json (atomic sibling of comparison.json) and tallies heightDelta per page', async () => { - const origin = join(TMP, 'origin'); - const replica = join(TMP, 'replica'); - // 40px height loss, identical cropped content: score 1, height gate fails. - buildDir(origin, 'https://origin.test/p', 'p', { w: 1440, h: 940, color: [7, 7, 7, 255] }); - buildDir(replica, 'http://localhost:8881/p', 'p', { w: 1440, h: 900, color: [7, 7, 7, 255] }); - const res = await compareHandler({ originDir: origin, replicaDir: replica }, fakeCtx()); - expect(res.isError).toBeFalsy(); - const summary = JSON.parse(res.content[0].text) as { - results: unknown[]; - heightGate: { maxHeightDelta: number; perPage: Array<{ pathname: string; desktop: number | null; mobile: number | null }> }; - repairTasks: { count: number; path: string }; - }; - expect(summary.heightGate.maxHeightDelta).toBe(8); - expect(summary.heightGate.perPage).toEqual([{ pathname: '/p', desktop: 40, mobile: 40 }]); - expect(summary.repairTasks.count).toBe(2); // desktop + mobile height tasks - const tasksPath = join(replica, 'repair-tasks.json'); - expect(summary.repairTasks.path).toBe(tasksPath); - expect(existsSync(tasksPath)).toBe(true); - const onDisk = JSON.parse(readFileSync(tasksPath, 'utf8')) as { - schema: number; floor: number; maxHeightDelta: number; - tasks: Array<{ surface: string; kind: string; pathname: string; viewport: string; heightDelta: number | null }>; - }; - expect(onDisk.schema).toBe(1); - expect(onDisk.tasks).toHaveLength(2); - expect(onDisk.tasks.every((t) => t.surface === 'frontend' && t.kind === 'height' && t.pathname === '/p')).toBe(true); - }); - - it('passing comparison writes an empty task list', async () => { - const origin = join(TMP, 'origin'); - const replica = join(TMP, 'replica'); - buildDir(origin, 'https://origin.test/ok', 'ok', { w: 1440, h: 900, color: [7, 7, 7, 255] }); - buildDir(replica, 'http://localhost:8881/ok', 'ok', { w: 1440, h: 900, color: [7, 7, 7, 255] }); - const res = await compareHandler({ originDir: origin, replicaDir: replica }, fakeCtx()); - const summary = JSON.parse(res.content[0].text) as { repairTasks: { count: number } }; - expect(summary.repairTasks.count).toBe(0); - const onDisk = JSON.parse(readFileSync(join(replica, 'repair-tasks.json'), 'utf8')) as { tasks: unknown[] }; - expect(onDisk.tasks).toEqual([]); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/compare.ts b/packages/data-liberation-agent/src/mcp-server/handlers/compare.ts deleted file mode 100644 index 86484cc60d..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/compare.ts +++ /dev/null @@ -1,62 +0,0 @@ -// -// liberate_compare -// ================ -// Thin MCP wrapper over compareScreenshotDirs. Reads an origin screenshot -// dir and a replica screenshot dir, returns the per-pathname desktop/mobile -// parity scores (incl. the pre-crop heightDelta co-gate), and writes -// comparison.json + diff PNGs + repair-tasks.json into the replica dir. -// -import { writeFileSync, renameSync } from 'node:fs'; -import { join } from 'node:path'; -import type { Handler } from '../handler-types.js'; -import type { ViewportId } from '../../lib/screenshot/compare.js'; - -export const compareHandler: Handler = async (args, ctx) => { - const originDir = args.originDir as string; - const replicaDir = args.replicaDir as string; - if (!originDir || !replicaDir) { - return ctx.errorResult('liberate_compare requires originDir + replicaDir'); - } - const start = Date.now(); - try { - const { compareScreenshotDirs, buildRepairTasks, DEFAULT_MAX_HEIGHT_DELTA } = await import( - '../../lib/screenshot/compare.js' - ); - const maxHeightDelta = (args.maxHeightDelta as number | undefined) ?? DEFAULT_MAX_HEIGHT_DELTA; - const floor = (args.floor as number | undefined) ?? 0.99; - const result = await compareScreenshotDirs({ - originDir, - replicaDir, - viewports: args.viewports as ViewportId[] | undefined, - diffOutputDir: args.diffOutputDir as string | undefined, - maxHeightDelta, - }); - // Structured repair tasks — pure derivation from the results; atomic - // sibling of comparison.json (same tmp+rename convention as the other - // run artifacts). Empty list is still written: its absence vs emptiness - // must be distinguishable to consumers. - const tasks = buildRepairTasks(result.results, { floor }); - const tasksPath = join(replicaDir, 'repair-tasks.json'); - const tasksTmp = `${tasksPath}.tmp.${process.pid}`; - writeFileSync(tasksTmp, JSON.stringify({ schema: 1, floor, maxHeightDelta, tasks }, null, 2) + '\n'); - renameSync(tasksTmp, tasksPath); - console.error(`[compare] ${JSON.stringify({ tool: 'compare', originDir, replicaDir, count: result.results.length, repairTasks: tasks.length, durationMs: Date.now() - start })}`); - return ctx.textResult({ - ...result, - // Per-page height tally: the score alone is blind to height loss (the - // min-crop hides it) — surface the gate's measurements alongside. - heightGate: { - maxHeightDelta, - perPage: result.results.map((r) => ({ - pathname: r.pathname, - desktop: r.desktop.heightDelta ?? null, - mobile: r.mobile.heightDelta ?? null, - })), - }, - repairTasks: { count: tasks.length, path: tasksPath }, - }); - } catch (e) { - console.error(`[compare] ${JSON.stringify({ tool: 'compare', originDir, replicaDir, ok: false, durationMs: Date.now() - start })}`); - return ctx.errorResult((e as Error).message); - } -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/compose-instantiate.ts b/packages/data-liberation-agent/src/mcp-server/handlers/compose-instantiate.ts deleted file mode 100644 index 5bf826c7aa..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/compose-instantiate.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { Handler } from '../handler-types.js'; -import { composeInstantiate, type LayoutSkeleton } from '../../lib/replicate/compose-instantiate.js'; - -export const composeInstantiateHandler: Handler = async (args, ctx) => { - const skeleton = args.skeleton as LayoutSkeleton | undefined; - const pageContent = args.pageContent as Record<string, string | number> | undefined; - if (!skeleton || !pageContent) return ctx.errorResult('skeleton and pageContent are required'); - return ctx.textResult(composeInstantiate(skeleton, pageContent, (args.mediaMap as Record<string, string>) ?? {})); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site-conservation-contract.ts b/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site-conservation-contract.ts deleted file mode 100644 index 1f01cf8ee0..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site-conservation-contract.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { RegionSelectionReport } from '@automattic/blocks-engine/theme'; -import type { SourceLandmark } from '../../lib/replicate/section-extract.js'; - -export const LOCAL_CONSERVATION_REPORT_SCHEMA = 1; -export const LOCAL_CONSERVATION_RAIL_LINK_THRESHOLD = 2; -export const LOCAL_CONSERVATION_HARD_FAIL_ARG = 'failOnConservationRailDrop'; -export const LOCAL_CONSERVATION_HARD_FAIL_ROLES = ['nav', 'complementary'] as const; - -export type LocalConservationStatus = 'pass' | 'warn' | 'fail'; -export type LocalConservationHardFailRole = typeof LOCAL_CONSERVATION_HARD_FAIL_ROLES[number]; - -export interface LocalConservationSummary { - ok: boolean; - status: LocalConservationStatus; - unassignedRegions: number; - hardFailRegions: number; - artifact: string; - railHardFail: { - enabled: boolean; - roles: readonly LocalConservationHardFailRole[]; - minLinks: number; - }; -} - -export interface LocalConservationRegionAudit { - schema: typeof LOCAL_CONSERVATION_REPORT_SCHEMA; - site: string; - pages: RegionSelectionReport[]; - unassignedRegions: number; - hardFailRegions: SourceLandmark[]; -} diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site-jetpack-contract.ts b/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site-jetpack-contract.ts deleted file mode 100644 index 1155447239..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site-jetpack-contract.ts +++ /dev/null @@ -1,36 +0,0 @@ -export const JETPACK_FORMS_PLUGIN_INSTALL = { - pluginSlug: 'jetpack', - wpArgs: ['plugin', 'install', 'jetpack', '--activate'], - gateDescription: 'formsConverted >= 1', - flowLocation: 'after theme activation and before optional local interactivity plugin activation', - failureMode: 'non-fatal warning only', - warningPrefix: 'jetpack install/activate failed', - localFormsNote: 'Jetpack Forms blocks render and store submissions locally without a WordPress.com connection.', -} as const; - -export const JETPACK_FORMS_MODULE_ACTIVATE = { - moduleSlug: 'contact-form', - wpArgs: ['jetpack', 'module', 'activate', 'contact-form'], - gateDescription: JETPACK_FORMS_PLUGIN_INSTALL.gateDescription, - flowLocation: 'after Jetpack plugin install/activate succeeds and before optional local interactivity plugin activation', - failureMode: JETPACK_FORMS_PLUGIN_INSTALL.failureMode, - warningPrefix: 'jetpack contact-form module activate failed', - localFormsNote: JETPACK_FORMS_PLUGIN_INSTALL.localFormsNote, -} as const; - -export const JETPACK_FORMS_COMMAND_SEQUENCE = [ - JETPACK_FORMS_PLUGIN_INSTALL.wpArgs, - JETPACK_FORMS_MODULE_ACTIVATE.wpArgs, -] as const; - -export function shouldInstallJetpackFormsPlugin(formsConverted: number): boolean { - return formsConverted >= 1; -} - -export function jetpackFormsPluginInstallWarning(error: Error): string { - return `${JETPACK_FORMS_PLUGIN_INSTALL.warningPrefix}: ${error.message}. ${JETPACK_FORMS_PLUGIN_INSTALL.localFormsNote}`; -} - -export function jetpackFormsModuleActivateWarning(error: Error): string { - return `${JETPACK_FORMS_MODULE_ACTIVATE.warningPrefix}: ${error.message}. ${JETPACK_FORMS_MODULE_ACTIVATE.localFormsNote}`; -} diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site.test.ts deleted file mode 100644 index b7e8dc80b6..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/convert-local-site.test.ts +++ /dev/null @@ -1,2770 +0,0 @@ -// src/mcp-server/handlers/convert-local-site.test.ts -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from 'node:fs'; -import { homedir } from 'node:os'; -import { join } from 'node:path'; -import { PNG } from 'pngjs'; -import type { HandlerContext, ToolResult } from '../handler-types.js'; -import { JETPACK_FORM_PARITY_CSS } from '../../lib/replicate/local-theme/jetpack-form-parity-contract.js'; -import { - JETPACK_FORMS_COMMAND_SEQUENCE, - JETPACK_FORMS_MODULE_ACTIVATE, - JETPACK_FORMS_PLUGIN_INSTALL, -} from './convert-local-site-jetpack-contract.js'; - -// Mock BOTH exec seams before importing the handler: -// - node:child_process execFile → studio wp activation/option/meta commands -// - post-install installPost → page creation (it shells out internally) -// Per-test failure injection: a test sets execFailFor / installFailFor; -// beforeEach resets both so tests stay independent. - -// Heavy design-capture + compare seams — mocked at the module level so the -// handler never invokes Playwright or the pixel-matcher in unit tests. -// capturedRuns tracks calls so tests can assert on source + replica invocations. -// force is tracked so repair-loop tests can assert re-capture uses force:true. -const capturedRuns: Array<{ urls: string[]; outputDir: string; force?: boolean }> = []; - -// Repair-loop fixture seam: set per-test; cleared in beforeEach. -// When repairReplicaManifestEntries is non-empty, the captureScreenshots mock -// writes a proper manifest (pathname→slug) + red-pixel diff PNGs for the replica -// capture, so the repair loop can read them without real Playwright/pixelmatch. -let repairReplicaManifestEntries: Record<string, { slug: string }> = {}; -let repairDiffPng: Buffer | null = null; -const buildJetpackFormParityCssMock = vi.hoisted(() => vi.fn(() => ({ css: '' }))); -const regionCensusFailure = vi.hoisted(() => ({ throwOnExtract: false })); -const assembleLocalThemeCalls = vi.hoisted( - () => - [] as Array<{ - mainClass?: string; - mainWrapperClass?: string; - interiorChromeTemplates?: Array<{ - partSlug: string; - layoutWrapperTag?: string; - layoutWrapperClasses?: string[]; - layoutWrapperRailPosition?: 'beforeMain' | 'afterMain'; - }>; - }>, -); - -// Passthrough the block fixer: convert calls the real ingest handler, which now -// canonicalizes each page through a jsdom HTTP subprocess (~2.5s/test + parallel -// flakiness). These tests assert orchestration, not @wordpress/blocks -// canonicalization (covered by blockFixer.smoke.test.js); the stub keeps the -// suite fast + deterministic. Passthrough → sidecars equal the composed markup. -vi.mock('../../lib/streaming/block-fixer-client.js', () => ({ - BlockFixerClient: class { - async start(): Promise<void> {} - async stop(): Promise<void> {} - async fix(items: string[]): Promise<Array<{ html: string; changed: boolean; fixedIssues: string[] }>> { - return items.map((html) => ({ html, changed: false, fixedIssues: [] })); - } - }, -})); - -vi.mock('../../lib/screenshot/screenshotter.js', () => ({ - captureScreenshots: vi.fn(async (opts: { urls: string[]; outputDir: string; force?: boolean }) => { - capturedRuns.push({ urls: opts.urls, outputDir: opts.outputDir, force: opts.force }); - // Fabricate aggregate files the handler reads after source capture. - const { mkdirSync: md, writeFileSync: wf } = await import('node:fs'); - const { join: j } = await import('node:path'); - md(j(opts.outputDir, 'screenshots'), { recursive: true }); - wf(j(opts.outputDir, 'palette.json'), JSON.stringify({ version: 1, sampledUrls: 1, colors: [{ hex: '#0e2a30', count: 10, urls: 1 }, { hex: '#f7f2e9', count: 9, urls: 1 }, { hex: '#e2573b', count: 5, urls: 1 }] })); - wf(j(opts.outputDir, 'typography.json'), JSON.stringify({ version: 1, sampledUrls: 1, bySelector: { body: [{ fontFamily: 'X', fontSize: '16px', fontWeight: '400', lineHeight: '24px', urls: 1 }] } })); - wf(j(opts.outputDir, 'breakpoints.json'), JSON.stringify({ version: 1, sampledUrls: 1, minWidth: [], maxWidth: [] })); - // Replica captures: write a proper manifest + diff PNGs when the repair seam is active. - const hasRepairEntries = Object.keys(repairReplicaManifestEntries).length > 0; - if (hasRepairEntries && opts.outputDir.endsWith('/replica')) { - wf(j(opts.outputDir, 'screenshots', 'manifest.json'), JSON.stringify({ version: 1, entries: repairReplicaManifestEntries })); - if (repairDiffPng) { - md(j(opts.outputDir, 'screenshots', 'diff'), { recursive: true }); - for (const entry of Object.values(repairReplicaManifestEntries)) { - for (const vp of ['desktop', 'mobile']) { - wf(j(opts.outputDir, 'screenshots', 'diff', `${entry.slug}.${vp}.diff.png`), repairDiffPng); - } - } - } - } else { - wf(j(opts.outputDir, 'screenshots', 'manifest.json'), JSON.stringify({ version: 1, entries: {} })); - } - return { captured: opts.urls.length, failed: 0, skipped: 0, browserRestarts: 0, durationMs: 0, manifestPath: j(opts.outputDir, 'screenshots', 'manifest.json') }; - }), -})); -vi.mock('../../lib/screenshot/compare.js', () => ({ - compareScreenshotDirs: vi.fn(async () => ({ - version: 1, - comparedAt: 'TEST', - results: [ - { pathname: '/', originUrl: 'o', replicaUrl: 'r', desktop: { status: 'ok', score: 0.91 }, mobile: { status: 'ok', score: 0.88 } }, - { pathname: '/about/', originUrl: 'o', replicaUrl: 'r', desktop: { status: 'ok', score: 0.95 }, mobile: { status: 'ok', score: 0.9 } }, - ], - })), -})); -vi.mock('../../lib/replicate/local-theme/google-fonts.js', () => ({ - selfHostGoogleFonts: vi.fn(async () => ({ faces: [], localizedCss: '', errors: [] })), -})); -vi.mock('../../lib/replicate/local-theme/theme-files.js', async (importOriginal) => { - const actual = await importOriginal<typeof import('../../lib/replicate/local-theme/theme-files.js')>(); - return { - ...actual, - assembleLocalTheme: (opts: Parameters<typeof actual.assembleLocalTheme>[0]) => { - assembleLocalThemeCalls.push(opts); - return actual.assembleLocalTheme(opts); - }, - }; -}); -vi.mock('../../lib/replicate/local-site/jetpack-form-css.js', () => ({ - buildJetpackFormParityCss: buildJetpackFormParityCssMock, -})); -vi.mock('@automattic/blocks-engine/theme', async (importOriginal) => { - const actual = await importOriginal<typeof import('@automattic/blocks-engine/theme')>(); - return { - ...actual, - siteToTheme: vi.fn(async (...args: Parameters<typeof actual.siteToTheme>) => actual.siteToTheme(...args)), - extractSourceLandmarksFromHtml: (html: string) => { - if (regionCensusFailure.throwOnExtract) throw new Error('synthetic region census failure'); - return actual.extractSourceLandmarksFromHtml(html); - }, - }; -}); - -// Repair-loop seam: mock probePair (heavy Playwright + CSS snapshot) while keeping -// FREEZE_MOTION_CSS real so the handler's freezeMotion helper stays functional. -vi.mock('../../lib/replicate/parity/parity-probe.js', async (importOriginal) => { - const actual = await importOriginal<typeof import('../../lib/replicate/parity/parity-probe.js')>(); - return { ...actual, probePair: vi.fn(async () => []) }; -}); - -// Repair-loop seam: stub chromium.launch so the loop never opens a real browser. -// probePair is mocked so the browser stub only needs close() in the finally block. -vi.mock('playwright', async (importOriginal) => { - const actual = await importOriginal<typeof import('playwright')>(); - return { - ...actual, - chromium: { ...actual.chromium, launch: vi.fn(async () => ({ close: vi.fn(async () => {}) })) }, - }; -}); - -const execCalls: string[][] = []; -let execFailFor: string | null = null; -vi.mock('node:child_process', async (importOriginal) => { - const actual = await importOriginal<typeof import('node:child_process')>(); - return { - ...actual, - execFile: vi.fn((cmd: string, args: string[], _opts: unknown, cb: (e: Error | null, r: { stdout: string; stderr: string }) => void) => { - execCalls.push([cmd, ...args]); - const joined = [cmd, ...args].join(' '); - if (execFailFor && joined.includes(execFailFor)) { - cb(new Error(`synthetic exec failure: ${execFailFor}`), { stdout: '', stderr: '' }); - return; - } - // Studio assigns random ports — the handler resolves the replica base URL - // via `wp option get siteurl`; a distinctive port here lets tests assert - // the RESOLVED url (not a hardcoded default) drives replica capture. - cb(null, { stdout: joined.includes('option get siteurl') ? 'http://localhost:7777\n' : '', stderr: '' }); - }), - }; -}); -const installedPosts: Array<{ slug: string; sourceUrl: string; content: string }> = []; -let installFailFor: string | null = null; -vi.mock('../../lib/streaming/post-install.js', () => ({ - installPost: vi.fn(async ({ item }: { item: { slug: string; sourceUrl: string; content: string } }) => { - if (installFailFor && item.slug === installFailFor) { - throw new Error(`synthetic install failure: ${item.slug}`); - } - installedPosts.push({ slug: item.slug, sourceUrl: item.sourceUrl, content: item.content }); - return { sourceUrl: item.sourceUrl, postId: installedPosts.length, action: 'inserted' as const }; - }), -})); - -// Site-finalize seam (mirrors the installPost mock): the handler consolidates -// blogname + _wp_page_template assigns + the front-page pair into ONE -// finalizeSite eval-file call (Studio IPC flakes on bursts of argv commands). -// finalizeCalls captures payloads for assertions; finalizeResultOverride lets -// a test inject per-item errors or a whole-call rejection. -interface FinalizePayloadLike { - options: Record<string, string>; - templateAssigns: Array<{ postId: number; slug: string; template: string }>; - frontPageId?: number; -} -interface FinalizeResultLike { - ok: boolean; - applied: { options: string[]; templates: number[]; frontPage: boolean }; - errors: Array<{ item: string; error: string }>; -} -const finalizeCalls: Array<{ payload: FinalizePayloadLike; studioSitePath: string }> = []; -let finalizeResultOverride: ((payload: FinalizePayloadLike) => Promise<FinalizeResultLike>) | null = null; -vi.mock('../../lib/streaming/site-finalize.js', () => ({ - finalizeSite: vi.fn(async ({ payload, studioSitePath }: { payload: FinalizePayloadLike; studioSitePath: string }) => { - finalizeCalls.push({ payload, studioSitePath }); - if (finalizeResultOverride) return finalizeResultOverride(payload); - // Default: everything in the payload applied successfully. - return { - ok: true, - applied: { - options: Object.keys(payload.options), - templates: payload.templateAssigns.map((t) => t.postId), - frontPage: payload.frontPageId !== undefined, - }, - errors: [], - }; - }), -})); - -import { convertLocalSiteHandler } from './convert-local-site.js'; -import { ingestLocalSiteHandler } from './ingest-local-site.js'; -// Resolves to the vi.mock above — imported so tests can inject one-shot failures. -import { captureScreenshots } from '../../lib/screenshot/screenshotter.js'; -// Resolved vi.mocked instances for per-test once-value injection. -import { compareScreenshotDirs } from '../../lib/screenshot/compare.js'; -import { probePair } from '../../lib/replicate/parity/parity-probe.js'; -import { composedSidecarPath } from '../../lib/streaming/block-markup-validate.js'; -import { EDITABLE_PLUGIN_SLUG } from '../../blocks/editable-html-plugin.js'; -import { siteToTheme } from '@automattic/blocks-engine/theme'; - -const FIXTURE_TMP = join(process.cwd(), '.tmp-test'); - -const ctx = { - textResult: (data: unknown): ToolResult => ({ content: [{ type: 'text', text: JSON.stringify(data) }] }), - errorResult: (message: string): ToolResult => ({ content: [{ type: 'text', text: message }], isError: true }), -} as unknown as HandlerContext; - -function makeSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-site-')); - writeFileSync( - join(dir, 'index.html'), - // Footer anchor sits INSIDE the <p> — bare-<a> direct children hit emitChild's - // catch-all downgrade (href dropped; known limitation, tracked separately) and - // would mask the permalink-rewrite wiring this fixture exists to prove. - // Stage 1d: includes <link> + <script> so collectSourceAssets has linked assets to carry. - '<html><head><title>Home

Hi

', - ); - writeFileSync( - join(dir, 'about.html'), - 'About

Who

Us

', - ); - // Stage 1d carry: linked CSS + JS the collector picks up from the index.html document order. - writeFileSync(join(dir, 'styles.css'), 'body { background: #f7f2e9; }\n.hero h1 { font-size: 4rem; }'); - writeFileSync(join(dir, 'site.js'), "document.documentElement.classList.add('js');"); - return dir; -} - -function makeFixedSidebarOffsetSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-fixed-sidebar-offset-')); - writeFileSync( - join(dir, 'index.html'), - 'Docs' + - '
' + - '

Docs

Welcome.

' + - '', - ); - writeFileSync( - join(dir, 'about.html'), - 'About

About

', - ); - writeFileSync( - join(dir, 'styles.css'), - '.layout{display:flex}.sidebar{position:fixed;width:268px}.main-area{margin-left:268px}', - ); - return dir; -} - -function makeEditableIslandSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-editable-')); - writeFileSync( - join(dir, 'index.html'), - 'Editable
' + - '

Hi

' + - '
', - ); - return dir; -} - -function makeCarriedHeaderSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-carried-header-')); - writeFileSync( - join(dir, 'index.html'), - 'Home

Reviews

Hi

', - ); - writeFileSync( - join(dir, 'reviews.html'), - 'Reviews

Reviews

Five stars.

', - ); - writeFileSync(join(dir, 'styles.css'), '.bp-header { display: flex; gap: 1rem; }'); - return dir; -} - -function makeHeaderWithOverlayMountSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-header-overlay-mount-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '' + - '

Overview

Welcome.

' + - '' + - '', - ); - writeFileSync(join(dir, 'intro.html'), 'Intro

Intro

'); - writeFileSync(join(dir, 'api.html'), 'API

API

'); - writeFileSync(join(dir, 'styles.css'), '.site-header { display: flex; gap: 1rem; } #nav-overlay { position: fixed; inset: 0; }'); - writeFileSync(join(dir, 'site.js'), "document.documentElement.classList.add('js-ready');"); - return dir; -} - -function makeEmptyHeaderMountOnlySite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-empty-header-mount-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '

Hi

' + - '' + - '', - ); - writeFileSync(join(dir, 'styles.css'), '.runtime-header { min-height: 1px; }'); - writeFileSync(join(dir, 'site.js'), "document.getElementById('siteHeader')?.setAttribute('data-rendered', '1');"); - return dir; -} - -function makeSideRailSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-side-rail-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '

Overview

Welcome.

' + - '', - ); - writeFileSync( - join(dir, 'intro.html'), - 'Intro

Intro

', - ); - writeFileSync( - join(dir, 'api.html'), - 'API

API

', - ); - writeFileSync(join(dir, 'styles.css'), '.docs-sidebar { position: sticky; top: 0; }'); - return dir; -} - -function makeHeaderOverlaySideRailSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-header-overlay-side-rail-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '' + - '

Overview

Welcome.

' + - '' + - '', - ); - writeFileSync(join(dir, 'intro.html'), 'Intro

Intro

'); - writeFileSync(join(dir, 'api.html'), 'API

API

'); - writeFileSync(join(dir, 'styles.css'), '.site-header { display: flex; } .docs-sidebar { position: sticky; top: 0; }'); - writeFileSync(join(dir, 'site.js'), "document.documentElement.classList.add('js-ready');"); - return dir; -} - -function makeNavOverlayMountSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-nav-overlay-mount-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '' + - '

Overview

Welcome.

' + - '' + - '', - ); - writeFileSync(join(dir, 'intro.html'), 'Intro

Intro

'); - writeFileSync(join(dir, 'api.html'), 'API

API

'); - writeFileSync(join(dir, 'styles.css'), '#primary-nav { display: flex; gap: 1rem; }'); - writeFileSync(join(dir, 'site.js'), "document.documentElement.classList.add('js-ready');"); - return dir; -} - -function makeHeaderWithDroppedStandaloneNavSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-header-dropped-standalone-nav-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '' + - '

Hi

Body content survives.

' + - '', - ); - writeFileSync(join(dir, 'about.html'), 'About

About

'); - writeFileSync(join(dir, 'docs.html'), 'Docs

Docs

'); - writeFileSync(join(dir, 'intro.html'), 'Intro

Intro

'); - writeFileSync(join(dir, 'api.html'), 'API

API

'); - writeFileSync(join(dir, 'styles.css'), '.site-header { display: flex; gap: 1rem; } #standalone-nav { display: flex; gap: 1rem; }'); - return dir; -} - -function makeHeaderWithStandaloneNavSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-standalone-nav-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '' + - '

Hi

' + - '', - ); - writeFileSync( - join(dir, 'about.html'), - 'About

About

', - ); - writeFileSync( - join(dir, 'standalone.html'), - 'Standalone

Standalone

', - ); - writeFileSync(join(dir, 'styles.css'), '.site-header { display: flex; gap: 1rem; }'); - return dir; -} - -function makeInteriorRailLeakSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-conservation-leak-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '

Home

The home body is present in the emitted page content.

' + - '', - ); - writeFileSync( - join(dir, 'reference.html'), - 'Reference' + - '' + - '' + - '
' + - '

Reference

The reference body is present in the emitted page content.

' + - '
' + - '', - ); - writeFileSync(join(dir, 'styles.css'), '.site-header { display: flex; } .side-rail { position: sticky; top: 0; }'); - return dir; -} - -function makeInteriorChromeRailSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-interior-chrome-rail-')); - const header = ''; - const sidebar = (toc: string) => - ''; - writeFileSync( - join(dir, 'index.html'), - 'Home' + - header + - '

Home

The home body is present.

' + - '', - ); - writeFileSync( - join(dir, 'intro.html'), - 'Intro' + - header + - '
' + - sidebar('Intro start') + - '

Intro

The intro body is present.

' + - '
' + - '', - ); - writeFileSync( - join(dir, 'api.html'), - 'API' + - header + - '
' + - sidebar('API start') + - '

API

The API body is present.

' + - '
' + - '', - ); - writeFileSync(join(dir, 'styles.css'), '.site-header { display: flex; } .sidebar { position: fixed; top: 0; } .sidebar-nav { display: grid; }'); - return dir; -} - -function makeInteriorLayoutWrapperRailSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-interior-layout-wrapper-')); - const header = ''; - writeFileSync( - join(dir, 'index.html'), - 'Home' + - header + - '

Home

Home body.

' + - '', - ); - writeFileSync( - join(dir, 'intro.html'), - 'Intro' + - header + - '
' + - '' + - '

Intro

The intro body is present.

' + - '
' + - '', - ); - writeFileSync(join(dir, 'styles.css'), '.docs-grid { display: grid; grid-template-columns: 16rem 1fr; } .sidebar { position: sticky; top: 0; }'); - return dir; -} - -function makeInteriorRailWithoutSharedWrapperSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-interior-flat-rail-')); - const header = ''; - writeFileSync( - join(dir, 'index.html'), - 'Home' + - header + - '

Home

Home body.

' + - '', - ); - writeFileSync( - join(dir, 'intro.html'), - 'Intro' + - header + - '
' + - '' + - '
' + - '

Intro

The intro body is present.

' + - '', - ); - writeFileSync(join(dir, 'styles.css'), '.rail-shell { display: contents; } .sidebar { position: sticky; top: 0; }'); - return dir; -} - -function makeHomeComplementaryRailSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-home-complementary-rail-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '

Home

The home body remains installed.

' + - '', - ); - writeFileSync(join(dir, 'intro.html'), 'Intro

Intro

'); - writeFileSync(join(dir, 'api.html'), 'API

API

'); - return dir; -} - -function makeRepeatedComplementaryBodySite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-repeated-complementary-body-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '

First rail

IntroAPI
' + - '

Second rail

IntroContact
' + - '

Home

The home body remains installed.

' + - '', - ); - writeFileSync(join(dir, 'intro.html'), 'Intro

Intro

'); - writeFileSync(join(dir, 'api.html'), 'API

API

'); - writeFileSync(join(dir, 'contact.html'), 'Contact

Contact

'); - return dir; -} - -function makeHomePlainAsideSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-home-plain-aside-')); - writeFileSync( - join(dir, 'index.html'), - 'Home' + - '' + - '

Home

The home body remains installed.

' + - '', - ); - return dir; -} - -function makeFormSite(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-form-')); - writeFileSync( - join(dir, 'index.html'), - 'Contact

Contact

' + - '
' + - '' + - '' + - '' + - '' + - '' + - '' + - '' + - '
', - ); - writeFileSync(join(dir, 'styles.css'), '.contact-form label { display: block; }\n.contact-form button { background: #2255aa; }'); - return dir; -} - -/** Like makeSite but WITHOUT index.html — no 'home' slug, so no front page. */ -function makeSiteNoHome(): string { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const dir = mkdtempSync(join(FIXTURE_TMP, 'cls-nohome-')); - writeFileSync( - join(dir, 'about.html'), - 'About

Who

Us

', - ); - return dir; -} - -function makeStudioSite(): string { - const sitePath = mkdtempSync(join(FIXTURE_TMP, 'cls-studio-')); - mkdirSync(join(sitePath, 'wp-content'), { recursive: true }); - return sitePath; -} - -function wpArgsForExecCall(call: string[]): string[] { - const pathIndex = call.indexOf('--path'); - return pathIndex >= 0 ? call.slice(pathIndex + 2) : []; -} - -function jetpackWpCalls(): string[][] { - return execCalls.filter((call) => wpArgsForExecCall(call).some((arg) => arg.toLowerCase().includes('jetpack'))); -} - -function jetpackInstallCalls(): string[][] { - const expected = Array.from(JETPACK_FORMS_PLUGIN_INSTALL.wpArgs); - return execCalls.filter((call) => { - const wpArgs = wpArgsForExecCall(call); - return wpArgs.length === expected.length && expected.every((arg, index) => wpArgs[index] === arg); - }); -} - -// Base compareScreenshotDirs result — re-established each test so once-values -// from repair tests don't leak (repair tests chain mockResolvedValueOnce on top). -const BASE_COMPARE_RESULT = { - version: 1, - comparedAt: 'TEST', - results: [ - { pathname: '/', originUrl: 'o', replicaUrl: 'r', desktop: { status: 'ok', score: 0.91 }, mobile: { status: 'ok', score: 0.88 } }, - { pathname: '/about/', originUrl: 'o', replicaUrl: 'r', desktop: { status: 'ok', score: 0.95 }, mobile: { status: 'ok', score: 0.9 } }, - ], -}; - -beforeEach(() => { - execCalls.length = 0; - installedPosts.length = 0; - capturedRuns.length = 0; - finalizeCalls.length = 0; - execFailFor = null; - installFailFor = null; - finalizeResultOverride = null; - assembleLocalThemeCalls.length = 0; - // Repair seam: reset per-test; repair tests set these before calling handler. - repairReplicaManifestEntries = {}; - repairDiffPng = null; - regionCensusFailure.throwOnExtract = false; - // Reset + re-establish base for compare and probePair so unconsumed once-values - // from a failing repair test can't bleed into subsequent tests. - vi.mocked(compareScreenshotDirs).mockReset().mockResolvedValue(BASE_COMPARE_RESULT as unknown as Awaited>); - vi.mocked(probePair).mockReset().mockResolvedValue([]); - vi.mocked(siteToTheme).mockClear(); - buildJetpackFormParityCssMock.mockReset().mockReturnValue({ css: '' }); -}); - -describe('convertLocalSiteHandler', () => { - it('editableIslands: ingest converts text islands into bindable blocks', async () => { - const dir = makeEditableIslandSite(); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'cls-editable-out-')); - try { - const res = await ingestLocalSiteHandler({ dir, outputDir: outDir, editableIslands: true }, ctx); - expect(res.isError).toBeFalsy(); - const summary = JSON.parse(res.content[0].text) as { islandsConverted?: number }; - expect(summary.islandsConverted).toBeGreaterThan(0); - - const sidecar = readFileSync(composedSidecarPath(outDir, 'home'), 'utf8'); - expect(sidecar).toContain(''); - expect(sidecar).not.toMatch( - /\s*]*(?:id="contact-form"|class="contact-form")[\s\S]*?/, - ); - const summary = JSON.parse(res.content[0].text) as { formsConverted: number }; - expect(summary.formsConverted).toBeGreaterThanOrEqual(1); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('returns an error result for a dir with no html', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'empty-')); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: siteDir }, ctx); - expect(res.isError).toBe(true); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - } - }); - - it('summary and report include failure/empty fields on happy path', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site2-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out2-')); - writeFileSync(join(siteDir, 'index.html'), '

Page One

'); - writeFileSync(join(siteDir, 'about.html'), '

About

'); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir }, ctx); - expect(res.isError).toBeFalsy(); - const summary = JSON.parse(res.content[0].text) as { - pages: number; failedPageCount: number; failedPagesList: unknown[]; emptyPages: unknown[]; - }; - expect(summary.pages).toBe(2); - expect(summary.failedPageCount).toBe(0); - expect(summary.failedPagesList).toEqual([]); - expect(summary.emptyPages).toEqual([]); - const report = JSON.parse(readFileSync(join(outDir, 'normalize-report.json'), 'utf8')) as { - failedPages: unknown[]; emptyPages: unknown[]; - }; - expect(report.failedPages).toEqual([]); - expect(report.emptyPages).toEqual([]); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('filters static-card mounts to their source page before neutralizing', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-card-scope-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-card-scope-')); - const mount: MountSpec = { - selector: '#dla-cards-index', - sourceSelector: '.ledger-grid', - sourcePage: 'index.html', - sourceCall: 'html-cards:.ledger-grid', - query: { postType: 'post', perPage: -1, orderBy: 'date', order: 'ASC' }, - }; - writeFileSync( - join(siteDir, 'index.html'), - '

Journal

' + - '

Alpha

Alpha card text long enough.

' + - '

Beta

Beta card text long enough.

' + - '

Gamma

Gamma card text long enough.

' + - '
', - ); - writeFileSync( - join(siteDir, 'about.html'), - '

About

' + - '

Mission

Studio mission text that should remain prose.

' + - '

Process

Studio process text that should remain prose.

' + - '

Team

Studio team text that should remain prose.

' + - '
', - ); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir, cardMounts: [mount] }, ctx); - expect(res.isError).toBeFalsy(); - const homeSidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - const aboutSidecar = readFileSync(join(outDir, 'composed', 'about.blocks.html'), 'utf8'); - expect(homeSidecar).toContain('id="dla-cards-index"'); - expect(aboutSidecar).not.toContain('id="dla-cards-index"'); - expect(aboutSidecar).toContain('Mission'); - expect(aboutSidecar).toContain('Process'); - expect(aboutSidecar).toContain('Team'); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('does not apply a top-level index mount to a nested same-basename page', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-card-basename-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-card-basename-')); - const mount: MountSpec = { - selector: '#dla-cards-index', - sourceSelector: '.ledger-grid', - sourcePage: 'index.html', - sourceCall: 'html-cards:.ledger-grid', - query: { postType: 'post', perPage: -1, orderBy: 'date', order: 'ASC' }, - }; - mkdirSync(join(siteDir, 'blog'), { recursive: true }); - writeFileSync( - join(siteDir, 'index.html'), - '

Journal

' + - '

Alpha

Alpha card text long enough.

' + - '

Beta

Beta card text long enough.

' + - '

Gamma

Gamma card text long enough.

' + - '
', - ); - writeFileSync( - join(siteDir, 'blog', 'index.html'), - '

Blog

' + - '

Nested Editorial Alpha

Nested editorial alpha must survive untouched.

' + - '

Nested Editorial Beta

Nested editorial beta must survive untouched.

' + - '

Nested Editorial Gamma

Nested editorial gamma must survive untouched.

' + - '
', - ); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir, cardMounts: [mount] }, ctx); - expect(res.isError).toBeFalsy(); - const homeSidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - const blogSidecar = readFileSync(join(outDir, 'composed', 'blog.blocks.html'), 'utf8'); - expect(homeSidecar).toContain('id="dla-cards-index"'); - expect(blogSidecar).not.toContain('id="dla-cards-index"'); - expect(blogSidecar).toContain('Nested Editorial Alpha'); - expect(blogSidecar).toContain('Nested editorial beta must survive untouched.'); - expect(blogSidecar).toContain('Nested Editorial Gamma'); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('isolates a per-page compose failure: other pages still compose', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site3-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out3-')); - writeFileSync(join(siteDir, 'index.html'), '

Fine

'); - writeFileSync(join(siteDir, 'boom.html'), '

Kaboom

'); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir }, ctx); - expect(res.isError).toBeFalsy(); - const summary = JSON.parse(res.content[0].text) as { - pages: number; failedPageCount: number; failedPagesList: Array<{ slug: string; error: string }>; - }; - expect(summary.pages).toBe(2); - expect(summary.failedPageCount).toBe(1); - expect(summary.failedPagesList).toEqual([{ slug: 'boom', error: 'synthetic compose failure' }]); - expect(existsSync(join(outDir, 'composed', 'home.blocks.html'))).toBe(true); - expect(existsSync(join(outDir, 'composed', 'boom.blocks.html'))).toBe(false); - const report = JSON.parse(readFileSync(join(outDir, 'normalize-report.json'), 'utf8')) as { - failedPages: Array<{ slug: string; error: string }>; - }; - expect(report.failedPages).toEqual([{ slug: 'boom', error: 'synthetic compose failure' }]); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('rejects an outputDir containing .. traversal', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site4-')); - writeFileSync(join(siteDir, 'index.html'), '

Hi

'); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: '../escape' }, ctx); - expect(res.isError).toBe(true); - expect(res.content[0].text).toMatch(/traversal/); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - } - }); - - it('reports pages that compose to nothing in emptyPages and still writes their sidecar', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site5-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out5-')); - writeFileSync(join(siteDir, 'index.html'), '

Hi

'); - writeFileSync(join(siteDir, 'bare.html'), '

chrome only

'); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir }, ctx); - expect(res.isError).toBeFalsy(); - const summary = JSON.parse(res.content[0].text) as { pages: number; emptyPages: string[] }; - expect(summary.pages).toBe(2); - expect(summary.emptyPages).toEqual(['bare']); - expect(existsSync(join(outDir, 'composed', 'bare.blocks.html'))).toBe(true); - expect(readFileSync(join(outDir, 'composed', 'bare.blocks.html'), 'utf8')).toBe(''); - const report = JSON.parse(readFileSync(join(outDir, 'normalize-report.json'), 'utf8')) as { emptyPages: string[] }; - expect(report.emptyPages).toEqual(['bare']); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('nativeBehaviors: detects reveal from source assets and tags sidecar sections', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-nb-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-nb-')); - writeFileSync( - join(siteDir, 'index.html'), - '

Hi

', - ); - writeFileSync( - join(siteDir, 'styles.css'), - 'html.js section { opacity: 0; transform: translateY(18px); transition: opacity 600ms ease, transform 600ms ease; }', - ); - writeFileSync( - join(siteDir, 'site.js'), - "const obs = new IntersectionObserver((es) => es.forEach((e) => e.isIntersecting && e.target.classList.add('is-visible')), { threshold: 0.12 });\n" + - "document.querySelectorAll('section').forEach((s) => obs.observe(s));\n", - ); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir, nativeBehaviors: true }, ctx); - expect(res.isError).toBeFalsy(); - const sidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - expect(sidecar).toContain('wp:dla/reveal'); - expect(sidecar).toContain('data-wp-interactive="dla/reveal"'); - expect(sidecar).not.toContain('wp:group'); - const report = JSON.parse(readFileSync(join(outDir, 'normalize-report.json'), 'utf8')) as { - entries: Array<{ blockType: string }>; - }; - expect(report.entries.every((e) => e.blockType === 'dla/reveal')).toBe(true); - // Standalone observability: the summary surfaces what detection found - // (no artifact write — behavior-gaps.json stays the convert stage's). - const summary = JSON.parse(res.content[0].text) as { behaviors?: { reveal: boolean; gaps: number } }; - expect(summary.behaviors).toEqual({ reveal: true, tabs: 0, slider: 0, modal: 0, gaps: 0 }); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('nativeBehaviors: per-section detection tags tabs sidecars and counts ride the summary', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-nbtabs-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-nbtabs-')); - writeFileSync( - join(siteDir, 'index.html'), - '
' + - '

Hi

' + - '
' + - '' + - '
' + - '

Alpha

' + - '
' + - '
', - ); - writeFileSync( - join(siteDir, 'styles.css'), - 'html.js section { opacity: 0; transform: translateY(18px); transition: opacity 600ms ease, transform 600ms ease; }', - ); - writeFileSync( - join(siteDir, 'site.js'), - "const obs = new IntersectionObserver((es) => es.forEach((e) => e.isIntersecting && e.target.classList.add('is-visible')), { threshold: 0.12 });\n" + - "document.querySelectorAll('section').forEach((s) => obs.observe(s));\n" + - "document.querySelectorAll('[role=\"tab\"]').forEach((t) => t.addEventListener('click', () => {\n" + - " t.classList.add('is-active');\n" + - '}));\n', - ); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir, nativeBehaviors: true }, ctx); - expect(res.isError).toBeFalsy(); - const sidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - expect(sidecar).toContain('data-wp-interactive="dla/tabs"'); // specific section - expect(sidecar).toContain('data-wp-interactive="dla/reveal"'); // uniform fallback - expect(sidecar).toContain('role="tab"'); // verbatim inner - // Counts from the compose reports; the tabs driver js is CLAIMED once - // its section fired, so it does not inflate gaps. - const summary = JSON.parse(res.content[0].text) as { behaviors?: Record }; - expect(summary.behaviors).toEqual({ reveal: true, tabs: 1, slider: 0, modal: 0, gaps: 0 }); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('nativeBehaviors with no catalog match leaves sections as group', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-nbnone-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-nbnone-')); - // No reveal css gate, no observer js — detection finds nothing to map. - writeFileSync(join(siteDir, 'index.html'), '

Hi

'); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir, nativeBehaviors: true }, ctx); - expect(res.isError).toBeFalsy(); - const sidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - expect(sidecar).toContain('wp:group'); - expect(sidecar).not.toContain('dla/reveal'); - // No-match shape: key present (flag on), nothing found. - const summary = JSON.parse(res.content[0].text) as { behaviors?: { reveal: boolean; gaps: number } }; - expect(summary.behaviors).toEqual({ reveal: false, tabs: 0, slider: 0, modal: 0, gaps: 0 }); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('default ingest (no flag) never tags (regression)', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-nboff-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-nboff-')); - // Source HAS the reveal patterns, but the flag is off — no detection runs. - writeFileSync( - join(siteDir, 'index.html'), - '

Hi

', - ); - writeFileSync(join(siteDir, 'styles.css'), 'html.js section { opacity: 0; }'); - writeFileSync( - join(siteDir, 'site.js'), - "const obs = new IntersectionObserver((es) => es.forEach((e) => e.target.classList.add('is-visible')));\ndocument.querySelectorAll('section').forEach((s) => obs.observe(s));\n", - ); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir }, ctx); - expect(res.isError).toBeFalsy(); - const sidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - expect(sidecar).toContain('wp:group'); - expect(sidecar).not.toContain('dla/reveal'); - // Flag off → key absent (default summary byte-stable). - const summary = JSON.parse(res.content[0].text) as { behaviors?: unknown }; - expect(summary.behaviors).toBeUndefined(); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it('carry ingest (no flag): interactive scaffolding survives VERBATIM in a group wrapper', async () => { - mkdirSync(FIXTURE_TMP, { recursive: true }); - const siteDir = mkdtempSync(join(FIXTURE_TMP, 'site-carrytabs-')); - const outDir = mkdtempSync(join(FIXTURE_TMP, 'out-carrytabs-')); - // Tabs DOM pattern + its JS driver, flag OFF: the carry path must keep the - // scaffolding byte-true (emitChild's catch-all destroyed it — carry E2E - // unresolved missing tab/panel structural divergences) inside a plain - // group wrapper with no plugin dependency. - writeFileSync( - join(siteDir, 'index.html'), - '
' + - '
' + - '' + - '
' + - '

Alpha

' + - '
' + - '
', - ); - writeFileSync( - join(siteDir, 'site.js'), - 'document.querySelectorAll(\'[role="tab"]\').forEach((t) => t.addEventListener(\'click\', () => {\n' + - " t.classList.add('is-active');\n" + - '}));\n', - ); - try { - const res = await ingestLocalSiteHandler({ dir: siteDir, outputDir: outDir }, ctx); - expect(res.isError).toBeFalsy(); - const sidecar = readFileSync(join(outDir, 'composed', 'home.blocks.html'), 'utf8'); - expect(sidecar).toContain('role="tab"'); - expect(sidecar).toContain('aria-controls="p-a"'); - expect(sidecar).toContain('wp:group'); - // editable-html islands are the default now: the verbatim interactive scaffolding - // survives inside a dla/editable-html block (static save = byte-identical HTML), so - // role/aria above are still present. nativeBehaviors stays flag-gated: no behavior blocks. - expect(sidecar).not.toContain('dla/reveal'); - expect(sidecar).not.toContain('dla/sticky'); - expect(sidecar).not.toContain('data-wp-interactive'); - // Summary stays flag-gated. - const summary = JSON.parse(res.content[0].text) as { behaviors?: unknown }; - expect(summary.behaviors).toBeUndefined(); - } finally { - rmSync(siteDir, { recursive: true, force: true }); - rmSync(outDir, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/ingest-local-site.ts b/packages/data-liberation-agent/src/mcp-server/handlers/ingest-local-site.ts deleted file mode 100644 index 916ddd9632..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/ingest-local-site.ts +++ /dev/null @@ -1,229 +0,0 @@ -// -// liberate_ingest_local_site -// ========================== -// Stage 1a of the owned-source path: ingest a local static-site directory, -// normalize each page into native block markup (validated by the roundtrip -// oracle), and write composed sidecars + a normalize-report. No Playwright, -// no Studio. Downstream stages (theme-scaffold, install, compare) consume -// the composed sidecars. -// -import { mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import type { Handler } from '../handler-types.js'; -import { validateOutputDir } from '../../lib/screenshot/output-layout.js'; -import { composedSidecarPath, instanceStylesPath } from '../../lib/streaming/block-markup-validate.js'; -import { BlockFixerClient } from '../../lib/streaming/block-fixer-client.js'; -import { ingestLocalSite } from '../../lib/replicate/local-site/ingest.js'; -import { composePage } from '../../lib/replicate/normalize/compose-page.js'; -import { makeIslandsEditable } from '../../lib/replicate/normalize/make-islands-editable.js'; -import { InstanceStyleSheet } from '../../lib/replicate/normalize/instance-styles.js'; -import { neutralizeStaticCards } from '../../lib/replicate/local-data/neutralize-static-cards.js'; -import type { MountSpec } from '../../lib/replicate/local-data/types.js'; -import { - detectBehaviors, - detectSectionBehavior, - type BehaviorSourceAssets, -} from '../../lib/replicate/normalize/detect-behaviors.js'; -import { collectSourceAssets } from '@automattic/blocks-engine/theme'; -import type { - NormalizeReportEntry, - RevealBehavior, - Section, - SectionBehavior, -} from '../../lib/replicate/local-site/types.js'; - -// v2: the report envelope gained `stylingDrops` (styling-conservation diagnostic). -const NORMALIZE_REPORT_SCHEMA = 2; - -export const ingestLocalSiteHandler: Handler = async (args, ctx) => { - const dir = args.dir as string | undefined; - const outputDir = (args.outputDir as string | undefined) ?? dir; - const nativeBehaviors = args.nativeBehaviors === true; - // Default ON: core/html islands convert to in-canvas dla/editable-html (visible + styled - // in the editor). Opt OUT with editableIslands:false to force plain core/html. - const editableIslands = args.editableIslands !== false; - const cardMounts = (args.cardMounts as MountSpec[] | undefined) ?? []; - if (!dir) return ctx.errorResult('dir is required'); - if (!outputDir) return ctx.errorResult('outputDir is required'); - try { - validateOutputDir(outputDir); - } catch (err) { - return ctx.errorResult((err as Error).message); - } - - let site; - try { - site = ingestLocalSite(dir); - } catch (err) { - return ctx.errorResult(`ingest failed: ${(err as Error).message}`); - } - - // Per-section DOM-pattern detection runs on BOTH paths (pure + regex-fast): - // a tagged section keeps its inner VERBATIM — content survival is path- - // independent (carry E2E: emitChild destroyed tab/panel scaffolding and the - // carried source JS had nothing to drive). `native` only decides the - // WRAPPER: dla/ directives (nativeBehaviors) vs a plain core/group. - // reveal stays flag-gated — it changes visuals via the plugin, which the - // carry path never installs. Detection consumes the RAW collected css — - // assets.css has WP_COMPAT_CSS prepended, which is detection-immune (no - // html.js section gate, no scroll-listener patterns). The convert handler - // re-runs the same pure detection for sticky/gaps/plugin wiring — - // identical inputs, identical result (deterministic). - const assets = collectSourceAssets(dir, site.pages.map((p) => ({ relPath: p.relPath, html: p.html }))); - const assetSlice: BehaviorSourceAssets = { css: assets.css, js: assets.js }; - const detectSection = (s: Section): SectionBehavior | undefined => detectSectionBehavior(s.html, assetSlice); - let reveal: RevealBehavior | undefined; - if (nativeBehaviors) { - reveal = detectBehaviors(assetSlice).reveal; - } - - mkdirSync(join(outputDir, 'composed'), { recursive: true }); - - const entries: Array = []; - const failedPages: Array<{ slug: string; error: string }> = []; - const emptyPages: string[] = []; - let formsConverted = 0; - let islandsConverted = 0; - // Warning-level block-contract issues (emitter-bug dial — see block-contract.ts). - const contractIssues: Array<{ slug: string; code: string; blockName: string; detail: string }> = []; - // Warning-level styling-conservation drops: source classes a section's block - // conversion dropped (the carried CSS that targeted them no longer matches). - const stylingDrops: Array<{ slug: string; sectionId: string; droppedClasses: string[] }> = []; - - // One sheet across all pages: per-element inline styles are carried as - // lib-i classes (fixer-safe) + deduped stylesheet rules emitted to - // composed/instance-styles.css, which the convert stage ships + enqueues on - // BOTH the frontend and the editor canvas. - const instanceStyles = new InstanceStyleSheet(); - - // Canonicalize each page's composed markup through @wordpress/blocks (the - // real block save() functions) before writing the sidecar, so the carried - // blocks validate cleanly in the editor (no recovery / "unexpected content"). - // Best-effort: fix() passes the markup through unchanged if the sidecar can't - // start — the markup is already emitted fixer-valid by construction. - const blockFixer = new BlockFixerClient(); - await blockFixer.start().catch(() => { - /* best-effort — fix() passes through if the server didn't start */ - }); - - try { - for (const page of site.pages) { - // Per-page isolation: one bad page (roundtrip failure / compose misfit) - // must not abort the whole ingest — record it and keep going. - try { - const pageCardMounts = cardMounts.filter((m) => !m.sourcePage || m.sourcePage === page.relPath); - const neutralized = pageCardMounts.length > 0 ? neutralizeStaticCards(page.html, pageCardMounts) : undefined; - const composeInput = neutralized?.stamped.length ? { ...page, html: neutralized.html } : page; - const composed = composePage(composeInput, { - reveal, - detectSection, - native: nativeBehaviors, - // Internal .html hrefs in page bodies → /slug/ permalinks at emission. - pageSlugs: site.pages.map((sp) => sp.slug), - instanceStyles, - // Carry fidelity: keep inline-svg icons (search magnifier, card/meta - // glyphs), button menus, and empty CSS-background hooks verbatim — - // the block conversion otherwise silently drops them. Query-loop - // mounts are excluded (id-bearing), so the data path is unaffected. - verbatimInteractive: true, - jetpackForms: true, - }); - formsConverted += composed.formsConverted; - let { postContent } = composed; - const { report } = composed; - if (editableIslands) { - const editable = makeIslandsEditable(postContent); - postContent = editable.content; - islandsConverted += editable.converted; - } - if (postContent === '' && report.length === 0) emptyPages.push(page.slug); - const fixed = (await blockFixer.fix([postContent]))[0]; - const finalContent = fixed?.html ?? postContent; - writeFileSync(composedSidecarPath(outputDir, page.slug), finalContent); - for (const r of report) entries.push({ ...r, slug: page.slug }); - for (const issue of composed.contractIssues) contractIssues.push({ slug: page.slug, ...issue }); - for (const drop of composed.stylingDrops) stylingDrops.push({ slug: page.slug, ...drop }); - } catch (err) { - failedPages.push({ slug: page.slug, error: (err as Error).message }); - } - } - } finally { - await blockFixer.stop().catch(() => { - /* best-effort cleanup */ - }); - } - - // Persist the carried instance-style rules (atomic tmp+rename) for the convert - // stage. Always written (empty file when nothing was carried) so convert has a - // deterministic read target; an empty sheet emits no rules. - const instanceCssPath = instanceStylesPath(outputDir); - const instanceCssTmp = `${instanceCssPath}.tmp.${process.pid}`; - try { - writeFileSync(instanceCssTmp, instanceStyles.toCss()); - renameSync(instanceCssTmp, instanceCssPath); - } catch (err) { - try { unlinkSync(instanceCssTmp); } catch { /* ignore */ } - return ctx.errorResult(`failed to write instance-styles.css: ${(err as Error).message}`); - } - - // Per-kind counts come from the compose REPORTS (single source of truth — - // no re-detection drift), then the global detection RE-RUNS with the fired - // kinds so their driver js is claimed out of the gap residue. The second - // pass is pure + regex-fast and deterministic (same strings in → same - // reveal/sticky out); only residue claiming differs, and sectionKinds can - // only exist AFTER compose produced the reports — hence two passes. - let behaviorsSummary: - | { reveal: boolean; tabs: number; slider: number; modal: number; gaps: number } - | undefined; - if (nativeBehaviors) { - const countOf = (kind: 'tabs' | 'slider' | 'modal'): number => - entries.filter((e) => e.blockType === `dla/${kind}`).length; - const tabs = countOf('tabs'); - const slider = countOf('slider'); - const modal = countOf('modal'); - const sectionKinds = new Set<'tabs' | 'slider' | 'modal'>(); - if (tabs > 0) sectionKinds.add('tabs'); - if (slider > 0) sectionKinds.add('slider'); - if (modal > 0) sectionKinds.add('modal'); - const final = detectBehaviors(assetSlice, { sectionKinds }); - behaviorsSummary = { reveal: !!final.reveal, tabs, slider, modal, gaps: final.gaps.length }; - } - - // Atomic write (tmp + rename) — a crash mid-write must not leave a torn - // normalize-report.json behind. The composed/ recursive mkdir above already - // guarantees outputDir exists. - const reportPath = join(outputDir, 'normalize-report.json'); - const tmpPath = `${reportPath}.tmp.${process.pid}`; - try { - writeFileSync( - tmpPath, - JSON.stringify({ schema: NORMALIZE_REPORT_SCHEMA, site: dir, entries, failedPages, emptyPages, contractIssues, stylingDrops }, null, 2), - ); - renameSync(tmpPath, reportPath); - } catch (err) { - try { unlinkSync(tmpPath); } catch { /* ignore */ } - return ctx.errorResult(`failed to write normalize-report: ${(err as Error).message}`); - } - - return ctx.textResult({ - pages: site.pages.length, - sections: entries.length, - lowConfidence: entries.filter((e) => e.confidence < 1).length, - failedPageCount: failedPages.length, - failedPagesList: failedPages, - emptyPages, - reportPath, - contractIssues: contractIssues.length, - // Styling-conservation: sections whose conversion dropped a source class - // (detail in normalize-report.json). 0 = every source class survived. - stylingDrops: stylingDrops.length, - formsConverted, - ...(editableIslands ? { islandsConverted } : {}), - // Per-instance inline styles carried as lib-i classes + rules (editor-valid). - instanceStyleRules: instanceStyles.size, - // Standalone observability (key absent when the flag is off): what - // detection found + per-kind section counts from the compose reports. - // No artifact write here — behavior-gaps.json belongs to the convert stage. - ...(behaviorsSummary !== undefined ? { behaviors: behaviorsSummary } : {}), - }); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/inspect.ts b/packages/data-liberation-agent/src/mcp-server/handlers/inspect.ts deleted file mode 100644 index 79e40bf347..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/inspect.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { detect } from '../../lib/detect-platform/index.js'; -import { fetchSitemap, classifyUrl } from '../../lib/extraction/sitemap.js'; -import type { Handler } from '../handler-types.js'; - -export const inspectHandler: Handler = async (args, ctx) => { - const detection = await detect(args.url as string); - const result: Record = { - url: args.url, - platform: detection.platform, - confidence: detection.confidence, - signals: detection.signals, - sitemapFound: false, - urlCount: 0, - counts: {} as Record, - probeResults: [], - authRequired: false, - extractionFeasibility: detection.platform === 'unknown' ? 'limited' : 'ready', - }; - - const urls = await fetchSitemap(args.url as string); - result.sitemapFound = urls.length > 0; - result.urlCount = urls.length; - - const counts: Record = {}; - for (const url of urls) { - const type = classifyUrl(url); - counts[type] = (counts[type] || 0) + 1; - } - result.counts = counts; - - const adapter = ctx.findAdapter(detection.platform); - if (adapter && typeof adapter.probe === 'function') { - const opts = { token: args.token, cdpPort: args.cdpPort }; - result.probeResults = await adapter.probe(args.url as string, urls.slice(0, 3), opts); - } - - const { detectFeatures } = await import('../../lib/features/detect-features.js'); - const featureUrls = urls.length > 0 ? urls : [args.url as string]; - result.platformFeatures = detectFeatures(detection.platform, featureUrls, []); - - return ctx.textResult(result); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/install-theme.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/install-theme.test.ts deleted file mode 100644 index de321dbae9..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/install-theme.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; -import { - deriveInstallThemeSlug, - resolveInstallThemeSlug, - themeCacheFlushCommands, -} from './install-theme.js'; - -const TMP_ROOT = join(process.cwd(), '.tmp-test', 'install-theme'); -mkdirSync(TMP_ROOT, { recursive: true }); - -describe('deriveInstallThemeSlug', () => { - it('matches the streaming shell theme slug derived from outputDir', () => { - expect(deriveInstallThemeSlug('/tmp/www.swiftlumber.com')).toBe('www-swiftlumber-com-replica'); - }); -}); - -describe('resolveInstallThemeSlug', () => { - it('uses the existing shell theme when the requested slug differs', () => { - const wpRoot = mkdtempSync(join(TMP_ROOT, 'wp-')); - try { - const shellThemeDir = join( - wpRoot, - 'wp-content', - 'themes', - 'www-swiftlumber-com-replica', - ); - mkdirSync(shellThemeDir, { recursive: true }); - writeFileSync(join(shellThemeDir, 'style.css'), '/* shell */', 'utf8'); - - expect(resolveInstallThemeSlug({ - outputDir: '/tmp/www.swiftlumber.com', - requestedThemeSlug: 'swiftlumber-com-replica', - wpRoot, - })).toBe('www-swiftlumber-com-replica'); - } finally { - rmSync(wpRoot, { recursive: true, force: true }); - } - }); - - it('respects the requested slug when no shell theme exists', () => { - const wpRoot = mkdtempSync(join(TMP_ROOT, 'wp-')); - try { - expect(resolveInstallThemeSlug({ - outputDir: '/tmp/www.swiftlumber.com', - requestedThemeSlug: 'swiftlumber-com-replica', - wpRoot, - })).toBe('swiftlumber-com-replica'); - } finally { - rmSync(wpRoot, { recursive: true, force: true }); - } - }); -}); - -describe('themeCacheFlushCommands', () => { - const cmds = themeCacheFlushCommands(); - - it('flushes transients then the object cache before the pattern-file purge', () => { - expect(cmds[0]).toEqual(['transient', 'delete', '--all']); - expect(cmds[1]).toEqual(['cache', 'flush']); - }); - - it('explicitly purges the wp_theme_files_patterns transient so re-installed patterns re-register', () => { - // Regression guard: a newly-added per-page pattern stays UNregistered (its - // wp:pattern renders empty) unless this DB-backed transient is cleared — - // `cache flush` alone does not remove it on a non-persistent object cache. - const purge = cmds.find((c) => c[0] === 'eval'); - expect(purge).toBeDefined(); - expect(purge![1]).toContain('_transient_wp_theme_files_patterns-%'); - expect(purge![1]).toContain('_transient_timeout_wp_theme_files_patterns-%'); - }); - - it('also purges the SITE-transient pattern-file cache (single-site stores it as _site_transient_)', () => { - // Regression guard: WordPress caches the patterns/*.php file list as a SITE - // transient. `transient delete --all` does NOT clear site transients on a - // single-site install, and the regular `_transient_` DELETE misses the - // `_site_transient_` row — so a freshly-added per-page pattern resolves to - // an EMPTY wp:pattern (blank page body) until the TTL lapses. Both prefixes - // must be deleted. - const purge = cmds.find((c) => c[0] === 'eval'); - expect(purge![1]).toContain('_site_transient_wp_theme_files_patterns-%'); - expect(purge![1]).toContain('_site_transient_timeout_wp_theme_files_patterns-%'); - }); - - it('runs the purge through $wpdb via `wp eval`, NOT `wp db query` (MySQL-only — fails on Studio SQLite)', () => { - // Regression guard for the cache-flush-failed warning on every convert: `wp - // db query` shells out to the mysql binary and dies with "Undefined constant - // DB_HOST" on Studio's SQLite. `wp eval` runs the DELETE through $wpdb, which - // is the SQLite drop-in on Studio and MySQL elsewhere — driver-agnostic. - expect(cmds.some((c) => c[0] === 'db' && c[1] === 'query')).toBe(false); - const purge = cmds.find((c) => c[0] === 'eval'); - expect(purge![1]).toContain('$wpdb'); - expect(purge![1]).toContain('{$wpdb->options}'); // respects the table prefix - }); - - it('runs the pattern-file purge LAST (after cache flush, so it is not re-populated)', () => { - const evalIdx = cmds.findIndex((c) => c[0] === 'eval'); - const flushIdx = cmds.findIndex((c) => c[0] === 'cache'); - expect(evalIdx).toBeGreaterThan(flushIdx); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/install-theme.ts b/packages/data-liberation-agent/src/mcp-server/handlers/install-theme.ts deleted file mode 100644 index b7f4c028d9..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/install-theme.ts +++ /dev/null @@ -1,232 +0,0 @@ -// -// liberate_install_theme -// ====================== -// Streaming-friendly companion to liberate_preview. Installs replica theme -// files + block plugins into an *already-running* Studio site instead of -// creating one. Used by the streaming watch loop's theme-piece and -// archetype-template judgments — the pre-started Studio site already has -// streamed content, so we must not call liberate_preview (which routes -// through startStudioPreview and creates a `-2` duplicate site). -// -// Differences from liberate_preview: -// - No site creation. Caller passes `studioSitePath` to the running site. -// - No content import. Per-URL inserts already populated the DB. -// - Just writes files into wp-content/{themes,plugins} and activates. -// -// Returns the warnings collected during plugin/theme activate so the agent -// can surface non-fatal failures (e.g. activate fails because -// register_block_type errored — file was written but plugin didn't load). -// - -import { existsSync } from 'node:fs'; -import { basename, join, resolve } from 'node:path'; -import { - writeReplicaFilesToHost, - validateReplicaInputs, -} from '../../lib/preview/replica-install.js'; -import type { ReplicaFile, ReplicaBlockPlugin } from '../../lib/preview/types.js'; -import type { Handler } from '../handler-types.js'; -import { studioWp } from '../../lib/preview/studio.js'; - -interface InstallThemeArgs { - outputDir?: string; - studioSitePath?: string; - themeFiles?: ReplicaFile[]; - blockPlugins?: ReplicaBlockPlugin[]; - themeSlug?: string; -} - -export function deriveInstallThemeSlug(outputDir: string): string { - const base = basename(outputDir).toLowerCase(); - const sanitized = base.replace(/[^a-z0-9-]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); - return sanitized ? `${sanitized}-replica` : 'site-replica'; -} - -export function resolveInstallThemeSlug(args: { - outputDir: string; - requestedThemeSlug?: string; - wpRoot: string; -}): string { - const shellThemeSlug = deriveInstallThemeSlug(args.outputDir); - if (!args.requestedThemeSlug) return shellThemeSlug; - if (args.requestedThemeSlug === shellThemeSlug) return args.requestedThemeSlug; - - const shellStylePath = join( - args.wpRoot, - 'wp-content', - 'themes', - shellThemeSlug, - 'style.css', - ); - return existsSync(shellStylePath) ? shellThemeSlug : args.requestedThemeSlug; -} - -export const installThemeHandler: Handler = async (args, ctx) => { - const a = args as InstallThemeArgs; - - const outputDir = a.outputDir; - const studioSitePath = a.studioSitePath; - const themeFiles = a.themeFiles; - const blockPlugins = a.blockPlugins; - let themeSlug = a.themeSlug; - - if (!outputDir) { - return ctx.errorResult('liberate_install_theme requires `outputDir`.'); - } - if (!studioSitePath) { - return ctx.errorResult( - 'liberate_install_theme requires `studioSitePath` — the on-disk path to the running Studio site (e.g. ~/Studio/example-com).', - ); - } - // Studio mounts the host site directory at VFS path `/wordpress`. In current - // Studio versions the host site directory IS the wp-root (wp-content/ sits - // directly inside it). Older layouts nested everything under a `wordpress/` - // subdir on the host. Detect by probing for wp-content rather than assuming. - const sitePathResolved = resolve(studioSitePath); - let wpRoot = sitePathResolved; - if (!existsSync(join(wpRoot, 'wp-content'))) { - const nested = join(sitePathResolved, 'wordpress'); - if (existsSync(join(nested, 'wp-content'))) { - wpRoot = nested; - } else { - return ctx.errorResult( - `studioSitePath has no wp-content (looked at ${join(sitePathResolved, 'wp-content')} and ${join(nested, 'wp-content')}). Pass the running Studio site dir.`, - ); - } - } - - const hasTheme = !!themeFiles && themeFiles.length > 0; - const hasPlugins = !!blockPlugins && blockPlugins.length > 0; - if (!hasTheme && !hasPlugins) { - return ctx.errorResult( - 'liberate_install_theme needs themeFiles[] or blockPlugins[] (or both). Got neither.', - ); - } - if (hasTheme) { - themeSlug = resolveInstallThemeSlug({ - outputDir, - requestedThemeSlug: themeSlug, - wpRoot, - }); - } - - // Up-front validation — slug shape, path traversal, etc. — before any - // writes hit disk. - try { - validateReplicaInputs(themeFiles, blockPlugins, themeSlug); - } catch (err) { - return ctx.errorResult(`Replica input invalid: ${(err as Error).message}`); - } - - let written: { themeWritten: number; pluginsWritten: number; pluginSlugs: string[]; assetsCopied: number }; - try { - written = writeReplicaFilesToHost({ - wpRoot, - themeSlug, - themeFiles, - blockPlugins, - // Copy on-disk binary theme assets (self-hosted fonts, localized logo, icon - // SVGs) that string themeFiles[] can't carry — previously bridged by hand. - assetSourceDir: join(resolve(outputDir), 'theme'), - }); - } catch (err) { - return ctx.errorResult(`Failed to write replica files: ${(err as Error).message}`); - } - - const warnings: string[] = []; - for (const slug of written.pluginSlugs) { - try { - await studioWp(studioSitePath, ['plugin', 'activate', slug]); - } catch (err) { - warnings.push(`Plugin activate "${slug}" failed: ${(err as Error).message.trim()}`); - } - } - if (hasTheme && themeSlug) { - try { - await studioWp(studioSitePath, ['theme', 'activate', themeSlug]); - } catch (err) { - warnings.push(`Theme activate "${themeSlug}" failed: ${(err as Error).message.trim()}`); - } - // Flush caches so freshly-written block patterns + templates render - // immediately. Block themes cache `patterns/*.php` registration and - // template resolution; without a flush the just-installed front-page / - // patterns can render as the stale (empty) version until WP next clears - // its cache. Best-effort — never fatal to a successful install. - for (const wpArgs of themeCacheFlushCommands()) { - try { - await studioWp(studioSitePath, wpArgs); - } catch { /* best-effort */ } - } - } - - return ctx.textResult({ - ok: true, - studioSitePath, - themeSlug: themeSlug ?? null, - themeWritten: written.themeWritten, - assetsCopied: written.assetsCopied, - pluginsWritten: written.pluginsWritten, - pluginSlugs: written.pluginSlugs, - activated: { - theme: hasTheme && themeSlug ? !warnings.some((w) => w.startsWith(`Theme activate "${themeSlug}"`)) : null, - plugins: written.pluginSlugs.filter( - (s) => !warnings.some((w) => w.startsWith(`Plugin activate "${s}"`)), - ), - }, - warnings, - }); -}; - -/** - * The post-theme-activate cache-flush sequence, as ordered `wp` CLI argument - * vectors. Extracted as a pure function so the ordering + completeness can be - * unit-tested without shelling out to Studio. - * - * Three steps, each best-effort: - * 1. `transient delete --all` — clears DB-backed transients broadly. - * 2. `cache flush` — clears the runtime object cache. - * 3. `eval $wpdb->query(DELETE ... wp_theme_files_patterns-*)` — the - * load-bearing step for RE-installs (run via $wpdb, not `db query`, which - * is MySQL-only and fails on Studio's SQLite). `WP_Theme::get_block_patterns()` memoizes the theme's - * `patterns/*.php` file list in the `wp_theme_files_patterns-` - * transient. On a non-persistent object cache (Studio's SQLite), `cache - * flush` does NOT remove that DB-backed transient, so a newly-added pattern - * file stays UNregistered and its `wp:pattern` reference renders EMPTY — - * the page silently loses its reconstructed content. Deleting these - * transients forces the registry to rescan the patterns dir next request. - * (`transient delete --all` would also catch them, but is sometimes scoped - * to expired/timeout rows by site config, so we delete the pattern-file - * transients explicitly and unconditionally.) - */ -export function themeCacheFlushCommands(): string[][] { - return [ - ['transient', 'delete', '--all'], - ['cache', 'flush'], - // The theme's block-pattern file list (patterns/*.php) is cached in the - // `wp_theme_files_patterns` transient. WordPress stores it as a SITE - // transient (`_site_transient_*`), not a regular transient — on a - // single-site install `transient delete --all` does NOT clear site - // transients, so a newly-added patterns/page-.php never gets scanned - // and its `wp:pattern` resolves to empty until the transient TTL lapses. - // Delete BOTH the regular and the site-transient option rows (value + - // timeout) so the next request re-scans the patterns directory and - // registers the just-installed pattern. Without the `_site_transient_` - // variant the freshly-reconstructed page renders a blank pattern. - // - // Run the DELETE through `wp eval` (i.e. $wpdb), NOT `wp db query`: the - // latter shells out to the mysql client and dies with `Undefined constant - // DB_HOST` on Studio's SQLite, so the load-bearing purge silently failed on - // every convert (the recurring "cache flush failed (db query)" warning). - // $wpdb routes through the SQLite drop-in on Studio and MySQL elsewhere, so - // this is driver-agnostic. `{$wpdb->options}` respects the table prefix. - [ - 'eval', - 'global $wpdb; $wpdb->query("DELETE FROM {$wpdb->options} WHERE ' + - "option_name LIKE '_transient_wp_theme_files_patterns-%' " + - "OR option_name LIKE '_transient_timeout_wp_theme_files_patterns-%' " + - "OR option_name LIKE '_site_transient_wp_theme_files_patterns-%' " + - "OR option_name LIKE '_site_transient_timeout_wp_theme_files_patterns-%\");", - ], - ]; -} - diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/map-apis.ts b/packages/data-liberation-agent/src/mcp-server/handlers/map-apis.ts deleted file mode 100644 index bad9df7414..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/map-apis.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { Handler } from '../handler-types.js'; - -export const mapApisHandler: Handler = async (args, ctx) => { - const { mapApis } = await import('../../lib/probe/map-apis.js'); - const result = await mapApis({ - cdpPort: args.cdpPort as number, - url: args.url as string, - crawlUrls: (args.crawlUrls as string[]) ?? [], - followLinks: (args.followLinks as boolean) ?? false, - }); - return ctx.textResult(result); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/media-install.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/media-install.test.ts deleted file mode 100644 index 1317dc24cb..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/media-install.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; -import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; - -vi.mock('../../lib/streaming/media-install.js', () => ({ - installMediaForUrl: vi.fn().mockResolvedValue({ - installed: [], - skipped: [], - errors: [], - svg: { svgUploaded: 1, svgSubstituted: 2, svgFailed: 0, safeSvgEnsured: true }, - }), -})); - -import { wpRootFor, mediaInstallHandler } from './media-install.js'; - -describe('wpRootFor — Studio layout detection', () => { - let root: string; - beforeEach(() => { - root = mkdtempSync(join(tmpdir(), 'media-install-wproot-')); - }); - afterEach(() => { - rmSync(root, { recursive: true, force: true }); - }); - - it('flat Studio site (wp-content at site root) resolves to the site path itself', () => { - const sitePath = join(root, 'flat-site'); - mkdirSync(join(sitePath, 'wp-content'), { recursive: true }); - expect(wpRootFor({ kind: 'studio', sitePath })).toBe(sitePath); - }); - - it('nested Studio site (wordpress/wp-content) resolves to the nested wp-root', () => { - const sitePath = join(root, 'nested-site'); - mkdirSync(join(sitePath, 'wordpress', 'wp-content'), { recursive: true }); - expect(wpRootFor({ kind: 'studio', sitePath })).toBe(join(sitePath, 'wordpress')); - }); - - it('does not invent a phantom wordpress/ subdir for flat sites (regression)', () => { - const sitePath = join(root, 'flat-site-2'); - mkdirSync(join(sitePath, 'wp-content'), { recursive: true }); - // The previous implementation hardcoded `/wordpress`, which made - // uploads land in a directory the running flat site never serves. - expect(wpRootFor({ kind: 'studio', sitePath })).not.toBe(join(sitePath, 'wordpress')); - }); - -}); - -describe('mediaInstallHandler — SVG tally surfacing', () => { - it('includes the svg routing tally from the installer result', async () => { - const sitePath = mkdtempSync(join(tmpdir(), 'media-install-handler-')); - mkdirSync(join(sitePath, 'wp-content'), { recursive: true }); - try { - const ctx = { - adapters: [], - findAdapter: () => null, - textResult: (data: unknown) => ({ content: [{ type: 'text' as const, text: JSON.stringify(data) }] }), - errorResult: (message: string) => ({ content: [{ type: 'text' as const, text: message }], isError: true }), - server: {} as never, - }; - const result = await mediaInstallHandler( - { outputDir: '/tmp/out', url: 'https://example.com/', target: { kind: 'studio', sitePath } }, - ctx, - ); - const parsed = JSON.parse(result.content[0].text); - expect(parsed.svg).toEqual({ svgUploaded: 1, svgSubstituted: 2, svgFailed: 0, safeSvgEnsured: true }); - } finally { - rmSync(sitePath, { recursive: true, force: true }); - } - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/media-install.ts b/packages/data-liberation-agent/src/mcp-server/handlers/media-install.ts deleted file mode 100644 index 61945d7a37..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/media-install.ts +++ /dev/null @@ -1,102 +0,0 @@ -// -// liberate_media_install MCP handler -// ================================== -// Phase 1.5: install pending media for one URL into the running replica WP -// site. Wraps `installMediaForUrl` and surfaces its structured result. -// -// Args: -// outputDir: liberation output directory -// url: source URL whose media we're installing (kept for logging; -// the underlying installer processes ALL pending media each -// call thanks to MediaStubStore-keyed idempotency) -// target: { kind: 'studio', sitePath: string } — Studio site path -// -// Studio sitePath is the per-site directory Studio created (e.g. -// ~/Studio/); the WP install root inside it is `/wordpress`. -// -import { existsSync } from 'node:fs'; -import { join, resolve } from 'node:path'; -import type { Handler } from '../handler-types.js'; -import { installMediaForUrl, type MediaInstallResult } from '../../lib/streaming/media-install.js'; - -interface StudioTarget { kind: 'studio'; sitePath: string } -type Target = StudioTarget; - -function parseTarget(raw: unknown): Target | string { - if (!raw || typeof raw !== 'object') { - return 'target must be an object with kind ("studio") + sitePath'; - } - const t = raw as Record; - const kind = t.kind; - const sitePath = t.sitePath; - if (typeof sitePath !== 'string' || !sitePath) { - return 'target.sitePath must be a non-empty string'; - } - if (kind === 'studio') { - return { kind, sitePath }; - } - return 'target.kind must be "studio"'; -} - -/** - * Resolve the WP install root from a Studio target by probing the on-disk - * layout — mirrors the detection in install-theme.ts. Studio sites exist in - * two shapes: - * - flat: /wp-content (current Studio versions) - * - nested: /wordpress/wp-content (older layouts) - * Hardcoding `/wordpress` (the previous behavior) wrote uploads into - * a phantom `wordpress/` subdir on flat sites, so attachments never appeared in - * the running library. Probe instead. - */ -export function wpRootFor(target: Target): string { - const sitePath = resolve(target.sitePath); - if (existsSync(join(sitePath, 'wp-content'))) return sitePath; - const nested = join(sitePath, 'wordpress'); - if (existsSync(join(nested, 'wp-content'))) return nested; - // Neither layout found; fall back to flat (sitePath) and let the installer - // surface a concrete copy/eval error rather than silently using a wrong dir. - return sitePath; -} - -export const mediaInstallHandler: Handler = async (args, ctx) => { - const outputDir = args.outputDir as string | undefined; - const url = args.url as string | undefined; - const target = parseTarget(args.target); - - if (!outputDir || !url) { - return ctx.errorResult('liberate_media_install requires outputDir + url'); - } - if (typeof target === 'string') { - return ctx.errorResult(`liberate_media_install: ${target}`); - } - - const wpRoot = wpRootFor(target); - - let result: MediaInstallResult; - try { - result = await installMediaForUrl({ - outputDir, - url, - wpRoot, - }); - } catch (err) { - return ctx.errorResult(`liberate_media_install failed: ${(err as Error).message}`); - } - - return ctx.textResult({ - ok: result.errors.length === 0, - target: target.kind, - counts: { - installed: result.installed.length, - skipped: result.skipped.length, - errors: result.errors.length, - }, - // SVG routing tally (svg survival): how many SVG-origin assets uploaded - // as SVG vs. were substituted with their PNG raster sibling, plus whether - // safe-svg was auto-ensured for this batch. - svg: result.svg, - installed: result.installed, - skipped: result.skipped, - errors: result.errors, - }); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/paths.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/paths.test.ts deleted file mode 100644 index 0d4dd24917..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/paths.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { join } from 'node:path'; -import { homedir } from 'node:os'; -import { pathsHandler } from './paths.js'; - -const ctx = { textResult: (o: unknown) => o, errorResult: (m: string) => ({ error: m }) } as never; - -describe('pathsHandler', () => { - let prev: string | undefined; - let prevOutput: string | undefined; - beforeEach(() => { prev = process.env.STUDIO_SITES_DIR; prevOutput = process.env.DLA_OUTPUT_DIR; delete process.env.STUDIO_SITES_DIR; delete process.env.DLA_OUTPUT_DIR; }); - afterEach(() => { if (prev === undefined) delete process.env.STUDIO_SITES_DIR; else process.env.STUDIO_SITES_DIR = prev; if (prevOutput === undefined) delete process.env.DLA_OUTPUT_DIR; else process.env.DLA_OUTPUT_DIR = prevOutput; }); - - it('returns the base and per-site dir for a url', async () => { - const r = await pathsHandler({ url: 'https://example.com' }, ctx) as unknown as { base: string; siteDir: string | null }; - expect(r.base).toBe(join(homedir(), 'Studio', '_liberations')); - expect(r.siteDir).toBe(join(homedir(), 'Studio', '_liberations', 'example.com')); - }); - - it('returns base with null siteDir when no url given', async () => { - const r = await pathsHandler({}, ctx) as unknown as { base: string; siteDir: string | null }; - expect(r.siteDir).toBeNull(); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/paths.ts b/packages/data-liberation-agent/src/mcp-server/handlers/paths.ts deleted file mode 100644 index c4d7e6e9b0..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/paths.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { resolveOutputBase, siteOutputDir } from '../../lib/paths.js'; -import type { Handler } from '../handler-types.js'; - -export const pathsHandler: Handler = async (args, ctx) => { - const url = typeof args.url === 'string' ? args.url : undefined; - const base = resolveOutputBase(); - const siteDir = url ? siteOutputDir(base, url) : null; - return ctx.textResult({ base, siteDir }); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/preview.ts b/packages/data-liberation-agent/src/mcp-server/handlers/preview.ts deleted file mode 100644 index 5760b9d980..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/preview.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { spawn, execFileSync } from 'node:child_process'; -import type { Handler } from '../handler-types.js'; - -export const previewHandler: Handler = async (args, ctx) => { - const { startPreview } = await import('../../lib/preview/studio.js'); - const result = await startPreview({ - outputDir: args.outputDir as string, - themeFiles: args.themeFiles as import('../../lib/preview/types.js').ReplicaFile[] | undefined, - blockPlugins: args.blockPlugins as import('../../lib/preview/types.js').ReplicaBlockPlugin[] | undefined, - themeSlug: args.themeSlug as string | undefined, - siteName: args.siteName as string | undefined, - }); - if (result.status === 'ready' && args.open && result.url) { - const openBrowser = () => { - const cmd = process.platform === 'darwin' ? 'open' - : process.platform === 'win32' ? 'start' - : 'xdg-open'; - try { - spawn(cmd, [`${result.url}/wp-admin/`], { detached: true, stdio: 'ignore' }).unref(); - } catch { /* best-effort */ } - }; - const openStudioApp = (): boolean => { - try { - if (process.platform === 'darwin') { - spawn('open', ['-a', 'Studio'], { detached: true, stdio: 'ignore' }).unref(); - return true; - } - if (process.platform === 'win32') { - spawn('cmd', ['/c', 'start', '', 'Studio'], { detached: true, stdio: 'ignore' }).unref(); - return true; - } - if (process.platform === 'linux') { - const customCmd = process.env.STUDIO_APP_CMD; - if (customCmd) { - spawn('sh', ['-c', customCmd], { detached: true, stdio: 'ignore' }).unref(); - return true; - } - for (const bin of ['Studio', 'studio-app', 'wp-studio']) { - try { - execFileSync('which', [bin], { stdio: 'ignore', timeout: 1000 }); - spawn(bin, [], { detached: true, stdio: 'ignore' }).unref(); - return true; - } catch { /* try next */ } - } - } - return false; - } catch { return false; } - }; - if (result.source === 'studio' && openStudioApp()) { - /* launched Studio app */ - } else { - openBrowser(); - } - } - return ctx.textResult(result); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/probe.ts b/packages/data-liberation-agent/src/mcp-server/handlers/probe.ts deleted file mode 100644 index 7af96b4a67..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/probe.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { Handler } from '../handler-types.js'; - -export const probeHandler: Handler = async (args, ctx) => { - const { probeBrowser } = await import('../../lib/probe/browser-probe.js'); - const results = await probeBrowser( - args.cdpPort as number, - args.url as string | undefined, - ); - return ctx.textResult(results); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/qa.ts b/packages/data-liberation-agent/src/mcp-server/handlers/qa.ts deleted file mode 100644 index d2386f22b8..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/qa.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Handler } from '../handler-types.js'; - -export const qaHandler: Handler = async (args, ctx) => { - const { runQa } = await import('../../lib/qa/qa-runner.js'); - const result = await runQa({ - wxrFile: args.wxrFile as string, - fix: (args.fix as boolean) ?? false, - onProgress: (current, total, slug) => { - ctx.server.sendLoggingMessage({ - level: 'info', - data: `[qa] ${current}/${total} ${slug}`, - }); - }, - }); - return ctx.textResult(result); -}; diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/reconstruct-pages-carry-woo.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/reconstruct-pages-carry-woo.test.ts deleted file mode 100644 index d8d54b0e96..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/reconstruct-pages-carry-woo.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -// Integration test: WooCommerce auto-install wired into liberate_reconstruct_pages_carry. -// Exercises the REAL handler with the heavy edges mocked so we can assert the -// wiring contract: when products.csv/products.jsonl is present, ensurePlugin -// called once with 'woocommerce'; absent → not called; failure → warning not fatal. -// All data is fictional. Style mirrors reconstruct-pages-jetpack.test.ts. -import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; -import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; -import { join, resolve } from 'node:path'; - -const { ensureCalls, ensureState } = vi.hoisted(() => ({ - ensureCalls: [] as Array<{ sitePath: string; slug: string }>, - ensureState: { result: { ok: true, action: 'installed' } as { ok: true; action: string } | { ok: false; error: string } }, -})); - -vi.mock('node:child_process', async () => { - const { promisify } = await import('node:util'); - const execFile: any = (...args: unknown[]) => { - const cb = args[args.length - 1]; - if (typeof cb === 'function') (cb as (e: null, o: string, s: string) => void)(null, '', ''); - }; - execFile[promisify.custom] = async () => ({ stdout: '', stderr: '' }); - const spawn = vi.fn(); - return { execFile, spawn, default: { execFile, spawn } }; -}); - -vi.mock('../../lib/preview/ensure-plugin.js', () => ({ - ensurePlugin: vi.fn(async (sitePath: string, slug: string) => { - ensureCalls.push({ sitePath, slug }); - return ensureState.result; - }), -})); - -// Mock all heavy IO dependencies so the handler completes without real network/disk. -vi.mock('../../lib/replicate/css-collect.js', () => ({ - collectCss: vi.fn(async () => ''), -})); - -vi.mock('../../lib/replicate/page-reconstruct-carry.js', () => ({ - reconstructPageCarry: vi.fn(() => ({ - mainIsland: '
content
', - headerIsland: '
Logo
', - footerIsland: '
Footer
', - deepChrome: true, - scaffold: undefined, - chromeCss: '.fictional-header{color:red}', - mainCss: '.fictional-main{color:blue}', - })), -})); - -vi.mock('../../lib/replicate/theme-scaffold-carry.js', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - buildCarryThemeFiles: vi.fn(() => []), - }; -}); - -vi.mock('../../lib/replicate/carry-missing-media.js', () => ({ - fetchMissingCarriedMedia: vi.fn(async () => ({ downloaded: 0, failed: 0 })), -})); - -vi.mock('../../lib/replicate/carry-fonts.js', () => ({ - localizeCarryFonts: vi.fn(async () => ({ files: [], downloaded: 0, failed: 0 })), -})); - -vi.mock('../../lib/replicate/carry-cdn-audit.js', () => ({ - findExternalAssetRefs: vi.fn(() => ({ refs: [], byHost: {}, samples: [] })), -})); - -vi.mock('../../lib/replicate/carry-design-tokens.js', () => ({ - loadCarryDesignTokens: vi.fn(() => ({ themeJsonPalette: [], themeJsonFontFamilies: [] })), -})); - -vi.mock('../../lib/replicate/page-link-map.js', () => ({ - buildPageLinkMap: vi.fn(() => new Map()), -})); - -vi.mock('../../lib/replicate/carry-page-list.js', () => ({ - reconcileCarryIslands: vi.fn(), -})); - -vi.mock('../../lib/replicate/run-media-map.js', () => ({ - installRunMediaMap: vi.fn(async () => ({ mediaUrlMap: new Map(), result: { installed: [] } })), -})); - -vi.mock('../../lib/replicate/carry-responsive-assemble.js', () => ({ - assembleResponsiveMobile: vi.fn((html: string) => html), -})); - -vi.mock('../../lib/screenshot/dynamic-content.js', () => ({ - assessBody: vi.fn(() => ({ isolated: false })), - readPngHeight: vi.fn(() => null), - classifyEmptyBodies: vi.fn(() => []), -})); - -import { reconstructPagesCarryHandler } from './reconstruct-pages-carry.js'; -import type { HandlerContext } from '../handler-types.js'; - -const TMP_BASE = resolve('.tmp-test', `reconstruct-carry-woo-${process.pid}`); - -function makeFixture(name: string, opts: { withProductsCsv?: boolean; withProductsJsonl?: boolean } = {}) { - const base = join(TMP_BASE, name); - rmSync(base, { recursive: true, force: true }); - const outputDir = join(base, 'out'); - const studioSitePath = join(base, 'site'); - const wpContent = join(studioSitePath, 'wp-content'); - const htmlDir = join(outputDir, 'html'); - mkdirSync(htmlDir, { recursive: true }); - mkdirSync(wpContent, { recursive: true }); - // Minimal carried HTML so the handler doesn't fall back to a live fetch. - writeFileSync( - join(htmlDir, 'home.html'), - '
H
content
F
', - ); - if (opts.withProductsCsv) { - writeFileSync(join(outputDir, 'products.csv'), 'name\nFictional Product'); - } - if (opts.withProductsJsonl) { - writeFileSync(join(outputDir, 'products.jsonl'), '{"name":"Fictional Product"}'); - } - return { outputDir, studioSitePath }; -} - -const ctx: HandlerContext = { - adapters: [], - findAdapter: () => null, - textResult: (data: unknown) => ({ content: [{ type: 'text' as const, text: JSON.stringify(data, null, 2) }] }), - errorResult: (message: string) => ({ content: [{ type: 'text' as const, text: message }], isError: true }), - server: {} as HandlerContext['server'], -}; - -const PAGES = [{ slug: 'home', sourceUrl: 'https://fictional-store.example/', title: 'Home', isHome: true }]; - -beforeEach(() => { - ensureCalls.length = 0; - ensureState.result = { ok: true, action: 'installed' }; -}); - -afterAll(() => { - rmSync(TMP_BASE, { recursive: true, force: true }); -}); - -describe('liberate_reconstruct_pages_carry woocommerce auto-install', () => { - it('products.csv present → ensurePlugin(studioSitePath, "woocommerce") once + wooEnsured true', async () => { - const fx = makeFixture('csv', { withProductsCsv: true }); - const result = await reconstructPagesCarryHandler( - { outputDir: fx.outputDir, studioSitePath: fx.studioSitePath, pages: PAGES }, - ctx, - ); - const text = result.content[0].text; - expect(ensureCalls).toEqual([{ sitePath: fx.studioSitePath, slug: 'woocommerce' }]); - expect(text).toContain('"wooEnsured": true'); - expect(text).not.toContain('wooWarning'); - }); - - it('products.jsonl present → ensurePlugin called once', async () => { - const fx = makeFixture('jsonl', { withProductsJsonl: true }); - const result = await reconstructPagesCarryHandler( - { outputDir: fx.outputDir, studioSitePath: fx.studioSitePath, pages: PAGES }, - ctx, - ); - const text = result.content[0].text; - expect(ensureCalls).toEqual([{ sitePath: fx.studioSitePath, slug: 'woocommerce' }]); - expect(text).toContain('"wooEnsured": true'); - }); - - it('no products → ensurePlugin NOT called, wooEnsured false', async () => { - const fx = makeFixture('no-products'); - const result = await reconstructPagesCarryHandler( - { outputDir: fx.outputDir, studioSitePath: fx.studioSitePath, pages: PAGES }, - ctx, - ); - const text = result.content[0].text; - expect(ensureCalls).toEqual([]); - expect(text).toContain('"wooEnsured": false'); - expect(text).not.toContain('wooWarning'); - }); - - it('ensure failure → wooWarning in result text, run NOT fatal (pages still carry)', async () => { - const fx = makeFixture('ensure-fails', { withProductsCsv: true }); - ensureState.result = { ok: false, error: 'woo install exploded (fictional)' }; - const result = await reconstructPagesCarryHandler( - { outputDir: fx.outputDir, studioSitePath: fx.studioSitePath, pages: PAGES }, - ctx, - ); - const text = result.content[0].text; - expect(result.isError).not.toBe(true); - expect(text).toContain('"wooEnsured": false'); - expect(text).toContain('woo install exploded (fictional)'); - }); -}); diff --git a/packages/data-liberation-agent/src/mcp-server/handlers/reconstruct-pages-carry.test.ts b/packages/data-liberation-agent/src/mcp-server/handlers/reconstruct-pages-carry.test.ts deleted file mode 100644 index d4b5bd62a8..0000000000 --- a/packages/data-liberation-agent/src/mcp-server/handlers/reconstruct-pages-carry.test.ts +++ /dev/null @@ -1,278 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { assembleCarryTheme, extractStoreHeaderIsland } from './reconstruct-pages-carry.js'; - -describe('extractStoreHeaderIsland', () => { - it('captures the FULL header group (announcement bar + header) when present, not just
', () => { - // Fictional Shopify-style header group: an announcement section + a header section, - // siblings sharing `shopify-section-group-header-group`. - const island = - '

Sale on now

' + - '
' + - '
page body
'; - const out = extractStoreHeaderIsland(island); - expect(out).toContain('Sale on now'); // announcement bar kept - expect(out).toContain(' when there is no header group (non-Shopify)', () => { - const island = '
body
'; - const out = extractStoreHeaderIsland(island); - expect(out).toContain('site-header'); - expect(out).not.toContain('body'); - }); - - it('returns empty string when there is no header at all', () => { - expect(extractStoreHeaderIsland('
just content
')).toBe(''); - expect(extractStoreHeaderIsland('')).toBe(''); - }); -}); - -describe('assembleCarryTheme', () => { - it('builds theme files + per-page WXR content, with chrome CSS site-wide and main CSS per-page', () => { - const out = assembleCarryTheme({ - themeName: 'Acme Carry', - pages: [ - { - slug: 'home', - title: 'Home', - isHome: true, - bodyHtml: - '
H
Hi
F
', - css: '.hero{color:red} .h{color:green}', - }, - ], - mediaUrlMap: new Map(), - }); - const byPath = (p: string) => out.themeFiles.find((f) => f.path === p)?.content ?? ''; - // chrome rule is in the globally-enqueued site.css (after the reset), main rule in page sheet - expect(byPath('assets/css/site.css')).toContain(':where(body.lib-carry-site) .h'); - expect(byPath('assets/css/page-home.css')).toContain(':where(body.lib-carry-site.lib-carry-page-home) .hero'); - const home = out.wxrPages.find((p) => p.slug === 'home')!; - expect(home.postContent).toContain('\n
\n${html}\n
\n`; -} - -export function assembleCarryTheme(input: AssembleInput): AssembleOutput { - // Reconstruct every page once, preserving input order for the emitted files. - // A page whose markup trips reconstructPageCarry (e.g. carryHtml's injection gate on - // un-strippable rawtext) is SKIPPED — one bad page must not crash the whole build. - const skipped: string[] = []; - const recos = input.pages.flatMap((p) => { - try { - return [{ - p, - r: reconstructPageCarry({ - slug: p.slug, - isHome: p.isHome, - bodyHtml: p.bodyHtml, - css: p.css, - specs: p.specs ?? [], - mediaUrlMap: input.mediaUrlMap, - linkMap: input.linkMap, - mobile: p.mobile, - }), - }]; - } catch { - skipped.push(p.slug); - return []; - } - }); - - // Dedupe chrome so pages that render the SAME header/footer share one variant - // + part pair (e.g. a transparent-overlay home header and a solid interior - // header → two variants total, regardless of page count). Grouping is by - // canonical signature, not raw bytes (see registerChrome). The home page's - // chrome is registered FIRST so it becomes variant 0 — the canonical - // `header`/`footer` parts + index.html chrome. - // Group by canonical signature (instance ids + active-nav state normalized away) - // so Wix's per-page header instances collapse to one variant. Each distinct - // signature reserves a key + its representative (raw) islands and chrome CSS. - const keyBySig = new Map(); - const orderedKeys: string[] = []; - const repByKey = new Map(); - const ensureKey = (r: { headerIsland: string; footerIsland: string; chromeCss: string }): string => { - const sig = chromeSignature(r.headerIsland, r.footerIsland); - let key = keyBySig.get(sig); - if (!key) { - key = `c${orderedKeys.length}`; - keyBySig.set(sig, key); - orderedKeys.push(key); - repByKey.set(key, { headerIsland: r.headerIsland, footerIsland: r.footerIsland, chromeCss: r.chromeCss }); - } - return key; - }; - // Reserve the home page's signature as variant 0 (the canonical `header`/`footer` - // + index.html chrome) without counting it as a member yet. - const homeReco = recos.find((x) => x.p.isHome) ?? recos[0]; - if (homeReco) ensureKey(homeReco.r); - - const countByKey = new Map(); - const scaffoldPages: CarryPage[] = []; - const wxrPages: WxrPage[] = []; - for (const { p, r } of recos) { - const chromeKey = ensureKey(r); - countByKey.set(chromeKey, (countByKey.get(chromeKey) ?? 0) + 1); - scaffoldPages.push({ - slug: p.slug, - isHome: p.isHome, - postType: p.postType, - pageCss: r.mainCss, - scaffold: r.scaffold, - // Dual-viewport mobile carry now rides in the TEMPLATE (scaffoldedTemplate wraps - // post-content + the iframe), so post_content stays editable section blocks. - mobile: p.mobile, - chromeKey, - }); - wxrPages.push({ - slug: p.slug, - title: p.title, - isHome: p.isHome, - postType: p.postType, - postContent: r.mainIsland, - }); - } - - // Build the emitted variants. A variant used by ONE page keeps its active-nav - // highlight (e.g. the home header underlining "HOME"); a SHARED variant strips - // it, since one representative can't carry every member's "current" item. - const variants: ChromeVariant[] = orderedKeys.map((key) => { - const rep = repByKey.get(key)!; - const shared = (countByKey.get(key) ?? 0) > 1; - return { - key, - headerIsland: shared ? stripActiveNavState(rep.headerIsland) : rep.headerIsland, - footerIsland: shared ? stripActiveNavState(rep.footerIsland) : rep.footerIsland, - }; - }); - - // site.css holds EVERY distinct variant's chrome CSS (variant order). Safe to - // concatenate: each variant's rules key off the source's per-header comp-ids, so - // a variant's rules match nothing on a page rendering a different variant. - const siteCss = variants - .map((v) => repByKey.get(v.key)?.chromeCss ?? '') - .filter(Boolean) - .join('\n'); - - // Replicate the source classes (e.g. Wix's `responsive`) onto the WP body - // so body-state-gated carried rules — the whole mobile-reflow layout — behave like - // the source. Taken from the home page's carried HTML. - const bodyClasses = extractBodyClasses(homeReco?.p.bodyHtml ?? ''); - - // Store header for the WooCommerce templates. Product / shop / category-archive - // pages have no carried island, so isolate a header from a representative INTERIOR - // page (its solid header — the home page's is often a transparent overlay that - // vanishes on a white store page), falling back to any page that yields one. Only - // when the run has products; otherwise no store templates are emitted. - let storeHeaderIsland = ''; - if (input.hasProducts) { - // Smartest-match: carve from the header the MOST pages share (the site's "default" nav) - // rather than whichever page happens to come first — on a site with sectional navs the - // dominant one wins. Frequency is ranked by a HEADER-ONLY canonical signature: the variant - // map keys on header+footer combined, which would undercount a header shared across pages - // whose footers differ. Pages with no split-out header rank 0 (their inline-header fallback - // is weaker signal than a real region). Interior pages still outrank the home page, and the - // sort is stable, so original page order breaks remaining ties. - const headerSigOf = new Map<(typeof recos)[number], string>(); - for (const x of recos) { - if (x.r.headerIsland) headerSigOf.set(x, chromeSignature(x.r.headerIsland, '')); - } - const headerFreq = new Map(); - for (const sig of headerSigOf.values()) headerFreq.set(sig, (headerFreq.get(sig) ?? 0) + 1); - const freq = (x: (typeof recos)[number]): number => { - const sig = headerSigOf.get(x); - return sig ? headerFreq.get(sig) ?? 0 : 0; - }; - const ordered = [...recos].sort((a, b) => { - const homeRank = Number(a.p.isHome ?? false) - Number(b.p.isHome ?? false); - if (homeRank !== 0) return homeRank; - return freq(b) - freq(a); - }); - for (const cand of ordered) { - // The header may be a split-out region (headerIsland — now incl. the full Shopify header - // group), or ride inline in the main island. Prefer the region, then fall back to the body. - storeHeaderIsland = - extractStoreHeaderIsland(cand.r.headerIsland) || extractStoreHeaderIsland(cand.r.mainIsland); - if (storeHeaderIsland) break; - } - } - - const themeFiles = buildCarryThemeFiles({ - themeName: input.themeName, - chromeVariants: variants, - siteCss, - bodyClasses, - pages: scaffoldPages, - storeHeaderIsland, - hasProducts: input.hasProducts, - themeJsonPalette: input.themeJsonPalette, - themeJsonFontFamilies: input.themeJsonFontFamilies, - }); - - // Guardrail: a store run that couldn't isolate a header → no store templates → - // product/shop pages render with WooCommerce's bare defaults (the failure mode - // that shipped silently on getsnooz, 2026-06-04). Surface it instead of letting - // the operator discover it by eye. - const warnings: string[] = []; - if (input.hasProducts && !storeHeaderIsland) { - warnings.push( - 'Store pages (single-product / archive-product) will render WITHOUT site chrome: no
could be isolated from any carried page island, so WooCommerce defaults are used. Capture an interior page with a header, or add a header part manually.', - ); - } - - return { themeFiles, wxrPages, skipped, warnings }; -} - -// --------------------------------------------------------------------------- -// IO handler (not unit-tested — verified by typecheck + smoke import) -// --------------------------------------------------------------------------- - -interface PageArg { - slug: string; - sourceUrl: string; - title: string; - isHome?: boolean; - /** WP object type the slug resolves to. Default 'page'. Posts scope via is_single(). */ - postType?: 'page' | 'post'; - /** - * Override the cached-HTML filename stem (`html/.html`) when it - * differs from `slug` — e.g. posts captured as `post--.html` but whose - * WP post_name (and thus is_single() slug) is the bare ``. Defaults to slug. - */ - htmlSlug?: string; -} - -export const reconstructPagesCarryHandler: Handler = async (args, ctx) => { - const outputDir = args.outputDir as string | undefined; - const studioSitePath = args.studioSitePath as string | undefined; - const pages = args.pages as PageArg[] | undefined; - // Default: emit carried bodies as the in-canvas `dla/editable-html` block instead of a - // sandboxed `core/html` island, so the styled markup is visible (and editable) in the - // block editor. Front-end output is byte-identical (static save). Opt OUT with - // editableIslands:false to force plain core/html. - const editableIslands = args.editableIslands !== false; - - if (!outputDir) { - return ctx.errorResult('liberate_reconstruct_pages_carry requires `outputDir`.'); - } - if (!studioSitePath) { - return ctx.errorResult('liberate_reconstruct_pages_carry requires `studioSitePath`.'); - } - if (!Array.isArray(pages) || pages.length === 0) { - return ctx.errorResult( - 'liberate_reconstruct_pages_carry requires a non-empty `pages` array ({slug, sourceUrl, title, isHome?}).', - ); - } - - const wpRoot = studioWpRoot(studioSitePath); - if (!wpRoot) { - return ctx.errorResult(`studioSitePath has no wp-content: ${studioSitePath}`); - } - - // Derive carry theme slug from outputDir (parallel to block path, but suffixed -carry). - const themeName = (args.themeName as string | undefined) ?? 'Liberated (Carry)'; - const baseSlug = deriveInstallThemeSlug(outputDir); - // Strip the trailing "-replica" suffix the block path uses and append "-carry" so - // the two themes can coexist in wp-content/themes/ simultaneously. - const carrySlug = baseSlug.replace(/-replica$/, '') + '-carry'; - const themeRoot = join(wpRoot, 'wp-content', 'themes', carrySlug); - - // Collect HTML + CSS for each page. - const carryPages: CarryPageInput[] = []; - const fetchErrors: Array<{ slug: string; error: string }> = []; - // Phase 0 guardrail: pages whose captured body renders effectively empty (a JS app that - // never rendered — reviews/FAQ widgets, cross-origin iframes). The reliable signal is - // RENDERED HEIGHT (a chrome-only page is dramatically shorter than the page-set median), - // not DOM text (the app's DOM is present-but-blank, plus ~300 chars of cart boilerplate). - // Collected per page here, decided after the loop once the median is known. - const emptyBodies: EmptyBody[] = []; - const pageStats: PageStat[] = []; - const siteOrigin = (() => { - try { return new URL(pages[0].sourceUrl).origin; } catch { return undefined; } - })(); - - // Responsive-image map ({wix-media-id → mobile-variant URL}) captured at the - // mobile viewport by liberate_screenshot. Used to wrap carried s in a - // so the browser serves the mobile crop at narrow widths (no JS). - let responsiveImages: Record = {}; - try { - const riPath = join(resolve(outputDir), 'responsive-images.json'); - if (existsSync(riPath)) responsiveImages = JSON.parse(readFileSync(riPath, 'utf8')); - } catch { - /* best-effort — reconstruct without mobile variants on a missing/corrupt map */ - } - - // Mobile-DOM carry (classic/adaptive Wix). liberate_screenshot's mobile pass - // writes html-mobile/.html (the JS-built 320px mobile DOM, scripts stripped) - // + heights.json. When present, each page emits a dual island whose mobile half is - // an iframe of that document, served from the site's uploads/_carry-mobile/. - let mobileHeights: Record = {}; - try { - const hPath = join(resolve(outputDir), 'html-mobile', 'heights.json'); - if (existsSync(hPath)) mobileHeights = JSON.parse(readFileSync(hPath, 'utf8')); - } catch { - /* best-effort — no mobile-DOM carry on a missing/corrupt heights map */ - } - const carryMobileDir = join(wpRoot, 'wp-content', 'uploads', '_carry-mobile'); - - for (const p of pages) { - // Prefer cached rendered HTML written by liberate_screenshot. The filename - // stem is htmlSlug when given (posts: `post--.html`), else the slug. - const htmlPath = join(resolve(outputDir), 'html', `${p.htmlSlug ?? p.slug}.html`); - let bodyHtml = ''; - if (existsSync(htmlPath)) { - try { - bodyHtml = readFileSync(htmlPath, 'utf8'); - } catch { - /* fall through to live fetch */ - } - } - if (!bodyHtml) { - try { - const res = await fetch(p.sourceUrl); - bodyHtml = await res.text(); - } catch (err) { - fetchErrors.push({ - slug: p.slug, - error: err instanceof Error ? err.message : String(err), - }); - continue; - } - } - - // Phase 0: record the rendered height + body classification; the empty decision is - // made after the loop (needs the page-set median). - const pngPath = join(resolve(outputDir), 'screenshots', 'desktop', `${p.htmlSlug ?? p.slug}.png`); - pageStats.push({ slug: p.slug, height: readPngHeight(pngPath), assess: assessBody(bodyHtml, siteOrigin) }); - - // Collect external stylesheets referenced in the HTML. - let css = ''; - try { - css = await collectCss({ - html: bodyHtml, - inlineStyleText: '', - baseUrl: p.sourceUrl, - onError: () => { - /* swallow individual sheet errors — best-effort */ - }, - }); - } catch { - /* non-fatal: reconstruct with whatever CSS was collected */ - } - - // Cache the collected CSS alongside the HTML for debugging / re-runs. - try { - const cssCacheDir = join(resolve(outputDir), 'css'); - mkdirSync(cssCacheDir, { recursive: true }); - writeFileSync(join(cssCacheDir, `${p.slug}.css`), css); - } catch { - /* best-effort cache write */ - } - - // Mobile-DOM carry: if a mobile capture exists for this page, install it under - // uploads/_carry-mobile/.html and emit a dual island referencing it. - let mobile: { docUrl: string; height: number } | undefined; - const mobileSrc = join(resolve(outputDir), 'html-mobile', `${p.htmlSlug ?? p.slug}.html`); - if (existsSync(mobileSrc)) { - try { - mkdirSync(carryMobileDir, { recursive: true }); - const mobileDoc = readFileSync(mobileSrc, 'utf8'); - writeFileSync(join(carryMobileDir, `${p.slug}.html`), mobileDoc); - mobile = { - docUrl: `/wp-content/uploads/_carry-mobile/${p.slug}.html`, - height: mobileHeights[p.htmlSlug ?? p.slug] ?? 6000, - }; - } catch { - /* best-effort — fall back to desktop-only for this page */ - } - } - - carryPages.push({ - slug: p.slug, - title: p.title, - isHome: p.isHome, - postType: p.postType, - bodyHtml, - css, - mobile, - }); - } - - if (carryPages.length === 0) { - return ctx.errorResult( - `liberate_reconstruct_pages_carry: no pages could be loaded. fetchErrors: ${JSON.stringify(fetchErrors)}`, - ); - } - - // Phase 0 decision (pure, tested in dynamic-content.test.ts): a page whose rendered - // desktop capture is dramatically shorter than the page-set median AND isn't text-rich - // is chrome-only — the JS app never rendered, so it carries blank. Falls back to the - // text signal for any page without a usable screenshot height. - emptyBodies.push(...classifyEmptyBodies(pageStats)); - - // Internal-link rewrite map — same builder the block path uses (shared module), - // so carried nav + body hrefs resolve to the imported permalinks, not the source. - const linkMap = buildPageLinkMap(outputDir, pages.map((p) => p.sourceUrl)); - - // Pre-pass: download any image the carried HTML references but extraction never - // captured (no local copy exists, so repoint can't self-host it). Runs BEFORE install - // so the fetched assets enter the media map and the rewrite repoints them like the rest. - // Best-effort: a failed fetch (e.g. CDN 403) is recorded and left as a CDN ref. - const missingMedia = await fetchMissingCarriedMedia( - outputDir, - carryPages.map((p) => p.bodyHtml), - ); - - // Install the run's media into the alt site + build the CDN→local URL map via - // the SAME installMediaForUrl the block path uses (installRunMediaMap). Carried - // /url() references then point at this site's media library, not the CDN. - // Best-effort: media-install failure leaves the map empty (carried URLs survive). - let mediaUrlMap = new Map(); - const mediaErrors: Array<{ sourceUrl: string; error: string }> = []; - try { - const media = await installRunMediaMap({ - outputDir, - url: pages[0].sourceUrl, - wpRoot, - }); - mediaUrlMap = media.mediaUrlMap; - mediaErrors.push(...media.result.errors); - } catch (err) { - mediaErrors.push({ sourceUrl: '*', error: err instanceof Error ? err.message : String(err) }); - } - - // Store templates (single-product / archive-product) only make sense when the run - // produced WooCommerce products. - const hasProducts = - existsSync(join(resolve(outputDir), 'products.csv')) || - existsSync(join(resolve(outputDir), 'products.jsonl')); - - // WooCommerce auto-install: store templates only render correctly when WooCommerce - // is installed and active. Mirror the Jetpack pattern exactly — warning-not-fatal, - // wooEnsured tally. Called once when products are present; skipped otherwise. - let wooEnsured = false; - let wooWarning: string | undefined; - if (hasProducts) { - const wpExec: ExecFn = (sitePath, wpArgs) => - studioExecFileAsync(['wp', '--path', sitePath, ...wpArgs], { - timeout: 300_000, - maxBuffer: 16 * 1024 * 1024, - }).then((o) => o.stdout); - const ensured = await ensurePlugin(studioSitePath, 'woocommerce', wpExec); - if (ensured.ok) { - wooEnsured = true; - } else { - wooWarning = `WooCommerce auto-install failed (store templates will not render until WooCommerce is installed): ${ensured.error}`; - console.error(`[reconstruct-carry] ${wooWarning}`); - } - } - - // Register the captured palette/fonts in theme.json so the product-marketing core - // blocks (emitted later by enrich-product-marketing) resolve their token references. - const designTokens = loadCarryDesignTokens(outputDir); - const { themeFiles, wxrPages, skipped, warnings } = assembleCarryTheme({ - themeName, - pages: carryPages, - mediaUrlMap, - linkMap, - hasProducts, - themeJsonPalette: designTokens.themeJsonPalette, - themeJsonFontFamilies: designTokens.themeJsonFontFamilies, - }); - - // Self-host fonts: the scoped CSS carries the source @font-face rules with Wix-CDN - // src URLs. Download each into the theme's assets/fonts/ and rewrite the CSS url() to - // a local ../fonts/ path (sheets are enqueued from assets/css/). Best-effort. - const fontLocalize = await localizeCarryFonts(themeRoot, themeFiles, { wpRoot }); - - // Write theme files to disk under wp-content/themes/. - for (const f of fontLocalize.files) { - const full = join(themeRoot, f.path); - mkdirSync(dirname(full), { recursive: true }); - writeFileSync(full, f.content); - } - - // Final per-page island content. Wrap carried s with a captured mobile variant in - // + a `(max-width:750px)` , and append a single-column mobile grid next - // to any Wix pro-gallery — THEN self-host. Both inject steps reference the captured - // mobile-crop URLs (responsive-images.json), which are CDN-only (never downloaded), so the - // assembly repoints them to the installed DESKTOP local copy of the same media-id via the - // run media map. Mobile shows the desktop crop, but every image is self-hosted (zero - // source-CDN dependency) — see carry-responsive-assemble.ts. - let finalPages = wxrPages.map((w) => ({ - slug: w.slug, - title: w.title, - isHome: w.isHome, - postType: w.postType, - postContent: assembleResponsiveMobile(w.postContent, responsiveImages, mediaUrlMap), - })); - - // Editor visibility (opt-in). A carried body emits as one `core/html` island, which - // the block editor renders inside an isolated SandBox