Data Liberation: Add timeouts to browser calls so extraction cannot hang

packages/data-liberation-agent/dist/scripts/carry-reconstruct-drive.mjs

@@ -3,7 +3,7 @@ const require = __bundleCreateRequire(import.meta.url);
 import{a as $,b as y,c as w,d as S}from"./chunk-ENWLFTGA.mjs";import"./chunk-4TKBPMF4.mjs";import{writeFileSync as g,existsSync as P,mkdirSync as O,copyFileSync as D}from"node:fs";import{join as r}from"node:path";function N(s){console.log(`pages: ${s.pages.length} (${s.pages.filter(o=>o.postType==="post").length} posts, home=${s.pages.some(o=>o.isHome)})`),s.excluded.length&&console.log(`excluded (${s.excluded.length}): ${s.excluded.map(o=>o.slug).join(", ")}`),s.skipped.length&&console.log(`skipped \u2014 no captured html (${s.skipped.length}): ${s.skipped.map(o=>`${o.postType}:${o.postName}`).join(", ")}`)}async function k(){let s=process.argv.slice(2),o=s.includes("--list"),m=s.includes("--slim"),[n,d,_="Liberated (Carry)"]=s.filter(t=>t!=="--list"&&t!=="--slim");if((!n||!o&&!m&&!d)&&(console.error(`usage:
   node scripts/run.mjs carry-reconstruct-drive <outputDir> --slim   # slim output.wxr for provisioning (run BEFORE liberate_preview)
   node scripts/run.mjs carry-reconstruct-drive <outputDir> --list   # inspect the carry page list (no handler)
-  node scripts/run.mjs carry-reconstruct-drive <outputDir> <studioSitePath> "<Theme Name>"`),process.exit(2)),m){let{dropped:t,flipped:C}=S(n);console.log(`slimmed: dropped ${t} attachment items, flipped ${C} draft->publish; full WXR preserved at output.wxr.full`);return}let b=(process.env.EXCLUDE??"").split(",").map(t=>t.trim()).filter(Boolean),c=$(n,{exclude:b});if(g(r(n,"carry-pages.json"),JSON.stringify(c.pages,null,2)),N(c),o){console.log(`wrote ${r(n,"carry-pages.json")}`);return}let{reconstructPagesCarryHandler:x}=await import("./chunk-LTZ532K4.mjs"),E={textResult:t=>({_data:t}),errorResult:t=>{throw new Error(`handler error: ${t}`)}},f=!/^(0|false)$/i.test(process.env.EDITABLE_ISLANDS??""),e=(await x({outputDir:n,studioSitePath:d,themeName:_,pages:c.pages,editableIslands:f},E))._data;console.log(`theme: ${e.themeSlug}  media: ${e.mediaInstalled}  mediaErrors: ${e.mediaErrors.length}  fetchErrors: ${e.fetchErrors.length}  missingMediaDownloaded: ${e.missingMediaDownloaded??0}  fontsLocalized: ${e.fontsLocalized??0}`),e.missingMediaFailed?.length&&console.log("  missingMediaFailed:",JSON.stringify(e.missingMediaFailed)),e.fontsFailed?.length&&console.log("  fontsFailed:",JSON.stringify(e.fontsFailed)),f&&console.log(`editable islands: converted=${e.islandsConverted??0}  plugin=${e.editableHtmlPluginSlug??"(none)"}`);let p=e.residualCdnAssets??0;console.log(`self-host audit: residualCdnAssets=${p}${p?`  byHost=${JSON.stringify(e.residualCdnByHost)}`:" \u2713"}`),p&&console.log("  residualCdnSamples:",JSON.stringify(e.residualCdnSamples)),e.mediaErrors.length&&console.log("  mediaErrors sample:",JSON.stringify(e.mediaErrors.slice(0,3))),e.fetchErrors.length&&console.log("  fetchErrors:",JSON.stringify(e.fetchErrors));let i=r(d,"wp-content/uploads/_carry-islands");O(i,{recursive:!0});let u=e.pages.map(t=>t.slug),l=w(i,u);l.length&&console.log(`  removed ${l.length} stale island(s): ${l.slice(0,8).join(", ")}${l.length>8?" \u2026":""}`);for(let t of e.pages)g(r(i,`${t.slug}.html`),t.postContent);let v=`<?php
+  node scripts/run.mjs carry-reconstruct-drive <outputDir> <studioSitePath> "<Theme Name>"`),process.exit(2)),m){let{dropped:t,flipped:C}=S(n);console.log(`slimmed: dropped ${t} attachment items, flipped ${C} draft->publish; full WXR preserved at output.wxr.full`);return}let b=(process.env.EXCLUDE??"").split(",").map(t=>t.trim()).filter(Boolean),c=$(n,{exclude:b});if(g(r(n,"carry-pages.json"),JSON.stringify(c.pages,null,2)),N(c),o){console.log(`wrote ${r(n,"carry-pages.json")}`);return}let{reconstructPagesCarryHandler:x}=await import("./chunk-BEJIQB5M.mjs"),E={textResult:t=>({_data:t}),errorResult:t=>{throw new Error(`handler error: ${t}`)}},f=!/^(0|false)$/i.test(process.env.EDITABLE_ISLANDS??""),e=(await x({outputDir:n,studioSitePath:d,themeName:_,pages:c.pages,editableIslands:f},E))._data;console.log(`theme: ${e.themeSlug}  media: ${e.mediaInstalled}  mediaErrors: ${e.mediaErrors.length}  fetchErrors: ${e.fetchErrors.length}  missingMediaDownloaded: ${e.missingMediaDownloaded??0}  fontsLocalized: ${e.fontsLocalized??0}`),e.missingMediaFailed?.length&&console.log("  missingMediaFailed:",JSON.stringify(e.missingMediaFailed)),e.fontsFailed?.length&&console.log("  fontsFailed:",JSON.stringify(e.fontsFailed)),f&&console.log(`editable islands: converted=${e.islandsConverted??0}  plugin=${e.editableHtmlPluginSlug??"(none)"}`);let p=e.residualCdnAssets??0;console.log(`self-host audit: residualCdnAssets=${p}${p?`  byHost=${JSON.stringify(e.residualCdnByHost)}`:" \u2713"}`),p&&console.log("  residualCdnSamples:",JSON.stringify(e.residualCdnSamples)),e.mediaErrors.length&&console.log("  mediaErrors sample:",JSON.stringify(e.mediaErrors.slice(0,3))),e.fetchErrors.length&&console.log("  fetchErrors:",JSON.stringify(e.fetchErrors));let i=r(d,"wp-content/uploads/_carry-islands");O(i,{recursive:!0});let u=e.pages.map(t=>t.slug),l=w(i,u);l.length&&console.log(`  removed ${l.length} stale island(s): ${l.slice(0,8).join(", ")}${l.length>8?" \u2026":""}`);for(let t of e.pages)g(r(i,`${t.slug}.html`),t.postContent);let v=`<?php
 $dir = '/wordpress/wp-content/uploads/_carry-islands';
 $slugs = ${JSON.stringify(u)};
 foreach ($slugs as $slug) {

packages/data-liberation-agent/dist/scripts/carry-replica-shots.mjs

@@ -1,3 +1,3 @@
 import { createRequire as __bundleCreateRequire } from 'node:module';
 const require = __bundleCreateRequire(import.meta.url);
-import{b as y}from"./chunk-6FIZQDM2.mjs";import"./chunk-4TKBPMF4.mjs";import{chromium as A}from"playwright";async function b(r,o,n){let s=new Array(r.length),i=0,t=async()=>{for(let e=i++;e<r.length;e=i++)s[e]=await n(r[e],e)},u=Math.max(1,Math.min(Math.floor(o)||1,r.length||1));return await Promise.all(Array.from({length:u},()=>t())),s}import{readFileSync as d,writeFileSync as N,mkdirSync as R,existsSync as T}from"node:fs";import{join as a}from"node:path";var[f,O,m]=process.argv.slice(2);(!f||!O||!m)&&(console.error("usage: node scripts/run.mjs carry-replica-shots <originDir> <carryBaseUrl> <replicaDir>"),process.exit(2));var v=JSON.parse(d(a(f,"screenshots/manifest.json"),"utf8")),x=v.entries||v,w=null,P=a(f,"carry-pages.json");if(T(P)){let r=JSON.parse(d(P,"utf8"));w=new Set(r.map(o=>o.htmlSlug).filter(o=>!!o))}var $=new Map,S=a(f,"redirect-map.json");if(T(S)){let r=JSON.parse(d(S,"utf8"));for(let o of r)$.set(o.from.replace(/\/$/,""),o.to)}var C=r=>$.get(r.replace(/\/$/,""))??r;R(a(m,"desktop"),{recursive:!0});R(a(m,"mobile"),{recursive:!0});async function U(r){await r.evaluate(async()=>{let n=Math.min(document.body.scrollHeight,window.innerHeight*2);for(let t=0;t<n;t+=600)window.scrollTo(0,t),await new Promise(u=>setTimeout(u,60));window.scrollTo(0,0);let s=Array.from(document.images).filter(t=>t.getBoundingClientRect().top<window.innerHeight*2),i=Promise.all(s.map(t=>t.complete?Promise.resolve():t.decode().catch(()=>{})));await Promise.race([i,new Promise(t=>setTimeout(t,4e3))])});let o=await r.evaluate(()=>!!document.querySelector(".lib-carry-mobile-frame"));await r.waitForTimeout(o?2500:400)}var M=Math.max(1,Math.min(12,Number(process.argv[5]??process.env.CONCURRENCY??6)||6)),j="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",F=[{vp:{width:1440,height:900},scale:.7,dir:"desktop",mobile:!1},{vp:{width:390,height:844},scale:1,dir:"mobile",mobile:!0}];async function E(){let r=O.replace(/\/$/,""),o=await A.launch(),n=Object.entries(x).filter(([,e])=>e.slug&&(!w||w.has(e.slug))),s=new Map;for(let[e,c]of n)s.set(e,{slug:c.slug});let i=[];for(let[e,c]of n)for(let g of F)i.push({url:e,slug:c.slug,...g});let t=0;await b(i,M,async e=>{let c=new URL(e.url).pathname,g=await o.newContext(e.mobile?{viewport:e.vp,deviceScaleFactor:e.scale,isMobile:!0,hasTouch:!0,userAgent:j}:{viewport:e.vp,deviceScaleFactor:e.scale}),l=await g.newPage();await y(l);try{for(let h=1;;h++)try{await l.goto(r+C(c),{waitUntil:"load",timeout:45e3});break}catch(k){if(h>=3)throw k;await l.waitForTimeout(1500*h)}await U(l);let p=`${e.dir}/${e.slug}.png`;await l.screenshot({path:a(m,p),fullPage:!1}),s.get(e.url)[e.dir]=p}catch(p){t++,console.warn(`  fail ${e.dir} ${c}: ${p.message.slice(0,60)}`)}await l.close(),await g.close()});let u={version:1,entries:Object.fromEntries(s)};N(a(m,"manifest.json"),JSON.stringify(u,null,2)),console.log(`captured ${s.size} urls (${t} viewport failures, concurrency ${M}) -> ${m}`),await o.close()}E().catch(r=>{console.error(r),process.exit(1)});
+import{a as b}from"./chunk-O5KY3JUP.mjs";import{b as y}from"./chunk-6FIZQDM2.mjs";import"./chunk-4TKBPMF4.mjs";import{chromium as R}from"playwright";import{readFileSync as w,writeFileSync as U,mkdirSync as O,existsSync as $}from"node:fs";import{join as s}from"node:path";var[p,N,c]=process.argv.slice(2);(!p||!N||!c)&&(console.error("usage: node scripts/run.mjs carry-replica-shots <originDir> <carryBaseUrl> <replicaDir>"),process.exit(2));var v=JSON.parse(w(s(p,"screenshots/manifest.json"),"utf8")),j=v.entries||v,d=null,S=s(p,"carry-pages.json");if($(S)){let r=JSON.parse(w(S,"utf8"));d=new Set(r.map(i=>i.htmlSlug).filter(i=>!!i))}var T=new Map,P=s(p,"redirect-map.json");if($(P)){let r=JSON.parse(w(P,"utf8"));for(let i of r)T.set(i.from.replace(/\/$/,""),i.to)}var k=r=>T.get(r.replace(/\/$/,""))??r;O(s(c,"desktop"),{recursive:!0});O(s(c,"mobile"),{recursive:!0});async function A(r){await r.evaluate(async()=>{let l=Math.min(document.body.scrollHeight,window.innerHeight*2);for(let t=0;t<l;t+=600)window.scrollTo(0,t),await new Promise(f=>setTimeout(f,60));window.scrollTo(0,0);let o=Array.from(document.images).filter(t=>t.getBoundingClientRect().top<window.innerHeight*2),m=Promise.all(o.map(t=>t.complete?Promise.resolve():t.decode().catch(()=>{})));await Promise.race([m,new Promise(t=>setTimeout(t,4e3))])});let i=await r.evaluate(()=>!!document.querySelector(".lib-carry-mobile-frame"));await r.waitForTimeout(i?2500:400)}var M=Math.max(1,Math.min(12,Number(process.argv[5]??process.env.CONCURRENCY??6)||6)),F="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",E=[{vp:{width:1440,height:900},scale:.7,dir:"desktop",mobile:!1},{vp:{width:390,height:844},scale:1,dir:"mobile",mobile:!0}];async function x(){let r=N.replace(/\/$/,""),i=await R.launch(),l=Object.entries(j).filter(([,e])=>e.slug&&(!d||d.has(e.slug))),o=new Map;for(let[e,a]of l)o.set(e,{slug:a.slug});let m=[];for(let[e,a]of l)for(let u of E)m.push({url:e,slug:a.slug,...u});let t=0;await b(m,M,async e=>{let a=new URL(e.url).pathname,u=await i.newContext(e.mobile?{viewport:e.vp,deviceScaleFactor:e.scale,isMobile:!0,hasTouch:!0,userAgent:F}:{viewport:e.vp,deviceScaleFactor:e.scale}),n=await u.newPage();await y(n);try{for(let h=1;;h++)try{await n.goto(r+k(a),{waitUntil:"load",timeout:45e3});break}catch(C){if(h>=3)throw C;await n.waitForTimeout(1500*h)}await A(n);let g=`${e.dir}/${e.slug}.png`;await n.screenshot({path:s(c,g),fullPage:!1}),o.get(e.url)[e.dir]=g}catch(g){t++,console.warn(`  fail ${e.dir} ${a}: ${g.message.slice(0,60)}`)}await n.close(),await u.close()});let f={version:1,entries:Object.fromEntries(o)};U(s(c,"manifest.json"),JSON.stringify(f,null,2)),console.log(`captured ${o.size} urls (${t} viewport failures, concurrency ${M}) -> ${c}`),await i.close()}x().catch(r=>{console.error(r),process.exit(1)});