Data Liberation: Add timeouts to browser calls so extraction cannot hang

packages/data-liberation-agent/dist/scripts/carry-reconstruct-drive.mjs

@@ -3,7 +3,7 @@ const require = __bundleCreateRequire(import.meta.url);
 import{a as $,b as y,c as w,d as S}from"./chunk-ENWLFTGA.mjs";import"./chunk-4TKBPMF4.mjs";import{writeFileSync as g,existsSync as P,mkdirSync as O,copyFileSync as D}from"node:fs";import{join as r}from"node:path";function N(s){console.log(`pages: ${s.pages.length} (${s.pages.filter(o=>o.postType==="post").length} posts, home=${s.pages.some(o=>o.isHome)})`),s.excluded.length&&console.log(`excluded (${s.excluded.length}): ${s.excluded.map(o=>o.slug).join(", ")}`),s.skipped.length&&console.log(`skipped \u2014 no captured html (${s.skipped.length}): ${s.skipped.map(o=>`${o.postType}:${o.postName}`).join(", ")}`)}async function k(){let s=process.argv.slice(2),o=s.includes("--list"),m=s.includes("--slim"),[n,d,_="Liberated (Carry)"]=s.filter(t=>t!=="--list"&&t!=="--slim");if((!n||!o&&!m&&!d)&&(console.error(`usage:
   node scripts/run.mjs carry-reconstruct-drive <outputDir> --slim   # slim output.wxr for provisioning (run BEFORE liberate_preview)
   node scripts/run.mjs carry-reconstruct-drive <outputDir> --list   # inspect the carry page list (no handler)
-  node scripts/run.mjs carry-reconstruct-drive <outputDir> <studioSitePath> "<Theme Name>"`),process.exit(2)),m){let{dropped:t,flipped:C}=S(n);console.log(`slimmed: dropped ${t} attachment items, flipped ${C} draft->publish; full WXR preserved at output.wxr.full`);return}let b=(process.env.EXCLUDE??"").split(",").map(t=>t.trim()).filter(Boolean),c=$(n,{exclude:b});if(g(r(n,"carry-pages.json"),JSON.stringify(c.pages,null,2)),N(c),o){console.log(`wrote ${r(n,"carry-pages.json")}`);return}let{reconstructPagesCarryHandler:x}=await import("./chunk-LTZ532K4.mjs"),E={textResult:t=>({_data:t}),errorResult:t=>{throw new Error(`handler error: ${t}`)}},f=!/^(0|false)$/i.test(process.env.EDITABLE_ISLANDS??""),e=(await x({outputDir:n,studioSitePath:d,themeName:_,pages:c.pages,editableIslands:f},E))._data;console.log(`theme: ${e.themeSlug}  media: ${e.mediaInstalled}  mediaErrors: ${e.mediaErrors.length}  fetchErrors: ${e.fetchErrors.length}  missingMediaDownloaded: ${e.missingMediaDownloaded??0}  fontsLocalized: ${e.fontsLocalized??0}`),e.missingMediaFailed?.length&&console.log("  missingMediaFailed:",JSON.stringify(e.missingMediaFailed)),e.fontsFailed?.length&&console.log("  fontsFailed:",JSON.stringify(e.fontsFailed)),f&&console.log(`editable islands: converted=${e.islandsConverted??0}  plugin=${e.editableHtmlPluginSlug??"(none)"}`);let p=e.residualCdnAssets??0;console.log(`self-host audit: residualCdnAssets=${p}${p?`  byHost=${JSON.stringify(e.residualCdnByHost)}`:" \u2713"}`),p&&console.log("  residualCdnSamples:",JSON.stringify(e.residualCdnSamples)),e.mediaErrors.length&&console.log("  mediaErrors sample:",JSON.stringify(e.mediaErrors.slice(0,3))),e.fetchErrors.length&&console.log("  fetchErrors:",JSON.stringify(e.fetchErrors));let i=r(d,"wp-content/uploads/_carry-islands");O(i,{recursive:!0});let u=e.pages.map(t=>t.slug),l=w(i,u);l.length&&console.log(`  removed ${l.length} stale island(s): ${l.slice(0,8).join(", ")}${l.length>8?" \u2026":""}`);for(let t of e.pages)g(r(i,`${t.slug}.html`),t.postContent);let v=`<?php
+  node scripts/run.mjs carry-reconstruct-drive <outputDir> <studioSitePath> "<Theme Name>"`),process.exit(2)),m){let{dropped:t,flipped:C}=S(n);console.log(`slimmed: dropped ${t} attachment items, flipped ${C} draft->publish; full WXR preserved at output.wxr.full`);return}let b=(process.env.EXCLUDE??"").split(",").map(t=>t.trim()).filter(Boolean),c=$(n,{exclude:b});if(g(r(n,"carry-pages.json"),JSON.stringify(c.pages,null,2)),N(c),o){console.log(`wrote ${r(n,"carry-pages.json")}`);return}let{reconstructPagesCarryHandler:x}=await import("./chunk-T5UNQMBT.mjs"),E={textResult:t=>({_data:t}),errorResult:t=>{throw new Error(`handler error: ${t}`)}},f=!/^(0|false)$/i.test(process.env.EDITABLE_ISLANDS??""),e=(await x({outputDir:n,studioSitePath:d,themeName:_,pages:c.pages,editableIslands:f},E))._data;console.log(`theme: ${e.themeSlug}  media: ${e.mediaInstalled}  mediaErrors: ${e.mediaErrors.length}  fetchErrors: ${e.fetchErrors.length}  missingMediaDownloaded: ${e.missingMediaDownloaded??0}  fontsLocalized: ${e.fontsLocalized??0}`),e.missingMediaFailed?.length&&console.log("  missingMediaFailed:",JSON.stringify(e.missingMediaFailed)),e.fontsFailed?.length&&console.log("  fontsFailed:",JSON.stringify(e.fontsFailed)),f&&console.log(`editable islands: converted=${e.islandsConverted??0}  plugin=${e.editableHtmlPluginSlug??"(none)"}`);let p=e.residualCdnAssets??0;console.log(`self-host audit: residualCdnAssets=${p}${p?`  byHost=${JSON.stringify(e.residualCdnByHost)}`:" \u2713"}`),p&&console.log("  residualCdnSamples:",JSON.stringify(e.residualCdnSamples)),e.mediaErrors.length&&console.log("  mediaErrors sample:",JSON.stringify(e.mediaErrors.slice(0,3))),e.fetchErrors.length&&console.log("  fetchErrors:",JSON.stringify(e.fetchErrors));let i=r(d,"wp-content/uploads/_carry-islands");O(i,{recursive:!0});let u=e.pages.map(t=>t.slug),l=w(i,u);l.length&&console.log(`  removed ${l.length} stale island(s): ${l.slice(0,8).join(", ")}${l.length>8?" \u2026":""}`);for(let t of e.pages)g(r(i,`${t.slug}.html`),t.postContent);let v=`<?php
 $dir = '/wordpress/wp-content/uploads/_carry-islands';
 $slugs = ${JSON.stringify(u)};
 foreach ($slugs as $slug) {

packages/data-liberation-agent/dist/scripts/carry-replica-shots.mjs

@@ -1,3 +1,3 @@
 import { createRequire as __bundleCreateRequire } from 'node:module';
 const require = __bundleCreateRequire(import.meta.url);
-import{b as y}from"./chunk-6FIZQDM2.mjs";import"./chunk-4TKBPMF4.mjs";import{chromium as A}from"playwright";async function b(r,o,n){let s=new Array(r.length),i=0,t=async()=>{for(let e=i++;e<r.length;e=i++)s[e]=await n(r[e],e)},u=Math.max(1,Math.min(Math.floor(o)||1,r.length||1));return await Promise.all(Array.from({length:u},()=>t())),s}import{readFileSync as d,writeFileSync as N,mkdirSync as R,existsSync as T}from"node:fs";import{join as a}from"node:path";var[f,O,m]=process.argv.slice(2);(!f||!O||!m)&&(console.error("usage: node scripts/run.mjs carry-replica-shots <originDir> <carryBaseUrl> <replicaDir>"),process.exit(2));var v=JSON.parse(d(a(f,"screenshots/manifest.json"),"utf8")),x=v.entries||v,w=null,P=a(f,"carry-pages.json");if(T(P)){let r=JSON.parse(d(P,"utf8"));w=new Set(r.map(o=>o.htmlSlug).filter(o=>!!o))}var $=new Map,S=a(f,"redirect-map.json");if(T(S)){let r=JSON.parse(d(S,"utf8"));for(let o of r)$.set(o.from.replace(/\/$/,""),o.to)}var C=r=>$.get(r.replace(/\/$/,""))??r;R(a(m,"desktop"),{recursive:!0});R(a(m,"mobile"),{recursive:!0});async function U(r){await r.evaluate(async()=>{let n=Math.min(document.body.scrollHeight,window.innerHeight*2);for(let t=0;t<n;t+=600)window.scrollTo(0,t),await new Promise(u=>setTimeout(u,60));window.scrollTo(0,0);let s=Array.from(document.images).filter(t=>t.getBoundingClientRect().top<window.innerHeight*2),i=Promise.all(s.map(t=>t.complete?Promise.resolve():t.decode().catch(()=>{})));await Promise.race([i,new Promise(t=>setTimeout(t,4e3))])});let o=await r.evaluate(()=>!!document.querySelector(".lib-carry-mobile-frame"));await r.waitForTimeout(o?2500:400)}var M=Math.max(1,Math.min(12,Number(process.argv[5]??process.env.CONCURRENCY??6)||6)),j="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",F=[{vp:{width:1440,height:900},scale:.7,dir:"desktop",mobile:!1},{vp:{width:390,height:844},scale:1,dir:"mobile",mobile:!0}];async function E(){let r=O.replace(/\/$/,""),o=await A.launch(),n=Object.entries(x).filter(([,e])=>e.slug&&(!w||w.has(e.slug))),s=new Map;for(let[e,c]of n)s.set(e,{slug:c.slug});let i=[];for(let[e,c]of n)for(let g of F)i.push({url:e,slug:c.slug,...g});let t=0;await b(i,M,async e=>{let c=new URL(e.url).pathname,g=await o.newContext(e.mobile?{viewport:e.vp,deviceScaleFactor:e.scale,isMobile:!0,hasTouch:!0,userAgent:j}:{viewport:e.vp,deviceScaleFactor:e.scale}),l=await g.newPage();await y(l);try{for(let h=1;;h++)try{await l.goto(r+C(c),{waitUntil:"load",timeout:45e3});break}catch(k){if(h>=3)throw k;await l.waitForTimeout(1500*h)}await U(l);let p=`${e.dir}/${e.slug}.png`;await l.screenshot({path:a(m,p),fullPage:!1}),s.get(e.url)[e.dir]=p}catch(p){t++,console.warn(`  fail ${e.dir} ${c}: ${p.message.slice(0,60)}`)}await l.close(),await g.close()});let u={version:1,entries:Object.fromEntries(s)};N(a(m,"manifest.json"),JSON.stringify(u,null,2)),console.log(`captured ${s.size} urls (${t} viewport failures, concurrency ${M}) -> ${m}`),await o.close()}E().catch(r=>{console.error(r),process.exit(1)});
+import{b as y}from"./chunk-6FIZQDM2.mjs";import{a as b}from"./chunk-6FA2POBN.mjs";import"./chunk-4TKBPMF4.mjs";import{chromium as R}from"playwright";import{readFileSync as w,writeFileSync as U,mkdirSync as O,existsSync as $}from"node:fs";import{join as s}from"node:path";var[p,N,c]=process.argv.slice(2);(!p||!N||!c)&&(console.error("usage: node scripts/run.mjs carry-replica-shots <originDir> <carryBaseUrl> <replicaDir>"),process.exit(2));var v=JSON.parse(w(s(p,"screenshots/manifest.json"),"utf8")),j=v.entries||v,d=null,S=s(p,"carry-pages.json");if($(S)){let r=JSON.parse(w(S,"utf8"));d=new Set(r.map(i=>i.htmlSlug).filter(i=>!!i))}var T=new Map,P=s(p,"redirect-map.json");if($(P)){let r=JSON.parse(w(P,"utf8"));for(let i of r)T.set(i.from.replace(/\/$/,""),i.to)}var k=r=>T.get(r.replace(/\/$/,""))??r;O(s(c,"desktop"),{recursive:!0});O(s(c,"mobile"),{recursive:!0});async function A(r){await r.evaluate(async()=>{let l=Math.min(document.body.scrollHeight,window.innerHeight*2);for(let t=0;t<l;t+=600)window.scrollTo(0,t),await new Promise(f=>setTimeout(f,60));window.scrollTo(0,0);let o=Array.from(document.images).filter(t=>t.getBoundingClientRect().top<window.innerHeight*2),m=Promise.all(o.map(t=>t.complete?Promise.resolve():t.decode().catch(()=>{})));await Promise.race([m,new Promise(t=>setTimeout(t,4e3))])});let i=await r.evaluate(()=>!!document.querySelector(".lib-carry-mobile-frame"));await r.waitForTimeout(i?2500:400)}var M=Math.max(1,Math.min(12,Number(process.argv[5]??process.env.CONCURRENCY??6)||6)),F="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",E=[{vp:{width:1440,height:900},scale:.7,dir:"desktop",mobile:!1},{vp:{width:390,height:844},scale:1,dir:"mobile",mobile:!0}];async function x(){let r=N.replace(/\/$/,""),i=await R.launch(),l=Object.entries(j).filter(([,e])=>e.slug&&(!d||d.has(e.slug))),o=new Map;for(let[e,a]of l)o.set(e,{slug:a.slug});let m=[];for(let[e,a]of l)for(let u of E)m.push({url:e,slug:a.slug,...u});let t=0;await b(m,M,async e=>{let a=new URL(e.url).pathname,u=await i.newContext(e.mobile?{viewport:e.vp,deviceScaleFactor:e.scale,isMobile:!0,hasTouch:!0,userAgent:F}:{viewport:e.vp,deviceScaleFactor:e.scale}),n=await u.newPage();await y(n);try{for(let h=1;;h++)try{await n.goto(r+k(a),{waitUntil:"load",timeout:45e3});break}catch(C){if(h>=3)throw C;await n.waitForTimeout(1500*h)}await A(n);let g=`${e.dir}/${e.slug}.png`;await n.screenshot({path:s(c,g),fullPage:!1}),o.get(e.url)[e.dir]=g}catch(g){t++,console.warn(`  fail ${e.dir} ${a}: ${g.message.slice(0,60)}`)}await n.close(),await u.close()});let f={version:1,entries:Object.fromEntries(o)};U(s(c,"manifest.json"),JSON.stringify(f,null,2)),console.log(`captured ${o.size} urls (${t} viewport failures, concurrency ${M}) -> ${c}`),await i.close()}x().catch(r=>{console.error(r),process.exit(1)});

packages/data-liberation-agent/dist/scripts/chunk-6FA2POBN.mjs

@@ -0,0 +1,3 @@
+import { createRequire as __bundleCreateRequire } from 'node:module';
+const require = __bundleCreateRequire(import.meta.url);
+async function u(e,r,n){let i=new Array(e.length),t=0,a=async()=>{for(let o=t++;o<e.length;o=t++)i[o]=await n(e[o],o)},s=Math.max(1,Math.min(Math.floor(r)||1,e.length||1));return await Promise.all(Array.from({length:s},()=>a())),i}var m=class extends Error{label;constructor(r,n){super(`${r} timed out after ${n}ms`),this.name="TimeoutError",this.label=r}};async function c(e,r,n,i){let t,a=!1,s=new Promise((o,l)=>{t=setTimeout(()=>{a=!0,l(new m(n,r))},r)});try{return await Promise.race([e,s])}finally{t&&clearTimeout(t),a&&i&&e.then(i).catch(()=>{})}}export{u as a,c as b};

packages/data-liberation-agent/dist/scripts/chunk-RS7EWLKJ.mjs

@@ -0,0 +1,3 @@
+import { createRequire as __bundleCreateRequire } from 'node:module';
+const require = __bundleCreateRequire(import.meta.url);
+import{c as p}from"./chunk-LFW5THTQ.mjs";import{readFileSync as P}from"node:fs";var h=[{name:"loox",selector:'#looxReviews, .loox-reviews, [id^="looxReviews"], [data-loox]'},{name:"yotpo",selector:'.yotpo, [class*="yotpo-"]'},{name:"judgeme",selector:".jdgm-widget, .jdgm-rev-widg, [data-jdgm-widget]"},{name:"okendo",selector:"[data-oke-widget], .okeReviews"},{name:"stamped",selector:"#stamped-main-widget, .stamped-main-widget"},{name:"reviews-io",selector:"#reviewsio-carousel-widget, .ruk_rating_snippet"},{name:"zendesk",selector:'iframe[src*="zendesk"], [id*="zendesk"]'},{name:"gorgias",selector:'iframe[src*="gorgias"]'},{name:"elfsight",selector:'[class*="elfsight-app"]'}],_=h.map(t=>t.selector).join(", ");function L(t,r){let o=p(t);o("script, style, noscript, template, svg").remove();let i=o("body"),e=h.find(c=>i.find(c.selector).length>0)?.name??null,s=i.find("iframe[src]").toArray().some(c=>{let n=o(c).attr("src")||"";if(!/^https?:\/\//i.test(n))return!1;try{return!r||new URL(n).origin!==r}catch{return!0}}),g=i.clone();g.find('header, nav, footer, [role="banner"], [role="contentinfo"], [role="navigation"]').remove();let l=g.text().replace(/\s+/g," ").trim().length,d=l<200,w=s?"iframe":e?"app-widget":d?"thin":"ok",a=s?"cross-origin <iframe> body":e||`${l} chars of body text`;return{empty:d,reason:w,detail:a,widget:e,crossOriginIframe:s,mainTextLen:l}}function O(t){try{let r=P(t);if(r.length<24||r.readUInt32BE(0)!==2303741511||r.toString("latin1",12,16)!=="IHDR")return null;let o=r.readUInt32BE(20);return o>0&&o<=2e5?o:null}catch{return null}}var x=1e3,b=.5;function R(t){let r=t.map(e=>e.height).filter(e=>e!==null).sort((e,s)=>e-s),o=r.length?r[Math.floor(r.length/2)]:0,i=[];for(let e of t){let s=o>0&&e.height!==null&&e.height<o*b;if(!(e.height!==null?s&&e.assess.mainTextLen<x:e.assess.empty))continue;let l=e.assess.crossOriginIframe?"iframe":e.assess.widget?"app-widget":e.height!==null?"short-render":"thin",m=e.assess.crossOriginIframe?e.assess.detail:e.assess.widget?e.assess.widget:e.height!==null?`rendered ${e.height}px vs median ${o}px`:e.assess.detail;i.push({slug:e.slug,reason:l,detail:m})}return i}import{readFileSync as T}from"node:fs";import{join as f}from"node:path";function y(t){try{return JSON.parse(T(t,"utf8"))}catch{return null}}function B(t){return(t.split(",")[0]||"").trim().replace(/["']/g,"").toLowerCase().replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"")||"font"}function E(t){return(t.split(",")[0]||"").trim().replace(/["']/g,"")||t}function I(t,r=8){let o=y(f(t,"palette.json")),i=Array.isArray(o?.colors)?o.colors:[],e=[],s=[];for(let a of i){if(!a?.hex||!/^#?(?:[0-9a-f]{3}|[0-9a-f]{4}|[0-9a-f]{6}|[0-9a-f]{8})$/i.test(a.hex))continue;let c=`c${e.length+1}`,n=a.hex.startsWith("#")?a.hex:`#${a.hex}`;if(e.push({slug:c,hex:n}),s.push({slug:c,name:`Replica ${e.length}`,color:n}),e.length>=r)break}let l=y(f(t,"typography.json"))?.bySelector??{},m=new Set,d=[],w=[];for(let a of Object.values(l))for(let c of a??[]){let n=(c?.fontFamily||"").trim();if(!n)continue;let u=B(n);m.has(u)||(m.add(u),d.push({slug:u,family:n}),w.push({slug:u,name:E(n),fontFamily:n}))}return{paletteTokens:e,fontFamilies:d,themeJsonPalette:s,themeJsonFontFamilies:w}}async function v(){try{return await import("playwright")}catch{throw new Error("Playwright is required but is not installed. Run `npm install playwright` and `npx playwright install chromium` to set it up.")}}export{v as a,L as b,O as c,R as d,I as e};