Route studio create --from URLs through Data Liberation

apps/cli/commands/site/create.ts

@@ -76,6 +76,7 @@ import {
 } from 'cli/lib/cli-config/core';
 import { removeSiteFromConfig } from 'cli/lib/cli-config/sites';
 import { connectToDaemon, disconnectFromDaemon, emitCliEvent } from 'cli/lib/daemon-client';
+import { liberateWebsite } from 'cli/lib/data-liberation-client';
 import {
 	getAiInstructionsPath,
 	getWordPressVersionPath,
@@ -1132,7 +1133,10 @@ function coerceWpVersion( value: string ) {
 	return value;
 }
 
-export const registerCommand = ( yargs: StudioArgv ) => {
+export const registerCommand = (
+	yargs: StudioArgv,
+	dependencies: { liberate?: typeof liberateWebsite } = {}
+) => {
 	return yargs.command( {
 		command: 'create',
 		describe: __( 'Create a new site' ),
@@ -1481,11 +1485,25 @@ export const registerCommand = ( yargs: StudioArgv ) => {
 			};
 
 			try {
-				const importSource = argv.from;
-				// Remote URLs are rendered into a local source by Data Liberation before they
-				// reach SSI; until that path exists here, `resolveStaticSiteImporterSource`
-				// rejects them. `sourceUrl` still carries provenance for local captures.
+				let importSource = argv.from;
 				const sourceUrl = importSource && isUrl( importSource ) ? importSource : undefined;
+				let liberationOutputDir: string | undefined;
+				if ( sourceUrl ) {
+					liberationOutputDir = path.join(
+						path.dirname( sitePath ),
+						`${ path.basename( sitePath ) }-source`
+					);
+					defaultLogger.reportStart(
+						LoggerAction.IMPORT_SITE,
+						__( 'Preparing source website with Data Liberation…' )
+					);
+					importSource = await ( dependencies.liberate ?? liberateWebsite )(
+						sourceUrl,
+						liberationOutputDir,
+						{ onProgress: ( message ) => defaultLogger.reportProgress( message ) }
+					);
+					defaultLogger.reportSuccess( __( 'Source website prepared' ) );
+				}
 
 				if ( importSource ) {
 					config.blueprint = buildCreateFromSourceBlueprint(
@@ -1524,6 +1542,11 @@ export const registerCommand = ( yargs: StudioArgv ) => {
 
 				try {
 					await runCommand( sitePath, config );
+					if ( sourceUrl && liberationOutputDir ) {
+						await fs.promises
+							.rm( liberationOutputDir, { recursive: true, force: true } )
+							.catch( () => {} );
+					}
 				} finally {
 					const bundlePath = config.blueprint?.staticSiteImport?.bundlePath;
 					if ( bundlePath ) {

apps/cli/commands/site/tests/create.test.ts

@@ -394,6 +394,44 @@ describe( 'CLI: studio create', () => {
 	} );
 
 	describe( 'Success Cases', () => {
+		it( 'liberates a URL before handing the portable site to SSI', async () => {
+			const captureRoot = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-url-source-' ) );
+			const websiteDir = path.join( captureRoot, 'example.com', 'website' );
+			await fs.promises.mkdir( websiteDir, { recursive: true } );
+			fs.writeFileSync( path.join( websiteDir, 'index.html' ), '<main>Liberated</main>' );
+			const liberate = vi.fn().mockResolvedValue( websiteDir );
+			vi.spyOn( fs, 'writeFileSync' ).mockImplementation( () => {} );
+			const copySpy = vi.spyOn( fs.promises, 'cp' ).mockResolvedValue( undefined );
+			const rmSpy = vi.spyOn( fs.promises, 'rm' ).mockResolvedValue( undefined );
+			const parser = registerCommand(
+				yargs( [] ).option( 'path', { type: 'string', default: mockSitePath } ),
+				{ liberate }
+			).exitProcess( false );
+
+			await parser.parseAsync( [
+				'create',
+				'--from',
+				'https://example.com',
+				'--name',
+				'Liberated Site',
+				'--no-start',
+				'--skip-browser',
+			] );
+
+			const outputBase = path.normalize( `${ mockSitePath }-source` );
+			expect( liberate ).toHaveBeenCalledWith(
+				'https://example.com',
+				outputBase,
+				expect.objectContaining( { onProgress: expect.any( Function ) } )
+			);
+			expect( copySpy ).toHaveBeenCalledWith(
+				websiteDir,
+				path.join( mockSitePath, '.studio-import', 'source' ),
+				{ recursive: true, errorOnExist: true, force: false }
+			);
+			expect( rmSpy ).toHaveBeenCalledWith( outputBase, { recursive: true, force: true } );
+		} );
+
 		it( 'bundles a local Static Site Importer zip until Blueprint execution finishes', async () => {
 			const sourceDir = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-source-test-' ) );
 			const pluginDir = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-ssi-plugin-' ) );

apps/cli/lib/data-liberation-client.ts

@@ -0,0 +1,123 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { ensurePlaywrightChromiumInstalled } from 'cli/ai/browser-utils';
+
+type DataLiberationCliResult = {
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+};
+
+export type RunDataLiberationCli = (
+	args: string[],
+	onProgress?: ( message: string ) => void
+) => Promise< DataLiberationCliResult >;
+
+type LiberateWebsiteOptions = {
+	onProgress?: ( message: string ) => void;
+	runCli?: RunDataLiberationCli;
+};
+
+export function getDataLiberationCliPath(): string {
+	return path.join( import.meta.dirname, 'data-liberation-agent', 'dist', 'cli.bundle.mjs' );
+}
+
+function appendBounded( current: string, chunk: string ): string {
+	return ( current + chunk ).slice( -64 * 1024 );
+}
+
+async function runDataLiberationCli(
+	args: string[],
+	onProgress?: ( message: string ) => void
+): Promise< DataLiberationCliResult > {
+	const { chromium } = await import( 'playwright' );
+	const browserProblem = await ensurePlaywrightChromiumInstalled( chromium );
+	if ( browserProblem ) {
+		throw new Error( browserProblem );
+	}
+
+	const cliPath = getDataLiberationCliPath();
+	if ( ! fs.existsSync( cliPath ) ) {
+		throw new Error(
+			'Data Liberation CLI is not compiled. Run `npm -w data-liberation run build:mcp-bundle` and try again.'
+		);
+	}
+
+	return new Promise( ( resolve, reject ) => {
+		const child = spawn( process.execPath, [ cliPath, ...args ], {
+			cwd: path.dirname( path.dirname( cliPath ) ),
+			stdio: [ 'ignore', 'pipe', 'pipe' ],
+		} );
+		let stdout = '';
+		let stderr = '';
+		let pendingProgress = '';
+
+		child.stdout.setEncoding( 'utf8' );
+		child.stderr.setEncoding( 'utf8' );
+		child.stdout.on( 'data', ( chunk: string ) => {
+			stdout = appendBounded( stdout, chunk );
+		} );
+		child.stderr.on( 'data', ( chunk: string ) => {
+			stderr = appendBounded( stderr, chunk );
+			pendingProgress += chunk;
+			const lines = pendingProgress.split( /\r?\n/ );
+			pendingProgress = lines.pop() ?? '';
+			for ( const line of lines ) {
+				if ( line.trim() ) {
+					onProgress?.( line.trim() );
+				}
+			}
+		} );
+		child.once( 'error', reject );
+		child.once( 'close', ( code ) => {
+			if ( pendingProgress.trim() ) {
+				onProgress?.( pendingProgress.trim() );
+			}
+			resolve( { exitCode: code ?? 1, stdout, stderr } );
+		} );
+	} );
+}
+
+export async function liberateWebsite(
+	url: string,
+	outputBase: string,
+	options: LiberateWebsiteOptions = {}
+): Promise< string > {
+	const parsed = new URL( url );
+	if ( ! [ 'http:', 'https:' ].includes( parsed.protocol ) ) {
+		throw new Error( 'Source URLs must use HTTP or HTTPS.' );
+	}
+
+	const resolvedOutputBase = path.resolve( outputBase );
+	fs.mkdirSync( resolvedOutputBase, { recursive: true } );
+	const result = await ( options.runCli ?? runDataLiberationCli )(
+		[ parsed.href, '--output', resolvedOutputBase, '--resume' ],
+		options.onProgress
+	);
+	if ( result.exitCode !== 0 ) {
+		throw new Error( result.stderr.trim() || result.stdout.trim() || 'Data Liberation failed.' );
+	}
+
+	const siteLine = result.stdout
+		.trim()
+		.split( /\r?\n/ )
+		.reverse()
+		.find( ( line ) => line.startsWith( 'Site: ' ) );
+	if ( ! siteLine ) {
+		throw new Error( 'Data Liberation completed without reporting a website directory.' );
+	}
+
+	const websiteDir = path.resolve( siteLine.slice( 'Site: '.length ).trim() );
+	const relativeWebsiteDir = path.relative( resolvedOutputBase, websiteDir );
+	if (
+		relativeWebsiteDir === '..' ||
+		relativeWebsiteDir.startsWith( `..${ path.sep }` ) ||
+		! fs.existsSync( websiteDir ) ||
+		! fs.statSync( websiteDir ).isDirectory()
+	) {
+		throw new Error( 'Data Liberation reported an invalid website directory.' );
+	}
+
+	return websiteDir;
+}

apps/cli/lib/tests/data-liberation-client.test.ts

@@ -0,0 +1,81 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { liberateWebsite } from '../data-liberation-client';
+
+const tempDirs: string[] = [];
+
+function createOutput(): { outputBase: string; websiteDir: string } {
+	const outputBase = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-liberation-' ) );
+	tempDirs.push( outputBase );
+	const websiteDir = path.join( outputBase, 'example.com', 'website' );
+	fs.mkdirSync( websiteDir, { recursive: true } );
+	fs.writeFileSync( path.join( websiteDir, 'index.html' ), '<main>Liberated</main>' );
+	return { outputBase, websiteDir };
+}
+
+afterEach( () => {
+	for ( const dir of tempDirs.splice( 0 ) ) {
+		fs.rmSync( dir, { recursive: true, force: true } );
+	}
+} );
+
+describe( 'Data Liberation CLI', () => {
+	it( 'returns the portable website directory reported by the CLI', async () => {
+		const { outputBase, websiteDir } = createOutput();
+		const onProgress = vi.fn();
+		const runCli = vi.fn().mockResolvedValue( {
+			exitCode: 0,
+			stdout: `Liberated 1/1 routes\nSite: ${ websiteDir }\n`,
+			stderr: '',
+		} );
+
+		await expect(
+			liberateWebsite( 'https://example.com', outputBase, { runCli, onProgress } )
+		).resolves.toBe( websiteDir );
+		expect( runCli ).toHaveBeenCalledWith(
+			[ 'https://example.com/', '--output', outputBase, '--resume' ],
+			onProgress
+		);
+	} );
+
+	it( 'surfaces CLI failures', async () => {
+		const { outputBase } = createOutput();
+
+		await expect(
+			liberateWebsite( 'https://example.com', outputBase, {
+				runCli: vi.fn().mockResolvedValue( {
+					exitCode: 1,
+					stdout: '',
+					stderr: 'Capture failed',
+				} ),
+			} )
+		).rejects.toThrow( 'Capture failed' );
+	} );
+
+	it( 'rejects a website directory outside its output base', async () => {
+		const { outputBase } = createOutput();
+		const outsideDir = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-liberation-outside-' ) );
+		tempDirs.push( outsideDir );
+
+		await expect(
+			liberateWebsite( 'https://example.com', outputBase, {
+				runCli: vi.fn().mockResolvedValue( {
+					exitCode: 0,
+					stdout: `Site: ${ outsideDir }\n`,
+					stderr: '',
+				} ),
+			} )
+		).rejects.toThrow( 'invalid website directory' );
+	} );
+
+	it( 'rejects non-HTTP sources before invoking the CLI', async () => {
+		const runCli = vi.fn();
+
+		await expect(
+			liberateWebsite( 'file:///tmp/index.html', '/tmp/capture', { runCli } )
+		).rejects.toThrow( 'HTTP or HTTPS' );
+		expect( runCli ).not.toHaveBeenCalled();
+	} );
+} );

apps/cli/vite.config.base.ts

@@ -77,8 +77,9 @@ export function buildLocalUiPlugin() {
 // Copy the committed data-liberation-agent/dist into the dist/cli/data-liberation-agent.
 function copyDataLiberationEngine( outDir: string ) {
 	const serverBundlePath = resolve( dataLiberationSourcePath, 'dist', 'mcp-server.bundle.mjs' );
+	const cliBundlePath = resolve( dataLiberationSourcePath, 'dist', 'cli.bundle.mjs' );
 	const scriptsDistPath = resolve( dataLiberationSourcePath, 'dist', 'scripts' );
-	if ( ! existsSync( serverBundlePath ) || ! existsSync( scriptsDistPath ) ) {
+	if ( ! existsSync( serverBundlePath ) || ! existsSync( cliBundlePath ) ) {
 		throw new Error(
 			'Data Liberation engine bundles are missing under packages/data-liberation-agent/dist/. ' +
 				'Run `npm -w data-liberation run build:mcp-bundle` and commit the updated artifacts.'
@@ -88,16 +89,23 @@ function copyDataLiberationEngine( outDir: string ) {
 	const engineOutDir = resolve( outDir, 'data-liberation-agent' );
 	mkdirSync( resolve( engineOutDir, 'dist' ), { recursive: true } );
 	copyFileSync( serverBundlePath, resolve( engineOutDir, 'dist', 'mcp-server.bundle.mjs' ) );
+	copyFileSync( cliBundlePath, resolve( engineOutDir, 'dist', 'cli.bundle.mjs' ) );
+	copyFileSync(
+		resolve( dataLiberationSourcePath, 'package.json' ),
+		resolve( engineOutDir, 'package.json' )
+	);
 	cpSync( resolve( dataLiberationSourcePath, 'skills' ), resolve( engineOutDir, 'skills' ), {
 		recursive: true,
 	} );
 
 	// The skills also invoke pipeline drivers via `node scripts/run.mjs <name>`.
 	// Ship the launcher plus the self-contained driver bundles it falls back to
 	// when no dev dependencies resolve next to it (dist/scripts/).
-	cpSync( scriptsDistPath, resolve( engineOutDir, 'dist', 'scripts' ), {
-		recursive: true,
-	} );
+	if ( existsSync( scriptsDistPath ) ) {
+		cpSync( scriptsDistPath, resolve( engineOutDir, 'dist', 'scripts' ), {
+			recursive: true,
+		} );
+	}
 	mkdirSync( resolve( engineOutDir, 'scripts' ), { recursive: true } );
 	copyFileSync(
 		resolve( dataLiberationSourcePath, 'scripts', 'run.mjs' ),

packages/data-liberation-agent/.claude-plugin/marketplace.json

@@ -0,0 +1,13 @@
+{
+  "name": "data-liberation",
+  "owner": {
+    "name": "Automattic"
+  },
+  "plugins": [
+    {
+      "name": "data-liberation",
+      "source": "./",
+      "description": "Liberate any website (GoDaddy Websites & Marketing, Hostinger, HubSpot, Shopify, Squarespace, Webflow, Weebly, Wix, or any other site) into a complete, portable HTML site. Optionally reconstruct it in WordPress."
+    }
+  ]
+}

packages/data-liberation-agent/.claude-plugin/plugin.json

@@ -1,12 +1,12 @@
 {
   "name": "data-liberation",
-  "description": "Extract content from closed web platforms (GoDaddy Websites & Marketing, Hostinger, HubSpot, Shopify, Squarespace, Webflow, Weebly, Wix) into WordPress-compatible WXR files. Inspect, extract, QA, and import to WordPress.",
-  "version": "1.0.0",
+  "description": "Liberate any website (GoDaddy Websites & Marketing, Hostinger, HubSpot, Shopify, Squarespace, Webflow, Weebly, Wix, or any other site) into a complete, portable HTML site, then publish it to a live URL.",
+  "version": "0.2.2",
   "author": {
     "name": "Automattic"
   },
-  "homepage": "https://github.com/Automattic/studio",
-  "repository": "https://github.com/Automattic/studio",
+  "homepage": "https://github.com/Automattic/data-liberation-agent",
+  "repository": "https://github.com/Automattic/data-liberation-agent",
   "license": "GPL-2.0-or-later",
   "keywords": [
     "content-extraction",
@@ -20,8 +20,9 @@
     "webflow",
     "weebly",
     "wix",
-    "wordpress",
-    "wxr"
+    "html",
+    "static-site",
+    "website-migration"
   ],
   "mcpServers": "./.mcp.json"
 }