diff --git a/apps/cli/commands/pull-reprint.ts b/apps/cli/commands/pull-reprint.ts index afed7cbbaa..ae2229984a 100644 --- a/apps/cli/commands/pull-reprint.ts +++ b/apps/cli/commands/pull-reprint.ts @@ -39,6 +39,7 @@ import { type ReprintProcessResult, runReprintCommandUntilComplete, } from 'cli/lib/pull/migration-client'; +import { overallPercent, PullStep, withPercent } from 'cli/lib/pull/pull-progress'; import { getCoreRoots, getReprintMetadata, @@ -56,8 +57,13 @@ import { loadImportedRuntimeStartOptionsNative, } from 'cli/lib/pull/runtime-start-options'; import { buildAutoLoginUrl } from 'cli/lib/site-utils'; +import { ensureSqliteIntegrationForImportedSite } from 'cli/lib/sqlite-integration'; import { fetchSyncableSites } from 'cli/lib/sync-api'; -import { getSyncSupportError, pickSyncSite } from 'cli/lib/sync-site-picker'; +import { + findSyncSiteByIdentifier, + getSyncSupportError, + pickSyncSite, +} from 'cli/lib/sync-site-picker'; import { startWordPressServer, stopWordPressServer, @@ -79,15 +85,15 @@ export const registerCommand = ( yargs: StudioArgv ) => { ), builder: ( builderYargs ) => { return builderYargs - .option( 'url', { + .option( 'remote-site', { type: 'string', - describe: __( 'URL of the remote WordPress site to pull from (remote source)' ), + describe: __( 'Remote site URL or ID' ), } ) .option( 'only', { type: 'string', array: true, describe: __( - 'Restrict the pull to specific wp-content folders (e.g. plugins/akismet, themes, uploads); repeatable.' + 'Restrict the pull to specific wp-content files or folders (e.g. plugins/akismet, themes, uploads/index.php); repeatable.' ), } ) .option( 'skip-database', { @@ -105,14 +111,13 @@ export const registerCommand = ( yargs: StudioArgv ) => { const verbose = argv.verbose; try { - await runCommand( argv.path, argv.url, verbose, { + await runCommand( argv.path, argv.remoteSite, verbose, { only: argv.only as string[] | undefined, skipDatabase: argv[ 'skip-database' ] as boolean, } ); } catch ( error ) { if ( error instanceof PullError ) { - logger.spinner.fail( __( 'Pull failed' ) ); - console.error( '\n' + chalk.bold.red( error.message ) ); + logger.reportError( error ); if ( verbose && error.technicalDetails ) { console.error( '\n' + chalk.dim( error.technicalDetails ) ); } else if ( error.technicalDetails ) { @@ -259,15 +264,21 @@ class PullError extends LoggerError { */ export async function runCommand( localPath: string, - remoteUrl?: string, + remoteSite?: string, verbose = false, cliSelection: CliSelectionOptions = {} ): Promise< void > { - logger.reportStart( LoggerAction.LOAD_SITES, __( 'Loading site…' ) ); + logger.reportStart( + LoggerAction.LOAD_SITES, + withPercent( __( 'Loading site…' ), overallPercent( PullStep.SETUP ) ) + ); const site = await getSiteByFolder( localPath ); logger.reportSuccess( __( 'Site loaded' ) ); - const sourceSite = await resolveSourceSite( remoteUrl ?? site.reprintOrigin?.remoteUrl, verbose ); + const sourceSite = await resolveSourceSite( + remoteSite ?? site.reprintOrigin?.remoteUrl, + verbose + ); if ( ! sourceSite ) { return; } @@ -379,13 +390,23 @@ export async function runCommand( remoteSiteUrl: preflight.siteurl || normalizedRemoteUrl, tablePrefix: preflight.table_prefix || undefined, }; - site.status = 'pulling'; site.reprintOrigin = origin; await updateSiteRecord( site.id, ( record ) => { - record.status = 'pulling'; record.reprintOrigin = origin; } ); + // `pulling` means "the site directory is half-written", so it is set + // only once the pull reaches the steps that rewrite it — not here. + // The downloads run entirely in the fs-root, so a dropped connection + // (by far the likeliest failure) leaves the site intact and startable + // rather than stranding it in `pull-failed`. + const markSiteBeingWritten = async () => { + site.status = 'pulling'; + await updateSiteRecord( site.id, ( record ) => { + record.status = 'pulling'; + } ); + }; + // db-apply (run inside `pull-db`) rewrites the remote site URL to // the local one the Studio server already serves — // `studioMetadata.localUrl` comes from the existing site's port, so @@ -404,7 +425,8 @@ export async function runCommand( verbose, ! isRepull, selection, - reprintMetadata + reprintMetadata, + markSiteBeingWritten ); // The site record already exists (created via `studio create`) and its @@ -412,7 +434,10 @@ export async function runCommand( // wires the generated runtime Blueprint onto it so `studio start` and // the daemon serve the imported runtime rather than the original blank // install. Idempotent — re-writing the same value on a resume is harmless. - logger.reportStart( LoggerAction.CREATE_SITE, `Linking pulled files to "${ site.name }"…` ); + logger.reportStart( + LoggerAction.CREATE_SITE, + withPercent( `Linking pulled files to "${ site.name }"…`, overallPercent( PullStep.LINK ) ) + ); site.runtimeBlueprintPath = studioMetadata.runtimeBlueprintPath; await updateSiteRecord( site.id, ( record ) => { record.runtimeBlueprintPath = studioMetadata.runtimeBlueprintPath; @@ -432,6 +457,13 @@ export async function runCommand( } ); } + // `flat-docroot --force` replaced wp-content — SQLite drop-in included — + // with a symlink into the fs-root. runtime.php wires SQLite up separately + // so the site boots, but phpMyAdmin and `wp sqlite` read the integration + // from wp-content. `studio site start` would reinstall it, except it + // returns early while the server runs — which it does right after a pull. + await ensureSqliteIntegrationForImportedSite( site ); + let runtimeStartOptions: StartServerOptions; if ( getSiteRuntime( site ) === SITE_RUNTIME_NATIVE_PHP ) { const nativeStartOptions = loadImportedRuntimeStartOptionsNative( studioMetadata ); @@ -458,7 +490,10 @@ export async function runCommand( const startOptionsPath = path.join( studioMetadata.runtimeDirectory, 'start-options.json' ); fs.writeFileSync( startOptionsPath, JSON.stringify( runtimeStartOptions, null, 2 ) + '\n' ); - logger.reportStart( LoggerAction.START_SITE, __( 'Starting WordPress server…' ) ); + logger.reportStart( + LoggerAction.START_SITE, + withPercent( __( 'Starting WordPress server…' ), overallPercent( PullStep.START ) ) + ); try { await connectToDaemon(); @@ -522,6 +557,10 @@ export async function runCommand( // again instead of silently reusing this run's choice. clearPullSelection( studioMetadata ); + // The percentage has to ride an in-progress message — `pullSite` only + // parses the token out of those, so a success line can't close the bar. + logger.reportProgress( withPercent( __( 'Pull complete' ), 100 ) ); + site.importComplete = true; site.status = 'ready'; await updateSiteRecord( site.id, ( record ) => { @@ -613,9 +652,15 @@ async function applySelection( params: { ]; }; - // Reuse the selection captured by a prior interrupted run. A folder - // selection can outlive the fs-root that made it a delta (damage wipe): - // re-anchor it with the core roots so the fresh initial sync still + // Reuse the selection captured by a prior interrupted run. The sidecar is + // written when a selection is chosen and removed once the pull succeeds, + // so its presence already means "a prior run started and did not finish" + // — reprint refuses to resume a files-pull whose `--only` set changed, so + // the choice is reused rather than re-derived. It is announced because it + // silently overrides this run's flags (and a UI pull's full-pull intent). + // + // A folder selection can outlive the fs-root that made it a delta (damage + // wipe): re-anchor it with the core roots so the fresh initial sync still // downloads WordPress core. const persisted = readPullSelection( session ); if ( persisted ) { @@ -624,6 +669,9 @@ async function applySelection( params: { persisted.fileOnlyPaths = healed.length > 0 ? healed : undefined; savePullSelection( session, persisted ); } + console.log( + __( 'Resuming the interrupted pull, so its original content selection is reused.' ) + ); return persisted; } @@ -749,7 +797,10 @@ async function runPreflight( return JSON.parse( fs.readFileSync( preflightCachePath, 'utf-8' ) ); } - logger.reportStart( LoggerAction.PREFLIGHT, __( 'Initiating the migration…' ) ); + logger.reportStart( + LoggerAction.PREFLIGHT, + withPercent( __( 'Initiating the migration…' ), overallPercent( PullStep.PREFLIGHT ) ) + ); let preflightResult: ReprintProcessResult; try { @@ -936,7 +987,13 @@ export async function runFullPull( verbose: boolean, isFirstPull: boolean, selection: PullSelection = {}, - reprintMetadata: ReprintMetadata = emptyReprintMetadata + reprintMetadata: ReprintMetadata = emptyReprintMetadata, + /** + * Invoked once, immediately before the first step that writes the site + * directory. Everything up to here lands in the fs-root, so a failure + * leaves the local site untouched — this is where it stops being true. + */ + onBeforeSiteWrite?: () => Promise< void > ): Promise< void > { const contentDir = reprintMetadata.sourceSite.contentDirectory; let importedSqlitePath: string; @@ -967,12 +1024,13 @@ export async function runFullPull( const reprintRuntime = runtime === SITE_RUNTIME_NATIVE_PHP ? 'nginx-fpm' : 'playground-cli'; const onlyArgs = ( selection.fileOnlyPaths ?? [] ).map( ( onlyPath ) => `--only=${ onlyPath }` ); - const runStep = ( progressLabel: string, args: string[] ) => + const runStep = ( step: PullStep, progressLabel: string, args: string[] ) => runReprintCommandUntilComplete( metadata.stateDirectory, metadata.rawDirectory, args, - ( progress ) => logger.reportProgress( progress ), + ( progress, fraction ) => + logger.reportProgress( withPercent( progress, overallPercent( step, fraction ) ) ), { progressLabel, mounts: [ @@ -984,14 +1042,18 @@ export async function runFullPull( } ); - logger.reportStart( LoggerAction.DOWNLOAD_FILES, __( 'Pulling site…' ) ); + logger.reportStart( + LoggerAction.DOWNLOAD_FILES, + withPercent( __( 'Pulling site…' ), overallPercent( PullStep.FILES ) ) + ); // 1. Files. `--only` restricts the download to selected paths. - await runStep( __( 'Pulling files' ), [ + await runStep( PullStep.FILES, __( 'Pulling files' ), [ 'pull-files', apiUrl, `--secret=${ secret }`, ...onlyArgs, + '--mode=mirror', '--no-adaptive', `--state-dir=${ metadata.stateDirectory }`, `--fs-root=${ metadata.rawDirectory }`, @@ -999,7 +1061,7 @@ export async function runFullPull( // 2. Database — only when selected. if ( ! selection.skipDatabase ) { - await runStep( __( 'Pulling database' ), [ + await runStep( PullStep.DATABASE, __( 'Pulling database' ), [ 'pull-db', apiUrl, `--secret=${ secret }`, @@ -1020,12 +1082,17 @@ export async function runFullPull( ensureScopedPullWpConfig( metadata, reprintMetadata ); } + // Everything above landed in the fs-root; from here on the site directory + // itself is rewritten — step 3 *moves* local wp-content entries out of it, + // and the flatten then replaces the directory. + await onBeforeSiteWrite?.(); + // 3. Fold the blank install's wp-content into the pulled one. The plugins, // themes and uploads it alone has move into the fs-root, so the symlink // step 4 puts in their place still reaches them. Reprint refuses to run // this before the file pull has finished, so it has to follow step 1. if ( isFirstPull ) { - await runStep( __( 'Merging local content' ), [ + await runStep( PullStep.MERGE, __( 'Merging local content' ), [ 'merge-wp-content', apiUrl, `--from=${ path.join( metadata.sitePath, 'wp-content' ) }`, @@ -1036,7 +1103,7 @@ export async function runFullPull( // 4. Flatten the raw download into the site directory. Reprint uses the // remote URL to locate the pull state, though this step makes no request. - await runStep( __( 'Flattening layout' ), [ + await runStep( PullStep.FLATTEN, __( 'Flattening layout' ), [ 'flat-docroot', apiUrl, `--flatten-to=${ metadata.sitePath }`, @@ -1049,7 +1116,7 @@ export async function runFullPull( // Reprint can generate runtime configuration when pull-db was skipped. // Reprint uses the remote URL to locate that state; --flat-document-root // replaces --fs-root (they are mutually exclusive). - await runStep( __( 'Preparing runtime' ), [ + await runStep( PullStep.RUNTIME, __( 'Preparing runtime' ), [ 'apply-runtime', apiUrl, `--runtime=${ reprintRuntime }`, @@ -1075,55 +1142,24 @@ export function normalizeSiteUrl( url: string ): string { return normalized.toString(); } -/** - * Finds the WordPress.com site in the user's connected list whose - * public URL best matches `url`. Matches on the full normalized URL - * first, then falls back to host-only matching so `example.com` and - * `example.com/blog` both resolve to the same WP.com site record. - * - * No existing Studio helper does this today — `getSiteByFolder` / - * `getHostnameFromUrl` operate on local sites or return a string, and - * `findSite` variants all key on site id. Keep this one local to - * pull-reprint; if a second caller ever needs the same shape, the - * natural home would be `cli/lib/wpcom-sites`. - */ -export function findMatchingWpComSite< T extends { url: string } >( - sites: T[], - url: string -): T | undefined { - const normalizedUrl = normalizeSiteUrl( url ); - const target = new URL( normalizedUrl ); - - return sites.find( ( site ) => { - try { - const normalizedSiteUrl = normalizeSiteUrl( site.url ); - if ( normalizedSiteUrl === normalizedUrl ) { - return true; - } - - return new URL( normalizedSiteUrl ).host === target.host; - } catch { - return false; - } - } ); -} - /** * Resolves the **remote source** to pull from. A valid source must be * present in the user's WordPress.com Jetpack API site list, which also * includes Pressable sites on the same platform. Handles two input * patterns: * - * 1. URL provided — resolve it against the connected WordPress.com/ - * Pressable sites, enable the exporter, and rotate a fresh secret. - * 2. No URL — among pullable (`syncable`) sites only: if the user has + * 1. Identifier provided — a site URL or WordPress.com site ID, resolved + * against the connected WordPress.com/Pressable sites through the same + * helper `pull` and `push` use, then the exporter is enabled and a fresh + * secret is rotated. + * 2. No identifier — among pullable (`syncable`) sites only: if the user has * exactly one, pick it; with several, show an interactive picker in a * TTY (returning `null` if the user cancels) or error out when run * non-interactively. Non-pullable sites (Simple, or missing hosting * features) are surfaced as disabled in the picker. */ export async function resolveSourceSite( - url?: string, + identifier?: string, verbose = false ): Promise< PullSource | null > { const token = await readAuthToken(); @@ -1144,18 +1180,11 @@ export async function resolveSourceSite( let resolvedUrl: string; let wpComSite: SyncSite; - if ( url ) { - const matched = findMatchingWpComSite( sites, url ); - if ( ! matched ) { - throw new LoggerError( - __( 'This URL is not a WordPress.com or Pressable site connected to your account.' ) - ); - } - if ( matched.syncSupport !== 'syncable' ) { - throw getSyncSupportError( matched ); - } - resolvedUrl = matched.url; - wpComSite = matched; + if ( identifier ) { + // Throws when nothing matches, when several sites share the hostname, + // or when the match isn't syncable. + wpComSite = findSyncSiteByIdentifier( sites, identifier ); + resolvedUrl = wpComSite.url; } else { // Only sites that can run the reprint exporter — those with hosting // features enabled (`syncable`) — are pull candidates. @@ -1178,11 +1207,13 @@ export async function resolveSourceSite( if ( pullableSites.length > 1 ) { // In a real terminal, let the user pick interactively. Outside a // TTY (CI, or Studio driving the command) there's no way to - // prompt, so exit with guidance to pass `--url` — the realistic + // prompt, so exit with guidance to pass `--remote-site` — the realistic // non-TTY caller already does. if ( ! process.stdin.isTTY ) { throw new LoggerError( - __( 'Multiple WordPress.com sites are available. Re-run with `--url `.' ) + __( + 'Multiple WordPress.com sites are available. Re-run with `--remote-site `.' + ) ); } diff --git a/apps/cli/commands/tests/pull-reprint.test.ts b/apps/cli/commands/tests/pull-reprint.test.ts index 00d4e0afb9..65db148512 100644 --- a/apps/cli/commands/tests/pull-reprint.test.ts +++ b/apps/cli/commands/tests/pull-reprint.test.ts @@ -12,7 +12,6 @@ import { pickSyncSite } from 'cli/lib/sync-site-picker'; import { runFullPull, ensureScopedPullWpConfig, - findMatchingWpComSite, getReprintApiUrlForSite, normalizeSiteUrl, resolveSourceSite, @@ -138,15 +137,6 @@ describe( 'CLI: studio pull-reprint helpers', () => { ).toBe( 'https://example.com/?reprint-api-jetpack' ); } ); - it( 'matches WordPress.com sites by normalized URL or host', () => { - expect( - findMatchingWpComSite( - [ { id: 1, name: 'Example', url: 'https://example.wordpress.com/' } ], - 'https://example.wordpress.com' - ) - ).toEqual( { id: 1, name: 'Example', url: 'https://example.wordpress.com/' } ); - } ); - it( 'synthesizes a wp-config when a scoped pull left only an empty symlink target', () => { const technicalSiteDirectory = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-wpconfig-' ) ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); @@ -337,6 +327,7 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'pull-files', 'https://example.com/?reprint-api', '--secret=hmac-secret', + '--mode=mirror', '--no-adaptive', `--state-dir=${ stateDirectory }`, `--fs-root=${ rawDirectory }`, @@ -935,7 +926,7 @@ describe( 'CLI: studio pull-reprint source resolution', () => { setTTY( false ); vi.mocked( fetchSyncableSites ).mockResolvedValue( sites ); - await expect( resolveSourceSite() ).rejects.toThrow( /Re-run with `--url/ ); + await expect( resolveSourceSite() ).rejects.toThrow( /Re-run with `--remote-site/ ); expect( pickSyncSite ).not.toHaveBeenCalled(); expect( rotateReprintSecret ).not.toHaveBeenCalled(); } ); @@ -1000,7 +991,7 @@ describe( 'CLI: studio pull-reprint source resolution', () => { expect( source ).toMatchObject( { url: 'https://one.wordpress.com', wpComSite: sites[ 0 ] } ); } ); - it( 'rotates a secret for a syncable site matched by --url', async () => { + it( 'rotates a secret for a syncable site matched by --remote-site URL', async () => { setTTY( true ); vi.mocked( fetchSyncableSites ).mockResolvedValue( sites ); @@ -1015,7 +1006,21 @@ describe( 'CLI: studio pull-reprint source resolution', () => { expect( pickSyncSite ).not.toHaveBeenCalled(); } ); - it( 'rejects a needs-transfer site passed via --url with the hosting-features message', async () => { + it( 'resolves a site passed to --remote-site as a numeric WordPress.com ID', async () => { + setTTY( true ); + vi.mocked( fetchSyncableSites ).mockResolvedValue( sites ); + + const source = await resolveSourceSite( '22' ); + + expect( source ).toMatchObject( { + url: 'https://two.wordpress.com', + wpComSite: sites[ 1 ], + } ); + expect( rotateReprintSecret ).toHaveBeenCalledWith( 22, token.accessToken, 'v1' ); + expect( pickSyncSite ).not.toHaveBeenCalled(); + } ); + + it( 'rejects a needs-transfer site passed via --remote-site with the hosting-features message', async () => { setTTY( true ); vi.mocked( fetchSyncableSites ).mockResolvedValue( [ syncSite( { @@ -1032,7 +1037,7 @@ describe( 'CLI: studio pull-reprint source resolution', () => { expect( rotateReprintSecret ).not.toHaveBeenCalled(); } ); - it( 'rejects a needs-upgrade site passed via --url with the plan-upgrade message', async () => { + it( 'rejects a needs-upgrade site passed via --remote-site with the plan-upgrade message', async () => { setTTY( true ); vi.mocked( fetchSyncableSites ).mockResolvedValue( [ syncSite( { @@ -1049,7 +1054,7 @@ describe( 'CLI: studio pull-reprint source resolution', () => { expect( rotateReprintSecret ).not.toHaveBeenCalled(); } ); - it( 'reports the specific reason when the only site is not pullable (no --url)', async () => { + it( 'reports the specific reason when the only site is not pullable (no --remote-site)', async () => { setTTY( true ); vi.mocked( fetchSyncableSites ).mockResolvedValue( [ syncSite( { @@ -1067,12 +1072,12 @@ describe( 'CLI: studio pull-reprint source resolution', () => { expect( rotateReprintSecret ).not.toHaveBeenCalled(); } ); - it( 'rejects a URL that is not connected to the WordPress.com account', async () => { + it( 'rejects an identifier that is not connected to the WordPress.com account', async () => { setTTY( true ); vi.mocked( fetchSyncableSites ).mockResolvedValue( sites ); await expect( resolveSourceSite( 'https://third-party.example' ) ).rejects.toThrow( - /not a WordPress\.com or Pressable site connected to your account/ + /No site found matching "https:\/\/third-party\.example"/ ); expect( pickSyncSite ).not.toHaveBeenCalled(); expect( rotateReprintSecret ).not.toHaveBeenCalled(); @@ -1389,7 +1394,6 @@ describe( 'CLI: studio pull-reprint first-pull selective sync', () => { const pullsRoot = path.join( fakeHome, '.studio', 'pulls' ); const technicalSiteDirectory = path.join( pullsRoot, 'fresh-id' ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); - const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); const sitePath = path.join( fakeHome, 'Studio', 'My-Fresh-Site' ); seedCliConfigSite( fakeHome, [ @@ -1500,6 +1504,105 @@ describe( 'CLI: studio pull-reprint first-pull selective sync', () => { ] ); expect( sidecar.skipDatabase ).toBe( true ); } ); + + // The site directory is only rewritten from `preserveUnselectedLocalContent` + // onwards; every download before that lands in the scratch. `pulling` marks + // "half-written", so it must not be set while the site is still intact. + describe( 'the site is only marked as being written once it actually is', () => { + async function runPullFailingAt( failingStep: string ) { + const { runCommand } = await loadRunCommandWithFakeHome(); + mockWpComPullSource(); + + const stateDirectory = path.join( fakeHome, '.studio', 'pulls', 'fresh-id', 'state' ); + const sitePath = path.join( fakeHome, 'Studio', 'My-Fresh-Site' ); + + seedCliConfigSite( fakeHome, [ + makeSiteRecord( { + id: 'fresh-id', + name: 'My Fresh Site', + path: sitePath, + status: 'ready', + } ), + ] ); + + fs.mkdirSync( path.join( sitePath, 'wp-content', 'database' ), { recursive: true } ); + fs.writeFileSync( path.join( sitePath, 'wp-content', 'database', '.ht.sqlite' ), 'local-db' ); + + fs.mkdirSync( stateDirectory, { recursive: true } ); + fs.writeFileSync( + path.join( stateDirectory, '.import-state.json' ), + JSON.stringify( { + preflight: { + data: { + database: { wp: { paths_urls: { content_dir: '/srv/htdocs/wp-content' } } }, + wp_detect: { roots: [ { path: '/wordpress/core/7.0' } ] }, + }, + }, + } ) + ); + + const migrationClientMod = await import( 'cli/lib/pull/migration-client' ); + vi.spyOn( migrationClientMod, 'runReprintCommandUntilComplete' ).mockImplementation( + async ( _stateDir, _rawDir, args ) => { + if ( args[ 0 ] === failingStep ) { + throw new Error( `stop at ${ failingStep }` ); + } + if ( args[ 0 ] === 'preflight' ) { + return { + stdout: JSON.stringify( { + data: { + ok: true, + database: { wp: { siteurl: 'https://example.com', table_prefix: 'wp_' } }, + php: { version: '8.3' }, + }, + } ), + stderr: '', + exitCode: 0, + }; + } + if ( args[ 0 ] === 'import-metadata' ) { + return { + stdout: JSON.stringify( { + ...emptyReprintMetadata, + sourceSite: { + ...emptyReprintMetadata.sourceSite, + contentDirectory: '/srv/htdocs/wp-content', + wordpressRoots: [ '/wordpress/core/7.0', '/wordpress/core' ], + }, + } ), + stderr: '', + exitCode: 0, + }; + } + return { stdout: '{"ok":true}', stderr: '', exitCode: 0 }; + } + ); + vi.spyOn( console, 'log' ).mockImplementation( () => undefined ); + vi.spyOn( console, 'error' ).mockImplementation( () => undefined ); + + await expect( runCommand( sitePath, 'https://example.com', false ) ).rejects.toThrow( + new RegExp( `stop at ${ failingStep }` ) + ); + + return readSeededCliConfig( fakeHome ).sites.find( ( s ) => s.id === 'fresh-id' )!; + } + + it( 'leaves a site untouched by a failed download as `ready`, not `pull-failed`', async () => { + // A dropped connection during pull-files is the likeliest failure, + // and the site is still the intact `studio create` install. + expect( ( await runPullFailingAt( 'pull-files' ) ).status ).toBe( 'ready' ); + } ); + + it( 'still records the origin so the failed run can be resumed without --remote-site', async () => { + expect( ( await runPullFailingAt( 'pull-files' ) ).reprintOrigin ).toMatchObject( { + remoteUrl: 'https://example.com/', + } ); + } ); + + it( 'marks `pull-failed` once the flatten has started rewriting the site', async () => { + expect( ( await runPullFailingAt( 'flat-docroot' ) ).status ).toBe( 'pull-failed' ); + } ); + } ); } ); describe( 'CLI: studio pull-reprint admin credentials re-apply', () => { diff --git a/apps/cli/index.ts b/apps/cli/index.ts index d685f2daeb..bc1c2267e5 100644 --- a/apps/cli/index.ts +++ b/apps/cli/index.ts @@ -214,9 +214,7 @@ async function main() { registerPushCommand( studioArgv ); registerPullCommand( studioArgv ); - if ( process.env.STUDIO_ENABLE_PULL_REPRINT ) { - registerPullReprintCommand( studioArgv ); - } + registerPullReprintCommand( studioArgv ); registerImportCommand( studioArgv ); registerExportCommand( studioArgv ); diff --git a/apps/cli/lib/pull/migration-client.test.ts b/apps/cli/lib/pull/migration-client.test.ts new file mode 100644 index 0000000000..29f53de0b5 --- /dev/null +++ b/apps/cli/lib/pull/migration-client.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { snapshotFraction } from './migration-client'; + +describe( 'snapshotFraction', () => { + it( 'derives the fraction from file counters', () => { + expect( snapshotFraction( { downloadedFiles: 42, totalFiles: 84 } ) ).toBe( 0.5 ); + } ); + + it( 'falls back to bytes when file totals are absent', () => { + expect( snapshotFraction( { downloadedBytes: 25, totalBytes: 100 } ) ).toBe( 0.25 ); + } ); + + it( 'falls back to statements while the database is being applied', () => { + expect( snapshotFraction( { statementsExecuted: 3, statementsTotal: 4 } ) ).toBe( 0.75 ); + } ); + + it( 'prefers files over bytes when reprint reports both', () => { + expect( + snapshotFraction( { + downloadedFiles: 1, + totalFiles: 10, + downloadedBytes: 90, + totalBytes: 100, + } ) + ).toBe( 0.1 ); + } ); + + it( 'returns undefined when nothing is countable, so the bar holds its band start', () => { + expect( snapshotFraction( {} ) ).toBeUndefined(); + expect( snapshotFraction( { phase: 'streaming' } ) ).toBeUndefined(); + expect( snapshotFraction( { downloadedFiles: 5 } ) ).toBeUndefined(); + } ); + + it( 'ignores a zero total rather than dividing by it', () => { + expect( snapshotFraction( { downloadedFiles: 0, totalFiles: 0 } ) ).toBeUndefined(); + } ); +} ); diff --git a/apps/cli/lib/pull/migration-client.ts b/apps/cli/lib/pull/migration-client.ts index 44e9625ae3..84ed290693 100644 --- a/apps/cli/lib/pull/migration-client.ts +++ b/apps/cli/lib/pull/migration-client.ts @@ -49,7 +49,7 @@ export async function runReprintCommandUntilComplete( stateDir: string, fsRoot: string, args: string[], - onProgress?: ( output: string ) => void, + onProgress?: ( output: string, fraction?: number ) => void, options: { mounts?: Array< { hostPath: string; vfsPath: string } >; progressLabel?: string; @@ -362,7 +362,7 @@ interface ProgressReporter { function createProgressReporter( label: string, startTime: number, - onProgress?: ( output: string ) => void + onProgress?: ( output: string, fraction?: number ) => void ): ProgressReporter { let lineBuffer = ''; let snapshot: ProgressSnapshot = {}; @@ -379,7 +379,7 @@ function createProgressReporter( snapshot = updateSnapshot( parsed, snapshot ); const msg = formatSnapshot( snapshot, label, elapsedSeconds() ); if ( msg ) { - onProgress( msg ); + onProgress( msg, snapshotFraction( snapshot ) ); } } }; @@ -391,7 +391,7 @@ function createProgressReporter( setInterval( () => { const msg = formatSnapshot( snapshot, label, elapsedSeconds() ); if ( msg ) { - onProgress( msg ); + onProgress( msg, snapshotFraction( snapshot ) ); } }, 250 ); @@ -512,6 +512,24 @@ function updateSnapshot( return next; } +/** + * How far through the current reprint sub-command we are, as 0–1, or + * undefined when nothing in the snapshot is countable. Files are preferred: + * a step that moves files reports totals reliably, whereas byte totals are + * absent on some phases and statement counts only appear while the database + * is being applied. Callers map this onto an overall percentage. + */ +export function snapshotFraction( snapshot: ProgressSnapshot ): number | undefined { + const ratio = ( done?: number, total?: number ) => + done !== undefined && total !== undefined && total > 0 ? done / total : undefined; + + return ( + ratio( snapshot.downloadedFiles, snapshot.totalFiles ) ?? + ratio( snapshot.downloadedBytes, snapshot.totalBytes ) ?? + ratio( snapshot.statementsExecuted, snapshot.statementsTotal ) + ); +} + /** * Formats the current progress snapshot into a single-line status string * like "Downloading files · 42/1337 files · 12.3 MB · 3m 12s". diff --git a/apps/cli/lib/pull/pull-progress.test.ts b/apps/cli/lib/pull/pull-progress.test.ts new file mode 100644 index 0000000000..1578a5ec35 --- /dev/null +++ b/apps/cli/lib/pull/pull-progress.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; +import { overallPercent, PullStep, withPercent } from './pull-progress'; + +describe( 'overallPercent', () => { + it( 'parks a step with no measurable progress at the start of its band', () => { + expect( overallPercent( PullStep.FILES ) ).toBe( 5 ); + expect( overallPercent( PullStep.DATABASE ) ).toBe( 60 ); + } ); + + it( 'interpolates a step fraction within that band', () => { + // Files span 5–60, so halfway through the files step is 33 overall. + expect( overallPercent( PullStep.FILES, 0.5 ) ).toBe( 33 ); + expect( overallPercent( PullStep.FILES, 1 ) ).toBe( 60 ); + } ); + + it( 'reaches exactly 100 only at the end of the final step', () => { + expect( overallPercent( PullStep.START, 1 ) ).toBe( 100 ); + expect( overallPercent( PullStep.RUNTIME, 1 ) ).toBeLessThan( 100 ); + } ); + + it( 'clamps a fraction that overshoots or goes negative so the bar cannot leave its band', () => { + // Reprint restarts partial transfers, so counters can regress or exceed + // their total mid-stream; neither may move the bar out of the band. + expect( overallPercent( PullStep.FILES, 5 ) ).toBe( 60 ); + expect( overallPercent( PullStep.FILES, -2 ) ).toBe( 5 ); + } ); + + it( 'never moves backwards across steps run in order', () => { + const order = [ + PullStep.SETUP, + PullStep.PREFLIGHT, + PullStep.FILES, + PullStep.DATABASE, + PullStep.MERGE, + PullStep.FLATTEN, + PullStep.RUNTIME, + PullStep.LINK, + PullStep.START, + ]; + const starts = order.map( ( step ) => overallPercent( step ) ); + expect( starts ).toEqual( [ ...starts ].sort( ( a, b ) => a - b ) ); + + // A step's completion never exceeds the next step's start, so skipping a + // step (e.g. --skip-database) jumps forward rather than stalling. + order.slice( 0, -1 ).forEach( ( step, index ) => { + expect( overallPercent( step, 1 ) ).toBeLessThanOrEqual( starts[ index + 1 ] ); + } ); + } ); +} ); + +describe( 'withPercent', () => { + it( 'appends the token pullSite parses out of the message', () => { + expect( withPercent( 'Pulling files · 42/1337 files', 30 ) ).toBe( + 'Pulling files · 42/1337 files (30%)' + ); + } ); +} ); diff --git a/apps/cli/lib/pull/pull-progress.ts b/apps/cli/lib/pull/pull-progress.ts new file mode 100644 index 0000000000..f5c7125a00 --- /dev/null +++ b/apps/cli/lib/pull/pull-progress.ts @@ -0,0 +1,57 @@ +/** + * Overall progress for `pull-reprint`. + * + * Reprint reports progress per sub-command ("42/1337 files"), with no notion + * of how far through the whole pull that is. Studio drives the command as a + * child process and scrapes a `(N%)` token out of the message to move its + * progress bar (see `pullSite` in `packages/common/sites/sync.ts`), so each + * step is mapped onto a slice of 0–100 and its own fraction interpolated + * within that slice. + * + * The weights are rough — files dominate wall-clock on a typical site, the + * database is a distant second — and they only need to be monotonic and + * non-stalling, not accurate. + */ + +export enum PullStep { + SETUP = 'setup', + PREFLIGHT = 'preflight', + FILES = 'files', + DATABASE = 'database', + MERGE = 'merge', + FLATTEN = 'flatten', + RUNTIME = 'runtime', + LINK = 'link', + START = 'start', +} + +const BANDS: Record< PullStep, readonly [ number, number ] > = { + [ PullStep.SETUP ]: [ 0, 3 ], + [ PullStep.PREFLIGHT ]: [ 3, 5 ], + [ PullStep.FILES ]: [ 5, 60 ], + [ PullStep.DATABASE ]: [ 60, 80 ], + [ PullStep.MERGE ]: [ 80, 84 ], + [ PullStep.FLATTEN ]: [ 84, 90 ], + [ PullStep.RUNTIME ]: [ 90, 94 ], + [ PullStep.LINK ]: [ 94, 96 ], + [ PullStep.START ]: [ 96, 100 ], +}; +/** + * `fraction` is clamped, so a reprint counter that overshoots or resets + * mid-transfer can't push the bar past its band or backwards out of it. + * Omitting it parks the bar at the start of the band, which is what the + * steps that report no measurable progress do. + */ +export function overallPercent( step: PullStep, fraction?: number ): number { + const [ start, end ] = BANDS[ step ]; + const clamped = Math.min( 1, Math.max( 0, fraction ?? 0 ) ); + return Math.round( start + ( end - start ) * clamped ); +} + +/** + * Appends the `(N%)` token `pullSite` looks for. Matches the format the + * `pull` command already emits, so both engines parse identically. + */ +export function withPercent( message: string, percent: number ): string { + return `${ message } (${ percent }%)`; +} diff --git a/apps/cli/lib/pull/reprint-selector.test.ts b/apps/cli/lib/pull/reprint-selector.test.ts index 642eb41211..4ea511f083 100644 --- a/apps/cli/lib/pull/reprint-selector.test.ts +++ b/apps/cli/lib/pull/reprint-selector.test.ts @@ -1,16 +1,31 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; +import { fetchPullTree } from 'cli/lib/sync-selector'; import { - filterTreeToDirectories, + fetchJetpackPullTree, mapCheckedNodesToSelection, mapCliOnlyToReprint, } from './reprint-selector'; import type { TreeNode } from 'cli/lib/tree-checkbox'; +vi.mock( 'cli/lib/sync-selector', () => ( { + fetchPullTree: vi.fn(), + buildTreeFromRemote: vi.fn(), +} ) ); + /** Minimal checked node — mapCheckedNodesToSelection only reads `value`. */ function checked( value: string, depth = 1 ): TreeNode { return { name: value, value, isDirectory: false, checked: true, expanded: false, depth }; } +describe( 'fetchJetpackPullTree', () => { + it( 'keeps file leaves from the remote tree', async () => { + const tree = [ checked( 'database', 0 ), checked( 'uploads/banner.jpg', 2 ) ]; + vi.mocked( fetchPullTree ).mockResolvedValue( { tree, rewindId: 'rewind-1' } ); + + expect( await fetchJetpackPullTree( 'token', 123 ) ).toEqual( tree ); + } ); +} ); + describe( 'mapCheckedNodesToSelection', () => { it( 'maps a full selection to no --only and keeps the database', () => { const selected = [ checked( 'database', 0 ), checked( 'wp-content', 0 ), checked( 'plugins' ) ]; @@ -34,6 +49,14 @@ describe( 'mapCheckedNodesToSelection', () => { ] ); } ); + it( 'maps a selected file to its wp-content path', () => { + expect( mapCheckedNodesToSelection( [ checked( 'uploads/2026/banner.jpg', 3 ) ] ) ).toEqual( { + fileOnlyPaths: [ ':wp-content:/uploads/2026/banner.jpg' ], + skipDatabase: true, + hasAnyFile: true, + } ); + } ); + it( 'collapses a fully-checked directory and keeps a deep partial selection as a path', () => { expect( mapCheckedNodesToSelection( [ checked( 'plugins' ), checked( 'plugins/akismet', 2 ) ] ) @@ -50,51 +73,6 @@ describe( 'mapCheckedNodesToSelection', () => { } ); } ); -describe( 'filterTreeToDirectories', () => { - it( 'keeps the database toggle and canonical directory hierarchy while dropping files', () => { - const tree: TreeNode[] = [ - checked( 'database', 0 ), - { - name: 'wp-content/', - value: 'wp-content', - isDirectory: true, - checked: true, - expanded: true, - depth: 0, - children: [ - checked( 'plugins/f26d-error.php', 2 ), - { - name: 'plugins/', - value: 'plugins/', - isDirectory: true, - checked: true, - expanded: false, - depth: 1, - children: [ - { - name: 'akismet/', - value: 'plugins/akismet/', - isDirectory: true, - checked: true, - expanded: false, - depth: 2, - }, - ], - }, - ], - }, - ]; - - const filtered = filterTreeToDirectories( tree ); - - expect( filtered.map( ( node ) => node.value ) ).toEqual( [ 'database', 'wp-content' ] ); - expect( filtered[ 1 ].children?.map( ( node ) => node.value ) ).toEqual( [ 'plugins' ] ); - expect( filtered[ 1 ].children?.[ 0 ].children?.map( ( node ) => node.value ) ).toEqual( [ - 'plugins/akismet', - ] ); - } ); -} ); - describe( 'mapCliOnlyToReprint', () => { it( 'maps wp-content-relative paths to the wp-content token', () => { expect( mapCliOnlyToReprint( [ 'plugins', 'plugins/akismet', 'themes', 'uploads' ] ) ).toEqual( diff --git a/apps/cli/lib/pull/reprint-selector.ts b/apps/cli/lib/pull/reprint-selector.ts index 0723cbebed..c18b60be22 100644 --- a/apps/cli/lib/pull/reprint-selector.ts +++ b/apps/cli/lib/pull/reprint-selector.ts @@ -9,7 +9,6 @@ import { __ } from '@wordpress/i18n'; import { fetchLatestRewindId, fetchRemoteFileTree } from 'cli/lib/sync-api'; import { buildTreeFromRemote, fetchPullTree } from 'cli/lib/sync-selector'; import treeCheckbox from 'cli/lib/tree-checkbox'; -import type { RemoteFileEntry } from '@studio/common/lib/sync/sync-api'; import type { TreeNode } from 'cli/lib/tree-checkbox'; const WP_CONTENT_TOKEN = ':wp-content:'; @@ -69,40 +68,12 @@ export function mapCheckedNodesToSelection( selected: TreeNode[] ): PullSelectio }; } -/** - * Reprint's `--only` values are directory roots. Keep the database toggle - * and directory nodes, but never expose files that Reprint cannot pull - * independently. Jetpack represents directory paths with trailing slashes; - * use canonical values before the picker or selection logic sees them. - */ -export function filterTreeToDirectories( tree: TreeNode[] ): TreeNode[] { - return tree.flatMap( ( node ) => { - if ( node.value === 'database' ) { - return [ node ]; - } - if ( ! node.isDirectory ) { - return []; - } - return [ - { - ...node, - value: node.value.replace( /\/+$/, '' ), - children: node.children ? filterTreeToDirectories( node.children ) : undefined, - }, - ]; - } ); -} - -function filterEntriesToDirectories( entries: RemoteFileEntry[] ): RemoteFileEntry[] { - return entries.filter( ( entry ) => entry.isDirectory ); -} - export async function fetchJetpackPullTree( token: string, remoteSiteId: number ): Promise< TreeNode[] > { const { tree } = await fetchPullTree( token, remoteSiteId ); - return filterTreeToDirectories( tree ); + return tree; } /** @@ -127,9 +98,7 @@ export async function selectPullItems( rewindId, `/wp-content/${ node.value }` ); - return filterTreeToDirectories( - buildTreeFromRemote( filterEntriesToDirectories( entries ), node.depth + 1 ) - ); + return buildTreeFromRemote( entries, node.depth + 1 ); } : undefined, } ); @@ -142,7 +111,7 @@ export async function selectPullItems( if ( ! selection.hasAnyFile && ! options.allowDatabaseOnly ) { console.log( __( - 'Refreshing the database on its own is not supported yet. Select at least one folder to refresh.' + 'Refreshing the database on its own is not supported yet. Select at least one path to refresh.' ) ); return undefined; diff --git a/apps/cli/lib/tests/sqlite-integration.test.ts b/apps/cli/lib/tests/sqlite-integration.test.ts new file mode 100644 index 0000000000..5f526c21da --- /dev/null +++ b/apps/cli/lib/tests/sqlite-integration.test.ts @@ -0,0 +1,82 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getWpFilesPath } from 'cli/lib/dependency-management/paths'; +import { ensureSqliteIntegrationForImportedSite } from 'cli/lib/sqlite-integration'; + +vi.mock( 'cli/lib/dependency-management/paths', () => ( { + getWpFilesPath: vi.fn(), +} ) ); + +const SQLITE_DIRNAME = 'sqlite-database-integration'; + +describe( 'ensureSqliteIntegrationForImportedSite', () => { + let tmpDir: string; + let sitePath: string; + + beforeEach( () => { + tmpDir = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-sqlite-imported-' ) ); + + const wpFilesPath = path.join( tmpDir, 'wp-files' ); + const sourcePath = path.join( wpFilesPath, SQLITE_DIRNAME ); + fs.mkdirSync( path.join( sourcePath, 'wp-includes', 'database' ), { recursive: true } ); + fs.writeFileSync( + path.join( sourcePath, 'db.copy' ), + " { + vi.restoreAllMocks(); + fs.rmSync( tmpDir, { recursive: true, force: true } ); + } ); + + // A full reprint pull replaces wp-content with a symlink into the pull + // scratch, dropping the drop-in and mu-plugin the blank install had. The + // site still boots through runtime.php, but phpMyAdmin resolves its SQLite + // driver from wp-content and needs them back. + it( 'installs the integration for an imported site that lost it', async () => { + await ensureSqliteIntegrationForImportedSite( { + path: sitePath, + runtimeBlueprintPath: path.join( tmpDir, 'runtime', 'blueprint.json' ), + } ); + + expect( + fs.existsSync( path.join( sitePath, 'wp-content', 'mu-plugins', SQLITE_DIRNAME ) ) + ).toBe( true ); + expect( fs.existsSync( path.join( sitePath, 'wp-content', 'db.php' ) ) ).toBe( true ); + } ); + + // The flattened layout a pull leaves behind: the install has to land in the + // scratch the symlink points at, which is what runtime.php mounts. + it( 'installs through the flattened wp-content symlink', async () => { + const scratchContent = path.join( tmpDir, 'raw', 'srv', 'htdocs', 'wp-content' ); + fs.mkdirSync( scratchContent, { recursive: true } ); + fs.rmSync( path.join( sitePath, 'wp-content' ), { recursive: true } ); + fs.symlinkSync( scratchContent, path.join( sitePath, 'wp-content' ) ); + + await ensureSqliteIntegrationForImportedSite( { + path: sitePath, + runtimeBlueprintPath: path.join( tmpDir, 'runtime', 'blueprint.json' ), + } ); + + expect( fs.existsSync( path.join( scratchContent, 'mu-plugins', SQLITE_DIRNAME ) ) ).toBe( + true + ); + expect( fs.existsSync( path.join( scratchContent, 'db.php' ) ) ).toBe( true ); + } ); + + it( 'leaves a non-imported site alone', async () => { + await ensureSqliteIntegrationForImportedSite( { path: sitePath } ); + + expect( + fs.existsSync( path.join( sitePath, 'wp-content', 'mu-plugins', SQLITE_DIRNAME ) ) + ).toBe( false ); + } ); +} ); diff --git a/apps/local/src/index.ts b/apps/local/src/index.ts index 20efce6e7b..cbc48ce5e0 100644 --- a/apps/local/src/index.ts +++ b/apps/local/src/index.ts @@ -69,6 +69,8 @@ import { } from '@studio/common/lib/shared-config'; import { fetchStudioAssistantQuota } from '@studio/common/lib/studio-assistant-quota'; import { isSyncCancelledError } from '@studio/common/lib/sync/cancel'; +import { resolvePullEngine } from '@studio/common/lib/sync/pull-engine'; +import { isSiteOverPushSizeLimit } from '@studio/common/lib/sync/pull-size-warning'; import { fetchLatestRewindId, fetchSyncableSites } from '@studio/common/lib/sync/sync-api'; import { detectInstalledApps } from '@studio/common/lib/user-settings/installed-apps'; import { isWordPressDevVersion } from '@studio/common/lib/wordpress-version-utils'; @@ -713,6 +715,19 @@ export async function startLocalServer( options: LocalServerOptions ): Promise< } ) ); + api.get( + '/sites/:id/push-size-over-limit', + asyncHandler( async ( req: Request, res: Response ) => { + const sites = await listSites( execute ); + const site = sites.find( ( candidate ) => candidate.id === req.params.id ); + if ( ! site ) { + res.status( 404 ).json( { error: `Site ${ req.params.id } not found` } ); + return; + } + res.json( { overLimit: await isSiteOverPushSizeLimit( site.path ) } ); + } ) + ); + // --- Site creation helpers + create --------------------------------------- // Pure server-side filesystem logic (the server runs on the user's machine), // plus the CLI `create`. The browser has no native folder picker, so the UI @@ -1332,19 +1347,17 @@ export async function startLocalServer( options: LocalServerOptions ): Promise< } const release = registerSyncAbort( req.params.id, remoteSiteId ); try { - await pullSite( - execute, - site.path, - remoteSiteId, - ( progress ) => { + await pullSite( execute, site.path, remoteSiteId, { + engine: await resolvePullEngine(), + emit: ( progress ) => { sseSend( { channel: 'sync-pull', payload: { ...progress, siteId: req.params.id, remoteSiteId }, } ); }, - options, - release.signal - ); + syncOptions: options, + signal: release.signal, + } ); } catch ( error ) { // A user cancel is an intentional stop, not a server error — report it // as a result so it doesn't surface as a 500. diff --git a/apps/studio/src/components/tests/content-tab-settings.test.tsx b/apps/studio/src/components/tests/content-tab-settings.test.tsx index 4b054a074f..b5d59a85d7 100644 --- a/apps/studio/src/components/tests/content-tab-settings.test.tsx +++ b/apps/studio/src/components/tests/content-tab-settings.test.tsx @@ -38,7 +38,7 @@ const snapshotTestActions = { let testStore = createTestStore( { preloadedState: { betaFeatures: { - features: { remoteSession: false, enableAgenticUi: false }, + features: { remoteSession: false, enableAgenticUi: false, reprintPull: false }, loading: false, }, }, @@ -49,7 +49,7 @@ function createCustomTestStore() { const store = createTestStore( { preloadedState: { betaFeatures: { - features: { remoteSession: false, enableAgenticUi: false }, + features: { remoteSession: false, enableAgenticUi: false, reprintPull: false }, loading: false, }, }, diff --git a/apps/studio/src/components/tests/remote-session-indicator.test.tsx b/apps/studio/src/components/tests/remote-session-indicator.test.tsx index 1b475e7270..47fb355972 100644 --- a/apps/studio/src/components/tests/remote-session-indicator.test.tsx +++ b/apps/studio/src/components/tests/remote-session-indicator.test.tsx @@ -32,7 +32,11 @@ function setupHooks( { isRunning: boolean; isLoading?: boolean; } ) { - vi.mocked( useBetaFeatures ).mockReturnValue( { remoteSession, enableAgenticUi: false } ); + vi.mocked( useBetaFeatures ).mockReturnValue( { + remoteSession, + enableAgenticUi: false, + reprintPull: false, + } ); vi.mocked( useAuth, { partial: true } ).mockReturnValue( { isAuthenticated } ); vi.mocked( useRemoteSessionStatus ).mockReturnValue( { status: isRunning ? { running: true } : undefined, diff --git a/apps/studio/src/ipc-handlers.ts b/apps/studio/src/ipc-handlers.ts index 6e2d2d68df..c64440811e 100644 --- a/apps/studio/src/ipc-handlers.ts +++ b/apps/studio/src/ipc-handlers.ts @@ -96,6 +96,7 @@ import { getSiteFileAccess } from '@studio/common/lib/site-file-access'; import { getSiteRuntime, siteModeFromRuntime } from '@studio/common/lib/site-runtime'; import { SYNC_IGNORE_DEFAULTS } from '@studio/common/lib/sync/constants'; import { shouldExcludeFromSync } from '@studio/common/lib/sync/exclude-from-sync'; +import { isSiteOverPushSizeLimit as isSiteOverPushSizeLimitForPath } from '@studio/common/lib/sync/pull-size-warning'; import { shouldLimitDepth } from '@studio/common/lib/sync/tree-utils'; import { getSessionsDirectory } from '@studio/common/lib/well-known-paths'; import { isWordPressDevVersion } from '@studio/common/lib/wordpress-version-utils'; @@ -2043,6 +2044,14 @@ export function getDirectorySize( _event: IpcMainInvokeEvent, siteId: string, su return calculateDirectorySizeForArchive( nodePath.join( site.details.path, ...subdir ) ); } +export async function isSiteOverPushSizeLimit( _event: IpcMainInvokeEvent, siteId: string ) { + const site = SiteServer.get( siteId ); + if ( ! site ) { + throw new Error( 'Site not found.' ); + } + return isSiteOverPushSizeLimitForPath( site.details.path ); +} + export function getFileSize( _event: IpcMainInvokeEvent, siteId: string, filePath: string[] ) { const site = SiteServer.get( siteId ); if ( ! site ) { diff --git a/apps/studio/src/ipc-types.d.ts b/apps/studio/src/ipc-types.d.ts index fe81a8cebc..32b297d72d 100644 --- a/apps/studio/src/ipc-types.d.ts +++ b/apps/studio/src/ipc-types.d.ts @@ -112,6 +112,7 @@ interface FeatureFlags {} interface BetaFeatures { remoteSession: boolean; enableAgenticUi: boolean; + reprintPull: boolean; } interface AppGlobals extends FeatureFlags { diff --git a/apps/studio/src/lib/beta-features.ts b/apps/studio/src/lib/beta-features.ts index d7573f064e..7b90c2a83f 100644 --- a/apps/studio/src/lib/beta-features.ts +++ b/apps/studio/src/lib/beta-features.ts @@ -15,6 +15,7 @@ export interface BetaFeatureDefinition { const BETA_FEATURE_DEFAULTS: Record< keyof BetaFeatures, boolean > = { remoteSession: false, enableAgenticUi: false, + reprintPull: false, }; /** @@ -35,6 +36,14 @@ export function getBetaFeaturesDefinition(): Record< keyof BetaFeatures, BetaFea default: BETA_FEATURE_DEFAULTS.enableAgenticUi, description: __( 'A redesigned interface with AI-powered site building.' ), }, + reprintPull: { + label: __( 'Reprint pull engine' ), + key: 'reprintPull', + default: BETA_FEATURE_DEFAULTS.reprintPull, + description: __( + 'Makes pull operations incremental and faster in the new Studio experience.' + ), + }, }; } diff --git a/apps/studio/src/modules/site-settings/tests/edit-site-details.test.tsx b/apps/studio/src/modules/site-settings/tests/edit-site-details.test.tsx index 536e667a61..da60b234e3 100644 --- a/apps/studio/src/modules/site-settings/tests/edit-site-details.test.tsx +++ b/apps/studio/src/modules/site-settings/tests/edit-site-details.test.tsx @@ -73,7 +73,7 @@ const renderWithProvider = ( children: React.ReactElement ) => { const store = createTestStore( { preloadedState: { betaFeatures: { - features: { remoteSession: false, enableAgenticUi: false }, + features: { remoteSession: false, enableAgenticUi: false, reprintPull: false }, loading: false, }, }, diff --git a/apps/studio/src/modules/sync/lib/ipc-handlers.ts b/apps/studio/src/modules/sync/lib/ipc-handlers.ts index 679e4d495d..bcc1ea740a 100644 --- a/apps/studio/src/modules/sync/lib/ipc-handlers.ts +++ b/apps/studio/src/modules/sync/lib/ipc-handlers.ts @@ -28,6 +28,7 @@ import { } from 'src/hooks/use-sync-states-progress-info'; import { sendIpcEventToRenderer, sendIpcEventToRendererWithWindow } from 'src/ipc-utils'; import { ACTIVE_SYNC_OPERATIONS } from 'src/lib/active-sync-operations'; +import { getBetaFeatures } from 'src/lib/beta-features'; import { download } from 'src/lib/download'; import { getSyncBackupTempPath } from 'src/lib/get-sync-backup-temp-path'; import { getAuthenticationToken } from 'src/lib/oauth'; @@ -525,12 +526,15 @@ export async function updateConnectedWpcomSites( } } -// Wraps the CLI `pull` command for apps/ui. The desktop renderer handles +// Wraps the CLI pull commands for apps/ui. The desktop renderer handles // pull via `pullSiteThunk` + `pollPullBackupThunk` using its own WPCOM // client to initiate + poll + download — that polling lives in the // renderer sync slice with no end-to-end IPC equivalent to reuse. Calling // the CLI instead keeps apps/ui free of wpcom-client setup and mirrors the -// simpler flow used by `push`. Exchanges everything (`--options all`). +// simpler flow used by `push`. Exchanges everything. +// +// The `reprintPull` beta feature is read per pull, so toggling the menu item +// takes effect on the next pull without reloading the renderer. export async function pullSiteFromLive( event: IpcMainInvokeEvent, siteId: string, @@ -541,6 +545,7 @@ export async function pullSiteFromLive( if ( ! site ) { throw new Error( 'Site not found.' ); } + const { reprintPull } = await getBetaFeatures(); const window = BrowserWindow.fromWebContents( event.sender ); // Registered under the same key the legacy renderer uses, so `cancelSyncOperation` // stops an agentic pull too. @@ -548,19 +553,17 @@ export async function pullSiteFromLive( const abortController = new AbortController(); SYNC_ABORT_CONTROLLERS.set( operationId, abortController ); try { - await pullSite( - executeCliCommand, - site.details.path, - remoteSiteId, - ( progress ) => { + await pullSite( executeCliCommand, site.details.path, remoteSiteId, { + emit: ( progress ) => { sendIpcEventToRendererWithWindow( window, 'sync-pull-progress', { siteId, ...progress, } ); }, - options, - abortController.signal - ); + engine: reprintPull ? 'reprint' : 'jetpack', + syncOptions: options, + signal: abortController.signal, + } ); return { cancelled: false }; } catch ( error ) { // A user cancel is an intentional stop, not a failure. Rejecting here would diff --git a/apps/studio/src/preload.ts b/apps/studio/src/preload.ts index 7b34c488e5..41e51a1fe7 100644 --- a/apps/studio/src/preload.ts +++ b/apps/studio/src/preload.ts @@ -166,6 +166,7 @@ const api: IpcApi = { ipcRendererInvoke( 'resumeSyncUpload', selectedSiteId, remoteSiteId ), getDirectorySize: ( id, subdir ) => ipcRendererInvoke( 'getDirectorySize', id, subdir ), getFileSize: ( id, filePath ) => ipcRendererInvoke( 'getFileSize', id, filePath ), + isSiteOverPushSizeLimit: ( id ) => ipcRendererInvoke( 'isSiteOverPushSizeLimit', id ), getPathForFile: ( file ) => webUtils.getPathForFile( file ), readLocalMediaFile: ( path ) => ipcRendererInvoke( 'readLocalMediaFile', path ), setWebviewViewport: ( webContentsId, viewport ) => diff --git a/apps/studio/src/stores/beta-features-slice.ts b/apps/studio/src/stores/beta-features-slice.ts index 4797893e20..9d7ab74f13 100644 --- a/apps/studio/src/stores/beta-features-slice.ts +++ b/apps/studio/src/stores/beta-features-slice.ts @@ -8,7 +8,7 @@ type BetaFeaturesState = { }; const initialState: BetaFeaturesState = { - features: { remoteSession: false, enableAgenticUi: false }, + features: { remoteSession: false, enableAgenticUi: false, reprintPull: false }, loading: false, }; diff --git a/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.test.ts b/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.test.ts new file mode 100644 index 0000000000..d793f0f7ba --- /dev/null +++ b/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from 'vitest'; +import { convertTreeToReprintPullOptions } from '@/components/selective-sync/lib/convert-tree-to-sync-options'; +import type { TreeNode } from '@/components/selective-sync/tree-view'; + +/** + * The cases below mirror `mapCheckedNodesToSelection`'s in + * `apps/cli/lib/pull/reprint-selector.test.ts`. The two reductions feed the + * same `--only` flag from different front ends, so they must not drift. + */ + +/** A wp-content entry. Remote tree paths are always slash-terminated. */ +function entry( relativePath: string, checked: boolean, children?: TreeNode[] ): TreeNode { + return { + id: relativePath, + name: relativePath.split( '/' ).pop() ?? relativePath, + label: relativePath, + checked, + indeterminate: ! checked && Boolean( children?.some( ( child ) => child.checked ) ), + path: `/wp-content/${ relativePath }/`, + pathId: `backup-id-${ relativePath }`, + children, + }; +} + +function tree( { + database, + wpContentChildren = [], + allFiles = false, +}: { + database: boolean; + wpContentChildren?: TreeNode[]; + allFiles?: boolean; +} ): TreeNode[] { + return [ + { id: 'sqls', name: 'sqls', label: 'Database', checked: database }, + { + id: 'filesAndFolders', + name: 'filesAndFolders', + label: 'Files', + checked: allFiles, + children: [ + { + id: 'wp-content', + name: 'wp-content', + label: 'wp-content', + checked: allFiles, + children: wpContentChildren, + }, + ], + }, + ]; +} + +describe( 'convertTreeToReprintPullOptions', () => { + it( 'maps a full selection to no --only and keeps the database', () => { + expect( convertTreeToReprintPullOptions( tree( { database: true, allFiles: true } ) ) ).toEqual( + { onlyPaths: [], skipDatabase: false } + ); + } ); + + it( 'skips the database when it is unchecked', () => { + expect( + convertTreeToReprintPullOptions( tree( { database: false, allFiles: true } ) ).skipDatabase + ).toBe( true ); + } ); + + it( 'maps selected directories to wp-content-relative paths', () => { + expect( + convertTreeToReprintPullOptions( + tree( { + database: true, + wpContentChildren: [ entry( 'plugins', true ), entry( 'themes', true ) ], + } ) + ) + ).toEqual( { onlyPaths: [ 'plugins', 'themes' ], skipDatabase: false } ); + } ); + + it( 'leaves out an unchecked sibling', () => { + expect( + convertTreeToReprintPullOptions( + tree( { + database: false, + wpContentChildren: [ entry( 'plugins', true ), entry( 'themes', false ) ], + } ) + ) + ).toEqual( { onlyPaths: [ 'plugins' ], skipDatabase: true } ); + } ); + + it( 'maps a selected file to its wp-content path', () => { + expect( + convertTreeToReprintPullOptions( + tree( { + database: false, + wpContentChildren: [ + entry( 'uploads', false, [ entry( 'uploads/2026/banner.jpg', true ) ] ), + ], + } ) + ) + ).toEqual( { onlyPaths: [ 'uploads/2026/banner.jpg' ], skipDatabase: true } ); + } ); + + it( 'collapses a fully-checked directory rather than listing its descendants', () => { + expect( + convertTreeToReprintPullOptions( + tree( { + database: false, + wpContentChildren: [ entry( 'plugins', true, [ entry( 'plugins/akismet', true ) ] ) ], + } ) + ).onlyPaths + ).toEqual( [ 'plugins' ] ); + } ); + + it( 'keeps a deep partial selection as its own path', () => { + expect( + convertTreeToReprintPullOptions( + tree( { + database: false, + wpContentChildren: [ entry( 'plugins', false, [ entry( 'plugins/akismet', true ) ] ) ], + } ) + ).onlyPaths + ).toEqual( [ 'plugins/akismet' ] ); + } ); + + it( 'returns no paths when only the database is checked', () => { + expect( convertTreeToReprintPullOptions( tree( { database: true } ) ) ).toEqual( { + onlyPaths: [], + skipDatabase: false, + } ); + } ); + + it( 'treats a checked wp-content root as everything, whatever its children say', () => { + expect( + convertTreeToReprintPullOptions( [ + { id: 'sqls', name: 'sqls', label: 'Database', checked: false }, + { + id: 'filesAndFolders', + name: 'filesAndFolders', + label: 'Files', + checked: false, + children: [ + { + id: 'wp-content', + name: 'wp-content', + label: 'wp-content', + checked: true, + children: [ entry( 'plugins', true ) ], + }, + ], + }, + ] ) + ).toEqual( { onlyPaths: [], skipDatabase: true } ); + } ); + + it( 'throws when the tree is missing the database or files branch', () => { + expect( () => convertTreeToReprintPullOptions( [] ) ).toThrow( + /Database or files and folders/ + ); + } ); +} ); diff --git a/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.ts b/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.ts index 968ec81ea9..6a2a6903ea 100644 --- a/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.ts +++ b/apps/ui/src/components/selective-sync/lib/convert-tree-to-sync-options.ts @@ -9,6 +9,22 @@ type PushOptionsWithSelections = { specificSelectionPaths?: string[]; }; +export type ReprintPullOptions = { + onlyPaths: string[]; + skipDatabase: boolean; +}; + +/** + * `/wp-content/plugins/akismet/` → `plugins/akismet`. Remote tree paths are + * always slash-terminated, files included. + */ +const toWpContentRelativePath = ( nodePath: string | undefined ): string | undefined => { + if ( ! nodePath ) { + return undefined; + } + return nodePath.replace( /^\/?wp-content\//, '' ).replace( /\/+$/, '' ) || undefined; +}; + const collectCheckedNodes = ( nodes: TreeNode[] | undefined ): TreeNode[] => { if ( ! nodes?.length ) { return []; @@ -126,3 +142,41 @@ export const convertTreeToPullOptions = ( tree: TreeNode[] ): PullSiteOptions => return pullOptions; }; + +/** + * The pull selection for the Reprint engine, which selects by + * wp-content-relative path (`--only`) rather than by Jetpack backup node id. + * + * This mirrors `mapCheckedNodesToSelection` in + * `apps/cli/lib/pull/reprint-selector.ts`, which reduces the CLI picker's + * checked nodes the same way — a fully-checked directory stands for its + * descendants, and a fully-checked `wp-content` needs no `--only` at all. + * The two reductions have to agree, so they are tested against the same cases. + * + * Paths stay wp-content-relative; the CLI's `mapCliOnlyToReprint` turns them + * into the `:wp-content:` sources Reprint expects. + */ +export const convertTreeToReprintPullOptions = ( tree: TreeNode[] ): ReprintPullOptions => { + const { isDatabaseSelected, filesAndFolders, wpContent } = getCommonNodes( tree ); + + if ( ! filesAndFolders || ! isDatabaseSelected ) { + throw new Error( + 'Error when converting tree to pull options. Database or files and folders not found' + ); + } + + const skipDatabase = ! isDatabaseSelected.checked; + + // Every file is selected, so the pull needs no `--only` to restrict it. + if ( filesAndFolders.checked || wpContent?.checked ) { + return { onlyPaths: [], skipDatabase }; + } + + // `collectCheckedNodes` descends only into partially-checked folders, so a + // checked folder arrives on its own and its descendants are left out. + const onlyPaths = collectCheckedNodes( wpContent?.children ?? [] ) + .map( ( node ) => toWpContentRelativePath( node.path ) ) + .filter( ( nodePath ): nodePath is string => Boolean( nodePath ) ); + + return { onlyPaths: [ ...new Set( onlyPaths ) ], skipDatabase }; +}; diff --git a/apps/ui/src/components/site-dropdown/index.test.tsx b/apps/ui/src/components/site-dropdown/index.test.tsx index 102f854038..eaab4ea5e5 100644 --- a/apps/ui/src/components/site-dropdown/index.test.tsx +++ b/apps/ui/src/components/site-dropdown/index.test.tsx @@ -34,6 +34,7 @@ vi.mock( '@/components/selective-sync/lib/get-ipc-api', () => ( { vi.mock( './disconnect-site-dialog', () => ( { DisconnectSiteDialog: () => null } ) ); vi.mock( '@/components/selective-sync/lib/convert-tree-to-sync-options', () => ( { convertTreeToPullOptions: () => ( { optionsToSync: [ 'all' ], include_path_list: [] } ), + convertTreeToReprintPullOptions: () => ( { onlyPaths: [], skipDatabase: false } ), convertTreeToPushOptions: () => ( { optionsToSync: [ 'all' ] } ), } ) ); vi.mock( './publish-picker-view', () => ( { PublishPickerView: () => null } ) ); @@ -90,7 +91,9 @@ describe( 'SiteDropdown sync dialog', () => { fireEvent.click( screen.getByRole( 'button', { name: 'Confirm pull' } ) ); expect( pullMutate ).toHaveBeenCalledWith( - expect.objectContaining( { siteId: site.id, remoteSiteId: liveSite.id } ) + expect.objectContaining( { siteId: site.id, remoteSiteId: liveSite.id } ), + // The size check the pull hands back to, once the files are on disk. + expect.objectContaining( { onSuccess: expect.any( Function ) } ) ); await waitFor( () => expect( screen.getByRole( 'button', { name: 'Pull from live' } ) ).toBeInTheDocument() diff --git a/apps/ui/src/components/site-dropdown/index.tsx b/apps/ui/src/components/site-dropdown/index.tsx index 3459363050..19dca23c6b 100644 --- a/apps/ui/src/components/site-dropdown/index.tsx +++ b/apps/ui/src/components/site-dropdown/index.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import * as Menu from '@/components/menu'; import { convertTreeToPullOptions, + convertTreeToReprintPullOptions, convertTreeToPushOptions, } from '@/components/selective-sync/lib/convert-tree-to-sync-options'; import { registerSelectiveSyncConnector } from '@/components/selective-sync/lib/get-ipc-api'; @@ -18,6 +19,7 @@ import { DisconnectSiteDialog } from './disconnect-site-dialog'; import { DropdownTrigger } from './dropdown-trigger'; import { MainView } from './main-view'; import { PublishPickerView } from './publish-picker-view'; +import { PulledSiteTooLargeDialog } from './pulled-site-too-large-dialog'; import styles from './style.module.css'; import { getSiteDropdownSecondary } from './trigger-secondary'; import { deriveSiteStatus, ensureProtocol, pickLatestSnapshot, pickLiveSite } from './utils'; @@ -52,6 +54,7 @@ export function SiteDropdown( { const reopenAfterDialogRef = useRef( false ); const [ disconnectOpen, setDisconnectOpen ] = useState( false ); const [ syncDialogType, setSyncDialogType ] = useState< 'push' | 'pull' | null >( null ); + const [ pulledSiteTooLarge, setPulledSiteTooLarge ] = useState( false ); const connector = useConnector(); const pushSiteToLive = usePushSiteToLive(); @@ -138,12 +141,28 @@ export function SiteDropdown( { const handleDialogPull = ( tree: TreeNode[] ) => { if ( ! liveSite ) return; const { optionsToSync, include_path_list: includePathList } = convertTreeToPullOptions( tree ); + // Both engines' forms of the same selection travel together — which one + // is used is decided where the pull runs, not here. + const { onlyPaths, skipDatabase } = convertTreeToReprintPullOptions( tree ); startSyncFromDialog( () => - pullSiteFromLive.mutate( { - siteId: site.id, - remoteSiteId: liveSite.id, - options: { optionsToSync, includePathList }, - } ) + pullSiteFromLive.mutate( + { + siteId: site.id, + remoteSiteId: liveSite.id, + options: { optionsToSync, includePathList, onlyPaths, skipDatabase }, + }, + { + // Only measurable once the files are on disk: a Reprint pull + // streams the site in pieces, so there is no archive to size up + // front the way the Jetpack pull does. + onSuccess: () => { + void connector + .isSiteOverPushSizeLimit( site.id ) + .then( setPulledSiteTooLarge ) + .catch( () => undefined ); + }, + } + ) ); }; @@ -202,6 +221,10 @@ export function SiteDropdown( { onOpenChange={ setDisconnectOpen } /> ) : null } + { liveSite && syncDialogType ? ( { + it( 'names the limit and hedges, since the tally is of uncompressed files', () => { + render( ); + + expect( screen.getByText( /over 5 GB/ ) ).toBeInTheDocument(); + expect( screen.getByText( /may prevent you from pushing it back/ ) ).toBeInTheDocument(); + } ); + + it( 'only dismisses, since the pull has already finished', async () => { + const onOpenChange = vi.fn(); + render( ); + + expect( screen.queryByRole( 'button', { name: /cancel/i } ) ).not.toBeInTheDocument(); + await userEvent.click( screen.getByRole( 'button', { name: 'Got it' } ) ); + + expect( onOpenChange.mock.calls[ 0 ][ 0 ] ).toBe( false ); + } ); + + it( 'renders nothing while closed', () => { + render( ); + + expect( screen.queryByText( /over 5 GB/ ) ).not.toBeInTheDocument(); + } ); +} ); diff --git a/apps/ui/src/components/site-dropdown/pulled-site-too-large-dialog.tsx b/apps/ui/src/components/site-dropdown/pulled-site-too-large-dialog.tsx new file mode 100644 index 0000000000..a865cc4f25 --- /dev/null +++ b/apps/ui/src/components/site-dropdown/pulled-site-too-large-dialog.tsx @@ -0,0 +1,44 @@ +import { SYNC_PUSH_SIZE_LIMIT_GB } from '@studio/common/lib/sync/constants'; +import { __, sprintf } from '@wordpress/i18n'; +import { Dialog } from '@wordpress/ui'; +import styles from './pulled-site-too-large-dialog.module.css'; + +type Props = { + open: boolean; + onOpenChange: ( open: boolean ) => void; +}; + +/** + * Shown after a pull that brought down more than the push limit allows. + * + * It reports rather than asks: the pull has already finished by the time the + * size is known, so there is nothing left to cancel. "May" is deliberate — the + * measurement is of uncompressed files, while the limit applies to a gzipped + * archive, so it over-reports. See `isSiteOverPushSizeLimit`. + */ +export function PulledSiteTooLargeDialog( { open, onOpenChange }: Props ) { + return ( + + + + { __( 'This site may be too large to push back' ) } + + +

+ { sprintf( + __( + 'The site you pulled is over %d GB, which may prevent you from pushing it back to WordPress.com. Removing unused media, plugins or themes from wp-content will bring it down.' + ), + SYNC_PUSH_SIZE_LIMIT_GB + ) } +

+ + + + { __( 'Got it' ) } + + + + + ); +} diff --git a/apps/ui/src/data/core/connectors/hosted/index.ts b/apps/ui/src/data/core/connectors/hosted/index.ts index b8e7390228..227e69bac5 100644 --- a/apps/ui/src/data/core/connectors/hosted/index.ts +++ b/apps/ui/src/data/core/connectors/hosted/index.ts @@ -294,6 +294,9 @@ export function createHostedConnector( { apiBaseUrl }: HostedConnectorOptions ): async getDirectorySize(): Promise< never > { throw new UnsupportedError( 'getDirectorySize' ); }, + async isSiteOverPushSizeLimit(): Promise< never > { + throw new UnsupportedError( 'isSiteOverPushSizeLimit' ); + }, async getFileSize(): Promise< never > { throw new UnsupportedError( 'getFileSize' ); }, diff --git a/apps/ui/src/data/core/connectors/ipc/index.ts b/apps/ui/src/data/core/connectors/ipc/index.ts index c5f68d5161..682816e3f4 100644 --- a/apps/ui/src/data/core/connectors/ipc/index.ts +++ b/apps/ui/src/data/core/connectors/ipc/index.ts @@ -677,6 +677,10 @@ export function createIpcConnector(): Connector { return ( await ipcApi.getFileSize( siteId, path ) ) as number; }, + async isSiteOverPushSizeLimit( siteId ): Promise< boolean > { + return ( await ipcApi.isSiteOverPushSizeLimit( siteId ) ) as boolean; + }, + async getIsMultisite( siteId ): Promise< boolean | undefined > { return ( await ipcApi.getIsMultisite( siteId ) ) as boolean | undefined; }, diff --git a/apps/ui/src/data/core/connectors/local/index.ts b/apps/ui/src/data/core/connectors/local/index.ts index 0cfe738bd8..f4d2ab0fc9 100644 --- a/apps/ui/src/data/core/connectors/local/index.ts +++ b/apps/ui/src/data/core/connectors/local/index.ts @@ -666,6 +666,14 @@ export function createLocalConnector( { apiBaseUrl }: LocalConnectorOptions ): C async getFileSize() { return 0; }, + // Unlike the per-file lookups above, the server can answer this one, so + // the browser UI warns about an oversized pull exactly as the desktop does. + async isSiteOverPushSizeLimit( siteId ) { + const { overLimit } = await api< { overLimit: boolean } >( + `/sites/${ siteId }/push-size-over-limit` + ); + return overLimit; + }, async getIsMultisite() { return undefined; }, diff --git a/apps/ui/src/data/core/types.ts b/apps/ui/src/data/core/types.ts index 15b1280b80..5ed1f07b79 100644 --- a/apps/ui/src/data/core/types.ts +++ b/apps/ui/src/data/core/types.ts @@ -354,6 +354,10 @@ export interface Connector { getDirectorySize( siteId: string, path: string[] ): Promise< number >; getFileSize( siteId: string, path: string[] ): Promise< number >; getIsMultisite( siteId: string ): Promise< boolean | undefined >; + // Whether a just-pulled site is too big to push back. Answered after a pull + // rather than before, because a Reprint pull has no single archive to size + // up front. See `isSiteOverPushSizeLimit` in packages/common. + isSiteOverPushSizeLimit( siteId: string ): Promise< boolean >; // URL to open in the browser when the user wants to publish a site that // isn't connected to WordPress.com yet (checkout + deep-link back to the // desktop app). Returns `undefined` when the connector can't provide one. diff --git a/apps/ui/src/data/queries/use-sync-site.ts b/apps/ui/src/data/queries/use-sync-site.ts index bf013a795e..77f7d0d83f 100644 --- a/apps/ui/src/data/queries/use-sync-site.ts +++ b/apps/ui/src/data/queries/use-sync-site.ts @@ -144,6 +144,9 @@ export function usePullSiteFromLive() { ) : __( "Studio couldn't copy the live site. Try again." ); reportSyncError( siteId, 'pull', message ); + // A failed pull can still have stopped the server and half-written + // the site, so the list is as stale here as it is on success. + void queryClient.invalidateQueries( { queryKey: SITES_QUERY_KEY } ); toast.error( __( "Pull didn't complete" ), { description: message, action: canOpenLogs diff --git a/docs/design-docs/cli.md b/docs/design-docs/cli.md index 1eefe77175..4a788b6378 100644 --- a/docs/design-docs/cli.md +++ b/docs/design-docs/cli.md @@ -56,12 +56,12 @@ Long-term, we might want to move in that direction, but for now, we are still bu ### Pulling a remote site (`pull-reprint`) -`studio pull-reprint` refreshes an **existing** local Studio site from a connected WordPress.com or Pressable source using the reprint pull tool. It is a state-transition on a site, not a site creator — the same shape as the WordPress.com sync `pull`. Third-party WordPress hosts are not supported: a source URL must resolve to a site returned by the user's authenticated WordPress.com Jetpack API site list. +`studio pull-reprint` refreshes an **existing** local Studio site from a connected WordPress.com or Pressable source using the reprint pull tool. It is a state-transition on a site, not a site creator — the same shape as the WordPress.com sync `pull`. Third-party WordPress hosts are not supported: the source must resolve to a site returned by the user's authenticated WordPress.com Jetpack API site list. The flow is: 1. `studio create` — create the local site (a full `SiteData` record plus a blank WordPress install). This is a prerequisite; `pull-reprint` never creates a site. -2. `studio pull-reprint --path --url ` — pull the remote into the local site resolved by `--path`. `--url` identifies only the remote source; if omitted, `pull-reprint` first reuses the site's saved `reprintOrigin.remoteUrl`, and if there is no saved origin, a WordPress.com/Pressable source picker runs (Pressable support is still WIP). The matched remote must be `syncable`. Each run rotates a fresh Reprint secret through the WordPress.com API, enables the exporter, then runs preflight once. The pull is idempotent: re-running it resumes an interrupted pull or performs a delta re-pull of an already-imported site. +2. `studio pull-reprint --path --remote-site ` — pull the remote into the local site resolved by `--path`. `--remote-site` identifies only the remote source and takes the same values as `pull`'s and `push`'s option of the same name (a site URL or a WordPress.com site ID, resolved by the shared `findSyncSiteByIdentifier`); if omitted, `pull-reprint` first reuses the site's saved `reprintOrigin.remoteUrl`, and if there is no saved origin, a WordPress.com/Pressable source picker runs (Pressable support is still WIP). The matched remote must be `syncable`. Each run rotates a fresh Reprint secret through the WordPress.com API, enables the exporter, then runs preflight once. The pull is idempotent: re-running it resumes an interrupted pull or performs a delta re-pull of an already-imported site. 3. `studio delete --path ` — the only teardown path. It trashes the site folder and the site's `technicalSiteDirectory`, which for a reprint-pulled site is the `siteId`-keyed scratch under `~/.studio/pulls/` (reprint's `.import-state.json`, the preflight cache, and the raw/runtime working dirs). `pull-reprint` records `technicalSiteDirectory` on the site at pull *start*, so the scratch is cleaned up even for a pull that failed before linking. There is no `--abort` verb. #### State model diff --git a/packages/common/lib/sync/pull-engine.ts b/packages/common/lib/sync/pull-engine.ts new file mode 100644 index 0000000000..ee0d3fd935 --- /dev/null +++ b/packages/common/lib/sync/pull-engine.ts @@ -0,0 +1,19 @@ +import { readAppConfig } from '@studio/common/lib/app-config'; +import type { PullEngine } from '@studio/common/types/sync'; + +/** + * Which engine a pull should use, read from the `reprintPull` beta feature the + * desktop app writes to `app.json`. + * + * The agentic UI runs against two backends — the Electron main process and the + * `studio ui` server — and both have to reach the same answer, so the default + * for an unset flag lives here rather than in either caller. + */ +export const DEFAULT_PULL_ENGINE: PullEngine = 'jetpack'; + +export async function resolvePullEngine(): Promise< PullEngine > { + const betaFeatures = ( await readAppConfig() ).betaFeatures as + | { reprintPull?: boolean } + | undefined; + return betaFeatures?.reprintPull ? 'reprint' : DEFAULT_PULL_ENGINE; +} diff --git a/packages/common/lib/sync/pull-size-warning.ts b/packages/common/lib/sync/pull-size-warning.ts new file mode 100644 index 0000000000..c9f962de51 --- /dev/null +++ b/packages/common/lib/sync/pull-size-warning.ts @@ -0,0 +1,34 @@ +import path from 'node:path'; +import { createDeployIgnoreFilter } from '@studio/common/lib/deploy-ignore'; +import { calculateDirectorySizeForArchive } from '@studio/common/lib/fs-utils'; +import { SYNC_PUSH_SIZE_LIMIT_BYTES } from '@studio/common/lib/sync/constants'; + +/** + * Whether a just-pulled site looks too big to push back. + * + * The Jetpack pull can ask before downloading, because there is a single + * backup archive whose `Content-Length` it can read. Reprint streams the site + * in pieces with no such total, so the only cheap answer is to measure what + * landed on disk and tell the user afterwards. + * + * That tally is deliberately not the number the push limit applies to: push + * uploads a gzipped archive, so this over-reports — a lot for a text-heavy + * site, barely at all for a media-heavy one whose uploads are already + * compressed. It is left uncorrected on purpose. Over-reporting costs a false + * alarm; under-reporting would stay quiet and let the user hit the ceiling at + * push time, which is the thing the warning exists to prevent. The wording it + * drives says "may", for the same reason. + * + * Written to be deleted: when push moves to Reprint there is no single upload + * to bind a ceiling to, and this check goes away rather than getting sharper. + */ +export async function isSiteOverPushSizeLimit( sitePath: string ): Promise< boolean > { + const deployIgnore = await createDeployIgnoreFilter( sitePath ); + const wpContentSize = await calculateDirectorySizeForArchive( + path.join( sitePath, 'wp-content' ), + deployIgnore, + 'wp-content' + ); + + return wpContentSize > SYNC_PUSH_SIZE_LIMIT_BYTES; +} diff --git a/packages/common/lib/sync/tests/pull-size-warning.test.ts b/packages/common/lib/sync/tests/pull-size-warning.test.ts new file mode 100644 index 0000000000..ed1342fa07 --- /dev/null +++ b/packages/common/lib/sync/tests/pull-size-warning.test.ts @@ -0,0 +1,58 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { SYNC_PUSH_SIZE_LIMIT_BYTES } from '@studio/common/lib/sync/constants'; +import { isSiteOverPushSizeLimit } from '@studio/common/lib/sync/pull-size-warning'; + +const { calculateDirectorySizeForArchive } = vi.hoisted( () => ( { + calculateDirectorySizeForArchive: vi.fn(), +} ) ); + +// The limit is 5 GB, so the tally is stubbed rather than written to disk. +vi.mock( '@studio/common/lib/fs-utils', () => ( { calculateDirectorySizeForArchive } ) ); + +function makeSite(): string { + const sitePath = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-pull-size-' ) ); + fs.mkdirSync( path.join( sitePath, 'wp-content' ), { recursive: true } ); + return sitePath; +} + +afterEach( () => { + vi.clearAllMocks(); +} ); + +describe( 'isSiteOverPushSizeLimit', () => { + it( 'measures wp-content through the deploy-ignore filter', async () => { + const sitePath = makeSite(); + calculateDirectorySizeForArchive.mockResolvedValue( 1024 ); + + await isSiteOverPushSizeLimit( sitePath ); + + expect( calculateDirectorySizeForArchive ).toHaveBeenCalledWith( + path.join( sitePath, 'wp-content' ), + expect.anything(), + 'wp-content' + ); + + fs.rmSync( sitePath, { recursive: true, force: true } ); + } ); + + it( 'reports a site over the limit', async () => { + const sitePath = makeSite(); + calculateDirectorySizeForArchive.mockResolvedValue( SYNC_PUSH_SIZE_LIMIT_BYTES + 1 ); + + expect( await isSiteOverPushSizeLimit( sitePath ) ).toBe( true ); + + fs.rmSync( sitePath, { recursive: true, force: true } ); + } ); + + it( 'stays quiet for a site exactly at the limit', async () => { + const sitePath = makeSite(); + calculateDirectorySizeForArchive.mockResolvedValue( SYNC_PUSH_SIZE_LIMIT_BYTES ); + + expect( await isSiteOverPushSizeLimit( sitePath ) ).toBe( false ); + + fs.rmSync( sitePath, { recursive: true, force: true } ); + } ); +} ); diff --git a/packages/common/sites/sync.test.ts b/packages/common/sites/sync.test.ts index 9f3fcb433b..2f130c509f 100644 --- a/packages/common/sites/sync.test.ts +++ b/packages/common/sites/sync.test.ts @@ -33,11 +33,104 @@ vi.mock( '@studio/common/lib/sync/constants', async ( importOriginal ) => ( { } ) ); describe( 'pullSite', () => { + it( 'runs the Jetpack-backup `pull` command by default', async () => { + const emitter = new EventEmitter(); + const execute = vi.fn( () => [ emitter, {} ] ) as unknown as ExecuteCliCommand; + + const pulling = pullSite( execute, '/sites/local', 42 ); + emitter.emit( 'success' ); + await pulling; + + expect( execute ).toHaveBeenCalledWith( + [ 'pull', '--path', '/sites/local', '--remote-site', '42', '--options', 'all' ], + { output: 'capture' } + ); + } ); + + it( 'runs `pull-reprint` with the same --remote-site identifier for the reprint engine', async () => { + const emitter = new EventEmitter(); + const execute = vi.fn( () => [ emitter, {} ] ) as unknown as ExecuteCliCommand; + + const pulling = pullSite( execute, '/sites/local', 42, { engine: 'reprint' } ); + emitter.emit( 'success' ); + await pulling; + + expect( execute ).toHaveBeenCalledWith( + [ 'pull-reprint', '--path', '/sites/local', '--remote-site', '42' ], + { output: 'capture' } + ); + } ); + + // The same selection travels in both engines' forms; reprint reads the + // wp-content-relative paths and ignores the backup node ids. + it( 'passes the selection to the reprint engine as --only and --skip-database', async () => { + const emitter = new EventEmitter(); + const execute = vi.fn( () => [ emitter, {} ] ) as unknown as ExecuteCliCommand; + + const pulling = pullSite( execute, '/sites/local', 42, { + engine: 'reprint', + syncOptions: { + optionsToSync: [ 'paths' ], + includePathList: [ 'ZjE6Lw==' ], + onlyPaths: [ 'plugins/akismet', 'themes' ], + skipDatabase: true, + }, + } ); + emitter.emit( 'success' ); + await pulling; + + expect( execute ).toHaveBeenCalledWith( + [ + 'pull-reprint', + '--path', + '/sites/local', + '--remote-site', + '42', + '--only=plugins/akismet', + '--only=themes', + '--skip-database', + ], + { output: 'capture' } + ); + } ); + + it( 'keeps the database and adds no --only when everything is selected', async () => { + const emitter = new EventEmitter(); + const execute = vi.fn( () => [ emitter, {} ] ) as unknown as ExecuteCliCommand; + + const pulling = pullSite( execute, '/sites/local', 42, { + engine: 'reprint', + syncOptions: { onlyPaths: [], skipDatabase: false }, + } ); + emitter.emit( 'success' ); + await pulling; + + expect( execute ).toHaveBeenCalledWith( + [ 'pull-reprint', '--path', '/sites/local', '--remote-site', '42' ], + { output: 'capture' } + ); + } ); + + it( 'leaves the reprint selection out of a jetpack pull', async () => { + const emitter = new EventEmitter(); + const execute = vi.fn( ( _args: string[] ) => [ emitter, {} ] ); + + const pulling = pullSite( execute as unknown as ExecuteCliCommand, '/sites/local', 42, { + syncOptions: { onlyPaths: [ 'plugins/akismet' ], skipDatabase: true }, + } ); + emitter.emit( 'success' ); + await pulling; + + const args = execute.mock.calls[ 0 ][ 0 ]; + expect( args ).not.toContain( '--only=plugins/akismet' ); + expect( args ).not.toContain( '--skip-database' ); + } ); + it( 'forwards live CLI messages and their percentage', async () => { const emitter = new EventEmitter(); const execute = vi.fn( () => [ emitter, {} ] ) as unknown as ExecuteCliCommand; const onProgress = vi.fn(); - const pulling = pullSite( execute, '/sites/local', 42, onProgress ); + const pulling = pullSite( execute, '/sites/local', 42, { emit: onProgress } ); emitter.emit( 'data', { data: { @@ -72,14 +165,7 @@ describe( 'pullSite', () => { const child = { pid: 4242 }; const execute = vi.fn( () => [ emitter, child ] ) as unknown as ExecuteCliCommand; const controller = new AbortController(); - const pulling = pullSite( - execute, - '/sites/local', - 42, - undefined, - undefined, - controller.signal - ); + const pulling = pullSite( execute, '/sites/local', 42, { signal: controller.signal } ); controller.abort(); @@ -90,7 +176,7 @@ describe( 'pullSite', () => { it( 'rejects immediately when the signal is already aborted', async () => { const execute = vi.fn() as unknown as ExecuteCliCommand; await expect( - pullSite( execute, '/sites/local', 42, undefined, undefined, AbortSignal.abort() ) + pullSite( execute, '/sites/local', 42, { signal: AbortSignal.abort() } ) ).rejects.toSatisfy( isSyncCancelledError ); expect( execute ).not.toHaveBeenCalled(); } ); @@ -99,9 +185,8 @@ describe( 'pullSite', () => { const emitter = new EventEmitter(); const execute = vi.fn( () => [ emitter, {} ] ) as unknown as ExecuteCliCommand; const includePathList = [ 'cjE6,ZjE6Lw==', 'cjI6,ZjI6Lw==', 'ZjM6Lw==' ]; - const pulling = pullSite( execute, '/sites/local', 42, undefined, { - optionsToSync: [ 'paths' ], - includePathList, + const pulling = pullSite( execute, '/sites/local', 42, { + syncOptions: { optionsToSync: [ 'paths' ], includePathList }, } ); emitter.emit( 'success' ); diff --git a/packages/common/sites/sync.ts b/packages/common/sites/sync.ts index 46c31f8a22..f8d4f3c5d9 100644 --- a/packages/common/sites/sync.ts +++ b/packages/common/sites/sync.ts @@ -14,6 +14,7 @@ import { createTusUpload } from '@studio/common/lib/sync/tus-upload'; import type { ExecuteCliCommand } from '@studio/common/lib/cli-process'; import type { ImportResponse, + PullEngine, PullSiteProgress, PullSyncOptions, PushOutput, @@ -229,41 +230,71 @@ function getExportMode( optionsToSync: SyncOption[] | undefined ): 'full' | 'con } /** - * Pull a local site from its connected WordPress.com live site via the CLI - * `pull` command. Exchanges everything (`--options all`) unless selective - * options are provided. Resolves on success, rejects on failure. + * Pull a local site from its connected WordPress.com live site via the CLI. + * `jetpack` runs `pull`, exchanging everything (`--options all`) unless + * selective options are provided; `reprint` runs `pull-reprint`, which pulls + * everything when driven non-interactively — except when it resumes a pull + * that was interrupted mid-flight, which reprint requires to keep its original + * content selection. Both commands take the same `--remote-site` identifier. + * Resolves on success, rejects on failure. + * + * Both engines honour `syncOptions`, through different flags: `jetpack` + * selects by backup node id (`--include-path-list`), reprint by + * wp-content-relative path (`--only`) plus `--skip-database`. The caller + * carries both forms, since the engine is resolved here rather than in the UI. + * + * Only the arguments differ per engine. The progress parsing below is the + * CLI-wide `reportProgress` envelope plus the `(N%)` token that `pull`, + * `pull-reprint`, `push` and `import` all emit, so both engines share it. */ export function pullSite( executeCliCommand: ExecuteCliCommand, siteFolder: string, remoteSiteId: number, - emit?: ( output: PullSiteProgress ) => void, - options?: PullSyncOptions, - signal?: AbortSignal + { + emit, + engine = 'jetpack', + syncOptions, + signal, + }: { + emit?: ( output: PullSiteProgress ) => void; + engine?: PullEngine; + syncOptions?: PullSyncOptions; + signal?: AbortSignal; + } = {} ): Promise< void > { + const target = [ '--path', siteFolder, '--remote-site', String( remoteSiteId ) ]; + const args = + engine === 'reprint' + ? [ + 'pull-reprint', + ...target, + // Reprint's include-list. Paths are wp-content-relative; the + // command turns them into `:wp-content:` sources itself. + ...( syncOptions?.onlyPaths ?? [] ).map( ( onlyPath ) => `--only=${ onlyPath }` ), + ...( syncOptions?.skipDatabase ? [ '--skip-database' ] : [] ), + ] + : [ + 'pull', + ...target, + '--options', + ( syncOptions?.optionsToSync?.length ? syncOptions.optionsToSync : [ 'all' ] ).join( + ',' + ), + // Pass each backup node id as its own argv value — ids can contain + // commas (e.g. themes `cjE6,ZjE6Lw==`), so a join/split would corrupt them. + ...( syncOptions?.includePathList?.length + ? [ '--include-path-list', ...syncOptions.includePathList ] + : [] ), + ]; + return new Promise( ( resolve, reject ) => { if ( signal?.aborted ) { reject( new SyncCancelledError() ); return; } - const [ emitter, child ] = executeCliCommand( - [ - 'pull', - '--path', - siteFolder, - '--remote-site', - String( remoteSiteId ), - '--options', - ( options?.optionsToSync?.length ? options.optionsToSync : [ 'all' ] ).join( ',' ), - // Pass each backup node id as its own argv value — ids can contain - // commas (e.g. themes `cjE6,ZjE6Lw==`), so a join/split would corrupt them. - ...( options?.includePathList?.length - ? [ '--include-path-list', ...options.includePathList ] - : [] ), - ], - { output: 'capture' } - ); + const [ emitter, child ] = executeCliCommand( args, { output: 'capture' } ); const stopPull = () => { // Reject even if the kill fails — otherwise a cancel would hang the diff --git a/packages/common/types/sync.ts b/packages/common/types/sync.ts index be2959faaa..b33ca5d364 100644 --- a/packages/common/types/sync.ts +++ b/packages/common/types/sync.ts @@ -122,6 +122,13 @@ export type PullSiteProgress = { action?: string; }; +/** + * Which CLI command backs a pull: the shipped Jetpack-backup `pull`, or the + * streaming `pull-reprint`. The desktop app picks this per pull from the + * `reprintPull` beta feature; every other surface stays on `jetpack`. + */ +export type PullEngine = 'jetpack' | 'reprint'; + // Pull backup API schemas export const pullSiteResponseSchema = z.object( { success: z.boolean(), @@ -191,4 +198,12 @@ export type PushSyncOptions = { export type PullSyncOptions = { optionsToSync?: SyncOption[]; includePathList?: string[]; + /** + * The same selection expressed for the `reprint` engine, which selects by + * wp-content-relative path rather than by Jetpack backup node id. Both + * forms are carried because the engine is resolved further down, in the + * main process or the `studio ui` server. + */ + onlyPaths?: string[]; + skipDatabase?: boolean; }; diff --git a/scripts/download-wp-server-files.ts b/scripts/download-wp-server-files.ts index 3beac90627..faf27562b9 100644 --- a/scripts/download-wp-server-files.ts +++ b/scripts/download-wp-server-files.ts @@ -131,7 +131,7 @@ const FILES_TO_DOWNLOAD: FileToDownload[] = [ { name: 'reprint', description: `reprint.phar`, - getUrl: () => 'https://github.com/WordPress/reprint/releases/download/v0.10.1/reprint.phar', + getUrl: () => 'https://github.com/WordPress/reprint/releases/download/v0.10.4/reprint.phar', destinationPath: path.join( WP_SERVER_FILES_PATH, 'reprint' ), }, ];